PR target/16286
[official-gcc.git] / gcc / loop.c
blob995df4863c2a994e1d6d6de41fe44ff7aac040ec
1 /* Perform various loop optimizations, including strength reduction.
2 Copyright (C) 1987, 1988, 1989, 1991, 1992, 1993, 1994, 1995, 1996, 1997,
3 1998, 1999, 2000, 2001, 2002, 2003, 2004 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 2, or (at your option) any later
10 version.
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING. If not, write to the Free
19 Software Foundation, 59 Temple Place - Suite 330, Boston, MA
20 02111-1307, USA. */
22 /* This is the loop optimization pass of the compiler.
23 It finds invariant computations within loops and moves them
24 to the beginning of the loop. Then it identifies basic and
25 general induction variables.
27 Basic induction variables (BIVs) are a pseudo registers which are set within
28 a loop only by incrementing or decrementing its value. General induction
29 variables (GIVs) are pseudo registers with a value which is a linear function
30 of a basic induction variable. BIVs are recognized by `basic_induction_var';
31 GIVs by `general_induction_var'.
33 Once induction variables are identified, strength reduction is applied to the
34 general induction variables, and induction variable elimination is applied to
35 the basic induction variables.
37 It also finds cases where
38 a register is set within the loop by zero-extending a narrower value
39 and changes these to zero the entire register once before the loop
40 and merely copy the low part within the loop.
42 Most of the complexity is in heuristics to decide when it is worth
43 while to do these things. */
45 #include "config.h"
46 #include "system.h"
47 #include "coretypes.h"
48 #include "tm.h"
49 #include "rtl.h"
50 #include "tm_p.h"
51 #include "function.h"
52 #include "expr.h"
53 #include "hard-reg-set.h"
54 #include "basic-block.h"
55 #include "insn-config.h"
56 #include "regs.h"
57 #include "recog.h"
58 #include "flags.h"
59 #include "real.h"
60 #include "cselib.h"
61 #include "except.h"
62 #include "toplev.h"
63 #include "predict.h"
64 #include "insn-flags.h"
65 #include "optabs.h"
66 #include "cfgloop.h"
67 #include "ggc.h"
69 /* Get the loop info pointer of a loop. */
70 #define LOOP_INFO(LOOP) ((struct loop_info *) (LOOP)->aux)
72 /* Get a pointer to the loop movables structure. */
73 #define LOOP_MOVABLES(LOOP) (&LOOP_INFO (LOOP)->movables)
75 /* Get a pointer to the loop registers structure. */
76 #define LOOP_REGS(LOOP) (&LOOP_INFO (LOOP)->regs)
78 /* Get a pointer to the loop induction variables structure. */
79 #define LOOP_IVS(LOOP) (&LOOP_INFO (LOOP)->ivs)
81 /* Get the luid of an insn. Catch the error of trying to reference the LUID
82 of an insn added during loop, since these don't have LUIDs. */
84 #define INSN_LUID(INSN) \
85 (INSN_UID (INSN) < max_uid_for_loop ? uid_luid[INSN_UID (INSN)] \
86 : (abort (), -1))
88 #define REGNO_FIRST_LUID(REGNO) \
89 (REGNO_FIRST_UID (REGNO) < max_uid_for_loop \
90 ? uid_luid[REGNO_FIRST_UID (REGNO)] \
91 : 0)
92 #define REGNO_LAST_LUID(REGNO) \
93 (REGNO_LAST_UID (REGNO) < max_uid_for_loop \
94 ? uid_luid[REGNO_LAST_UID (REGNO)] \
95 : INT_MAX)
97 /* A "basic induction variable" or biv is a pseudo reg that is set
98 (within this loop) only by incrementing or decrementing it. */
99 /* A "general induction variable" or giv is a pseudo reg whose
100 value is a linear function of a biv. */
102 /* Bivs are recognized by `basic_induction_var';
103 Givs by `general_induction_var'. */
105 /* An enum for the two different types of givs, those that are used
106 as memory addresses and those that are calculated into registers. */
107 enum g_types
109 DEST_ADDR,
110 DEST_REG
114 /* A `struct induction' is created for every instruction that sets
115 an induction variable (either a biv or a giv). */
117 struct induction
119 rtx insn; /* The insn that sets a biv or giv */
120 rtx new_reg; /* New register, containing strength reduced
121 version of this giv. */
122 rtx src_reg; /* Biv from which this giv is computed.
123 (If this is a biv, then this is the biv.) */
124 enum g_types giv_type; /* Indicate whether DEST_ADDR or DEST_REG */
125 rtx dest_reg; /* Destination register for insn: this is the
126 register which was the biv or giv.
127 For a biv, this equals src_reg.
128 For a DEST_ADDR type giv, this is 0. */
129 rtx *location; /* Place in the insn where this giv occurs.
130 If GIV_TYPE is DEST_REG, this is 0. */
131 /* For a biv, this is the place where add_val
132 was found. */
133 enum machine_mode mode; /* The mode of this biv or giv */
134 rtx mem; /* For DEST_ADDR, the memory object. */
135 rtx mult_val; /* Multiplicative factor for src_reg. */
136 rtx add_val; /* Additive constant for that product. */
137 int benefit; /* Gain from eliminating this insn. */
138 rtx final_value; /* If the giv is used outside the loop, and its
139 final value could be calculated, it is put
140 here, and the giv is made replaceable. Set
141 the giv to this value before the loop. */
142 unsigned combined_with; /* The number of givs this giv has been
143 combined with. If nonzero, this giv
144 cannot combine with any other giv. */
145 unsigned replaceable : 1; /* 1 if we can substitute the strength-reduced
146 variable for the original variable.
147 0 means they must be kept separate and the
148 new one must be copied into the old pseudo
149 reg each time the old one is set. */
150 unsigned not_replaceable : 1; /* Used to prevent duplicating work. This is
151 1 if we know that the giv definitely can
152 not be made replaceable, in which case we
153 don't bother checking the variable again
154 even if further info is available.
155 Both this and the above can be zero. */
156 unsigned ignore : 1; /* 1 prohibits further processing of giv */
157 unsigned always_computable : 1;/* 1 if this value is computable every
158 iteration. */
159 unsigned always_executed : 1; /* 1 if this set occurs each iteration. */
160 unsigned maybe_multiple : 1; /* Only used for a biv and 1 if this biv
161 update may be done multiple times per
162 iteration. */
163 unsigned cant_derive : 1; /* For giv's, 1 if this giv cannot derive
164 another giv. This occurs in many cases
165 where a giv's lifetime spans an update to
166 a biv. */
167 unsigned maybe_dead : 1; /* 1 if this giv might be dead. In that case,
168 we won't use it to eliminate a biv, it
169 would probably lose. */
170 unsigned auto_inc_opt : 1; /* 1 if this giv had its increment output next
171 to it to try to form an auto-inc address. */
172 unsigned shared : 1;
173 unsigned no_const_addval : 1; /* 1 if add_val does not contain a const. */
174 int lifetime; /* Length of life of this giv */
175 rtx derive_adjustment; /* If nonzero, is an adjustment to be
176 subtracted from add_val when this giv
177 derives another. This occurs when the
178 giv spans a biv update by incrementation. */
179 rtx ext_dependent; /* If nonzero, is a sign or zero extension
180 if a biv on which this giv is dependent. */
181 struct induction *next_iv; /* For givs, links together all givs that are
182 based on the same biv. For bivs, links
183 together all biv entries that refer to the
184 same biv register. */
185 struct induction *same; /* For givs, if the giv has been combined with
186 another giv, this points to the base giv.
187 The base giv will have COMBINED_WITH nonzero.
188 For bivs, if the biv has the same LOCATION
189 than another biv, this points to the base
190 biv. */
191 struct induction *same_insn; /* If there are multiple identical givs in
192 the same insn, then all but one have this
193 field set, and they all point to the giv
194 that doesn't have this field set. */
195 rtx last_use; /* For a giv made from a biv increment, this is
196 a substitute for the lifetime information. */
200 /* A `struct iv_class' is created for each biv. */
202 struct iv_class
204 unsigned int regno; /* Pseudo reg which is the biv. */
205 int biv_count; /* Number of insns setting this reg. */
206 struct induction *biv; /* List of all insns that set this reg. */
207 int giv_count; /* Number of DEST_REG givs computed from this
208 biv. The resulting count is only used in
209 check_dbra_loop. */
210 struct induction *giv; /* List of all insns that compute a giv
211 from this reg. */
212 int total_benefit; /* Sum of BENEFITs of all those givs. */
213 rtx initial_value; /* Value of reg at loop start. */
214 rtx initial_test; /* Test performed on BIV before loop. */
215 rtx final_value; /* Value of reg at loop end, if known. */
216 struct iv_class *next; /* Links all class structures together. */
217 rtx init_insn; /* insn which initializes biv, 0 if none. */
218 rtx init_set; /* SET of INIT_INSN, if any. */
219 unsigned incremented : 1; /* 1 if somewhere incremented/decremented */
220 unsigned eliminable : 1; /* 1 if plausible candidate for
221 elimination. */
222 unsigned nonneg : 1; /* 1 if we added a REG_NONNEG note for
223 this. */
224 unsigned reversed : 1; /* 1 if we reversed the loop that this
225 biv controls. */
226 unsigned all_reduced : 1; /* 1 if all givs using this biv have
227 been reduced. */
231 /* Definitions used by the basic induction variable discovery code. */
232 enum iv_mode
234 UNKNOWN_INDUCT,
235 BASIC_INDUCT,
236 NOT_BASIC_INDUCT,
237 GENERAL_INDUCT
241 /* A `struct iv' is created for every register. */
243 struct iv
245 enum iv_mode type;
246 union
248 struct iv_class *class;
249 struct induction *info;
250 } iv;
254 #define REG_IV_TYPE(ivs, n) ivs->regs[n].type
255 #define REG_IV_INFO(ivs, n) ivs->regs[n].iv.info
256 #define REG_IV_CLASS(ivs, n) ivs->regs[n].iv.class
259 struct loop_ivs
261 /* Indexed by register number, contains pointer to `struct
262 iv' if register is an induction variable. */
263 struct iv *regs;
265 /* Size of regs array. */
266 unsigned int n_regs;
268 /* The head of a list which links together (via the next field)
269 every iv class for the current loop. */
270 struct iv_class *list;
274 typedef struct loop_mem_info
276 rtx mem; /* The MEM itself. */
277 rtx reg; /* Corresponding pseudo, if any. */
278 int optimize; /* Nonzero if we can optimize access to this MEM. */
279 } loop_mem_info;
283 struct loop_reg
285 /* Number of times the reg is set during the loop being scanned.
286 During code motion, a negative value indicates a reg that has
287 been made a candidate; in particular -2 means that it is an
288 candidate that we know is equal to a constant and -1 means that
289 it is a candidate not known equal to a constant. After code
290 motion, regs moved have 0 (which is accurate now) while the
291 failed candidates have the original number of times set.
293 Therefore, at all times, == 0 indicates an invariant register;
294 < 0 a conditionally invariant one. */
295 int set_in_loop;
297 /* Original value of set_in_loop; same except that this value
298 is not set negative for a reg whose sets have been made candidates
299 and not set to 0 for a reg that is moved. */
300 int n_times_set;
302 /* Contains the insn in which a register was used if it was used
303 exactly once; contains const0_rtx if it was used more than once. */
304 rtx single_usage;
306 /* Nonzero indicates that the register cannot be moved or strength
307 reduced. */
308 char may_not_optimize;
310 /* Nonzero means reg N has already been moved out of one loop.
311 This reduces the desire to move it out of another. */
312 char moved_once;
316 struct loop_regs
318 int num; /* Number of regs used in table. */
319 int size; /* Size of table. */
320 struct loop_reg *array; /* Register usage info. array. */
321 int multiple_uses; /* Nonzero if a reg has multiple uses. */
326 struct loop_movables
328 /* Head of movable chain. */
329 struct movable *head;
330 /* Last movable in chain. */
331 struct movable *last;
335 /* Information pertaining to a loop. */
337 struct loop_info
339 /* Nonzero if there is a subroutine call in the current loop. */
340 int has_call;
341 /* Nonzero if there is a libcall in the current loop. */
342 int has_libcall;
343 /* Nonzero if there is a non constant call in the current loop. */
344 int has_nonconst_call;
345 /* Nonzero if there is a prefetch instruction in the current loop. */
346 int has_prefetch;
347 /* Nonzero if there is a volatile memory reference in the current
348 loop. */
349 int has_volatile;
350 /* Nonzero if there is a tablejump in the current loop. */
351 int has_tablejump;
352 /* Nonzero if there are ways to leave the loop other than falling
353 off the end. */
354 int has_multiple_exit_targets;
355 /* Nonzero if there is an indirect jump in the current function. */
356 int has_indirect_jump;
357 /* Register or constant initial loop value. */
358 rtx initial_value;
359 /* Register or constant value used for comparison test. */
360 rtx comparison_value;
361 /* Register or constant approximate final value. */
362 rtx final_value;
363 /* Register or constant initial loop value with term common to
364 final_value removed. */
365 rtx initial_equiv_value;
366 /* Register or constant final loop value with term common to
367 initial_value removed. */
368 rtx final_equiv_value;
369 /* Register corresponding to iteration variable. */
370 rtx iteration_var;
371 /* Constant loop increment. */
372 rtx increment;
373 enum rtx_code comparison_code;
374 /* Holds the number of loop iterations. It is zero if the number
375 could not be calculated. Must be unsigned since the number of
376 iterations can be as high as 2^wordsize - 1. For loops with a
377 wider iterator, this number will be zero if the number of loop
378 iterations is too large for an unsigned integer to hold. */
379 unsigned HOST_WIDE_INT n_iterations;
380 int used_count_register;
381 /* The loop iterator induction variable. */
382 struct iv_class *iv;
383 /* List of MEMs that are stored in this loop. */
384 rtx store_mems;
385 /* Array of MEMs that are used (read or written) in this loop, but
386 cannot be aliased by anything in this loop, except perhaps
387 themselves. In other words, if mems[i] is altered during
388 the loop, it is altered by an expression that is rtx_equal_p to
389 it. */
390 loop_mem_info *mems;
391 /* The index of the next available slot in MEMS. */
392 int mems_idx;
393 /* The number of elements allocated in MEMS. */
394 int mems_allocated;
395 /* Nonzero if we don't know what MEMs were changed in the current
396 loop. This happens if the loop contains a call (in which case
397 `has_call' will also be set) or if we store into more than
398 NUM_STORES MEMs. */
399 int unknown_address_altered;
400 /* The above doesn't count any readonly memory locations that are
401 stored. This does. */
402 int unknown_constant_address_altered;
403 /* Count of memory write instructions discovered in the loop. */
404 int num_mem_sets;
405 /* The insn where the first of these was found. */
406 rtx first_loop_store_insn;
407 /* The chain of movable insns in loop. */
408 struct loop_movables movables;
409 /* The registers used the in loop. */
410 struct loop_regs regs;
411 /* The induction variable information in loop. */
412 struct loop_ivs ivs;
413 /* Nonzero if call is in pre_header extended basic block. */
414 int pre_header_has_call;
417 /* Not really meaningful values, but at least something. */
418 #ifndef SIMULTANEOUS_PREFETCHES
419 #define SIMULTANEOUS_PREFETCHES 3
420 #endif
421 #ifndef PREFETCH_BLOCK
422 #define PREFETCH_BLOCK 32
423 #endif
424 #ifndef HAVE_prefetch
425 #define HAVE_prefetch 0
426 #define CODE_FOR_prefetch 0
427 #define gen_prefetch(a,b,c) (abort(), NULL_RTX)
428 #endif
430 /* Give up the prefetch optimizations once we exceed a given threshold.
431 It is unlikely that we would be able to optimize something in a loop
432 with so many detected prefetches. */
433 #define MAX_PREFETCHES 100
434 /* The number of prefetch blocks that are beneficial to fetch at once before
435 a loop with a known (and low) iteration count. */
436 #define PREFETCH_BLOCKS_BEFORE_LOOP_MAX 6
437 /* For very tiny loops it is not worthwhile to prefetch even before the loop,
438 since it is likely that the data are already in the cache. */
439 #define PREFETCH_BLOCKS_BEFORE_LOOP_MIN 2
441 /* Parameterize some prefetch heuristics so they can be turned on and off
442 easily for performance testing on new architectures. These can be
443 defined in target-dependent files. */
445 /* Prefetch is worthwhile only when loads/stores are dense. */
446 #ifndef PREFETCH_ONLY_DENSE_MEM
447 #define PREFETCH_ONLY_DENSE_MEM 1
448 #endif
450 /* Define what we mean by "dense" loads and stores; This value divided by 256
451 is the minimum percentage of memory references that worth prefetching. */
452 #ifndef PREFETCH_DENSE_MEM
453 #define PREFETCH_DENSE_MEM 220
454 #endif
456 /* Do not prefetch for a loop whose iteration count is known to be low. */
457 #ifndef PREFETCH_NO_LOW_LOOPCNT
458 #define PREFETCH_NO_LOW_LOOPCNT 1
459 #endif
461 /* Define what we mean by a "low" iteration count. */
462 #ifndef PREFETCH_LOW_LOOPCNT
463 #define PREFETCH_LOW_LOOPCNT 32
464 #endif
466 /* Do not prefetch for a loop that contains a function call; such a loop is
467 probably not an internal loop. */
468 #ifndef PREFETCH_NO_CALL
469 #define PREFETCH_NO_CALL 1
470 #endif
472 /* Do not prefetch accesses with an extreme stride. */
473 #ifndef PREFETCH_NO_EXTREME_STRIDE
474 #define PREFETCH_NO_EXTREME_STRIDE 1
475 #endif
477 /* Define what we mean by an "extreme" stride. */
478 #ifndef PREFETCH_EXTREME_STRIDE
479 #define PREFETCH_EXTREME_STRIDE 4096
480 #endif
482 /* Define a limit to how far apart indices can be and still be merged
483 into a single prefetch. */
484 #ifndef PREFETCH_EXTREME_DIFFERENCE
485 #define PREFETCH_EXTREME_DIFFERENCE 4096
486 #endif
488 /* Issue prefetch instructions before the loop to fetch data to be used
489 in the first few loop iterations. */
490 #ifndef PREFETCH_BEFORE_LOOP
491 #define PREFETCH_BEFORE_LOOP 1
492 #endif
494 /* Do not handle reversed order prefetches (negative stride). */
495 #ifndef PREFETCH_NO_REVERSE_ORDER
496 #define PREFETCH_NO_REVERSE_ORDER 1
497 #endif
499 /* Prefetch even if the GIV is in conditional code. */
500 #ifndef PREFETCH_CONDITIONAL
501 #define PREFETCH_CONDITIONAL 1
502 #endif
504 #define LOOP_REG_LIFETIME(LOOP, REGNO) \
505 ((REGNO_LAST_LUID (REGNO) - REGNO_FIRST_LUID (REGNO)))
507 #define LOOP_REG_GLOBAL_P(LOOP, REGNO) \
508 ((REGNO_LAST_LUID (REGNO) > INSN_LUID ((LOOP)->end) \
509 || REGNO_FIRST_LUID (REGNO) < INSN_LUID ((LOOP)->start)))
511 #define LOOP_REGNO_NREGS(REGNO, SET_DEST) \
512 ((REGNO) < FIRST_PSEUDO_REGISTER \
513 ? (int) hard_regno_nregs[(REGNO)][GET_MODE (SET_DEST)] : 1)
516 /* Vector mapping INSN_UIDs to luids.
517 The luids are like uids but increase monotonically always.
518 We use them to see whether a jump comes from outside a given loop. */
520 static int *uid_luid;
522 /* Indexed by INSN_UID, contains the ordinal giving the (innermost) loop
523 number the insn is contained in. */
525 static struct loop **uid_loop;
527 /* 1 + largest uid of any insn. */
529 static int max_uid_for_loop;
531 /* Number of loops detected in current function. Used as index to the
532 next few tables. */
534 static int max_loop_num;
536 /* Bound on pseudo register number before loop optimization.
537 A pseudo has valid regscan info if its number is < max_reg_before_loop. */
538 static unsigned int max_reg_before_loop;
540 /* The value to pass to the next call of reg_scan_update. */
541 static int loop_max_reg;
543 /* During the analysis of a loop, a chain of `struct movable's
544 is made to record all the movable insns found.
545 Then the entire chain can be scanned to decide which to move. */
547 struct movable
549 rtx insn; /* A movable insn */
550 rtx set_src; /* The expression this reg is set from. */
551 rtx set_dest; /* The destination of this SET. */
552 rtx dependencies; /* When INSN is libcall, this is an EXPR_LIST
553 of any registers used within the LIBCALL. */
554 int consec; /* Number of consecutive following insns
555 that must be moved with this one. */
556 unsigned int regno; /* The register it sets */
557 short lifetime; /* lifetime of that register;
558 may be adjusted when matching movables
559 that load the same value are found. */
560 short savings; /* Number of insns we can move for this reg,
561 including other movables that force this
562 or match this one. */
563 ENUM_BITFIELD(machine_mode) savemode : 8; /* Nonzero means it is a mode for
564 a low part that we should avoid changing when
565 clearing the rest of the reg. */
566 unsigned int cond : 1; /* 1 if only conditionally movable */
567 unsigned int force : 1; /* 1 means MUST move this insn */
568 unsigned int global : 1; /* 1 means reg is live outside this loop */
569 /* If PARTIAL is 1, GLOBAL means something different:
570 that the reg is live outside the range from where it is set
571 to the following label. */
572 unsigned int done : 1; /* 1 inhibits further processing of this */
574 unsigned int partial : 1; /* 1 means this reg is used for zero-extending.
575 In particular, moving it does not make it
576 invariant. */
577 unsigned int move_insn : 1; /* 1 means that we call emit_move_insn to
578 load SRC, rather than copying INSN. */
579 unsigned int move_insn_first:1;/* Same as above, if this is necessary for the
580 first insn of a consecutive sets group. */
581 unsigned int is_equiv : 1; /* 1 means a REG_EQUIV is present on INSN. */
582 unsigned int insert_temp : 1; /* 1 means we copy to a new pseudo and replace
583 the original insn with a copy from that
584 pseudo, rather than deleting it. */
585 struct movable *match; /* First entry for same value */
586 struct movable *forces; /* An insn that must be moved if this is */
587 struct movable *next;
591 static FILE *loop_dump_stream;
593 /* Forward declarations. */
595 static void invalidate_loops_containing_label (rtx);
596 static void find_and_verify_loops (rtx, struct loops *);
597 static void mark_loop_jump (rtx, struct loop *);
598 static void prescan_loop (struct loop *);
599 static int reg_in_basic_block_p (rtx, rtx);
600 static int consec_sets_invariant_p (const struct loop *, rtx, int, rtx);
601 static int labels_in_range_p (rtx, int);
602 static void count_one_set (struct loop_regs *, rtx, rtx, rtx *);
603 static void note_addr_stored (rtx, rtx, void *);
604 static void note_set_pseudo_multiple_uses (rtx, rtx, void *);
605 static int loop_reg_used_before_p (const struct loop *, rtx, rtx);
606 static rtx find_regs_nested (rtx, rtx);
607 static void scan_loop (struct loop*, int);
608 #if 0
609 static void replace_call_address (rtx, rtx, rtx);
610 #endif
611 static rtx skip_consec_insns (rtx, int);
612 static int libcall_benefit (rtx);
613 static rtx libcall_other_reg (rtx, rtx);
614 static void record_excess_regs (rtx, rtx, rtx *);
615 static void ignore_some_movables (struct loop_movables *);
616 static void force_movables (struct loop_movables *);
617 static void combine_movables (struct loop_movables *, struct loop_regs *);
618 static int num_unmoved_movables (const struct loop *);
619 static int regs_match_p (rtx, rtx, struct loop_movables *);
620 static int rtx_equal_for_loop_p (rtx, rtx, struct loop_movables *,
621 struct loop_regs *);
622 static void add_label_notes (rtx, rtx);
623 static void move_movables (struct loop *loop, struct loop_movables *, int,
624 int);
625 static void loop_movables_add (struct loop_movables *, struct movable *);
626 static void loop_movables_free (struct loop_movables *);
627 static int count_nonfixed_reads (const struct loop *, rtx);
628 static void loop_bivs_find (struct loop *);
629 static void loop_bivs_init_find (struct loop *);
630 static void loop_bivs_check (struct loop *);
631 static void loop_givs_find (struct loop *);
632 static void loop_givs_check (struct loop *);
633 static int loop_biv_eliminable_p (struct loop *, struct iv_class *, int, int);
634 static int loop_giv_reduce_benefit (struct loop *, struct iv_class *,
635 struct induction *, rtx);
636 static void loop_givs_dead_check (struct loop *, struct iv_class *);
637 static void loop_givs_reduce (struct loop *, struct iv_class *);
638 static void loop_givs_rescan (struct loop *, struct iv_class *, rtx *);
639 static void loop_ivs_free (struct loop *);
640 static void strength_reduce (struct loop *, int);
641 static void find_single_use_in_loop (struct loop_regs *, rtx, rtx);
642 static int valid_initial_value_p (rtx, rtx, int, rtx);
643 static void find_mem_givs (const struct loop *, rtx, rtx, int, int);
644 static void record_biv (struct loop *, struct induction *, rtx, rtx, rtx,
645 rtx, rtx *, int, int);
646 static void check_final_value (const struct loop *, struct induction *);
647 static void loop_ivs_dump (const struct loop *, FILE *, int);
648 static void loop_iv_class_dump (const struct iv_class *, FILE *, int);
649 static void loop_biv_dump (const struct induction *, FILE *, int);
650 static void loop_giv_dump (const struct induction *, FILE *, int);
651 static void record_giv (const struct loop *, struct induction *, rtx, rtx,
652 rtx, rtx, rtx, rtx, int, enum g_types, int, int,
653 rtx *);
654 static void update_giv_derive (const struct loop *, rtx);
655 static void check_ext_dependent_givs (const struct loop *, struct iv_class *);
656 static int basic_induction_var (const struct loop *, rtx, enum machine_mode,
657 rtx, rtx, rtx *, rtx *, rtx **);
658 static rtx simplify_giv_expr (const struct loop *, rtx, rtx *, int *);
659 static int general_induction_var (const struct loop *loop, rtx, rtx *, rtx *,
660 rtx *, rtx *, int, int *, enum machine_mode);
661 static int consec_sets_giv (const struct loop *, int, rtx, rtx, rtx, rtx *,
662 rtx *, rtx *, rtx *);
663 static int check_dbra_loop (struct loop *, int);
664 static rtx express_from_1 (rtx, rtx, rtx);
665 static rtx combine_givs_p (struct induction *, struct induction *);
666 static int cmp_combine_givs_stats (const void *, const void *);
667 static void combine_givs (struct loop_regs *, struct iv_class *);
668 static int product_cheap_p (rtx, rtx);
669 static int maybe_eliminate_biv (const struct loop *, struct iv_class *, int,
670 int, int);
671 static int maybe_eliminate_biv_1 (const struct loop *, rtx, rtx,
672 struct iv_class *, int, basic_block, rtx);
673 static int last_use_this_basic_block (rtx, rtx);
674 static void record_initial (rtx, rtx, void *);
675 static void update_reg_last_use (rtx, rtx);
676 static rtx next_insn_in_loop (const struct loop *, rtx);
677 static void loop_regs_scan (const struct loop *, int);
678 static int count_insns_in_loop (const struct loop *);
679 static int find_mem_in_note_1 (rtx *, void *);
680 static rtx find_mem_in_note (rtx);
681 static void load_mems (const struct loop *);
682 static int insert_loop_mem (rtx *, void *);
683 static int replace_loop_mem (rtx *, void *);
684 static void replace_loop_mems (rtx, rtx, rtx, int);
685 static int replace_loop_reg (rtx *, void *);
686 static void replace_loop_regs (rtx insn, rtx, rtx);
687 static void note_reg_stored (rtx, rtx, void *);
688 static void try_copy_prop (const struct loop *, rtx, unsigned int);
689 static void try_swap_copy_prop (const struct loop *, rtx, unsigned int);
690 static rtx check_insn_for_givs (struct loop *, rtx, int, int);
691 static rtx check_insn_for_bivs (struct loop *, rtx, int, int);
692 static rtx gen_add_mult (rtx, rtx, rtx, rtx);
693 static void loop_regs_update (const struct loop *, rtx);
694 static int iv_add_mult_cost (rtx, rtx, rtx, rtx);
695 static int loop_invariant_p (const struct loop *, rtx);
696 static rtx loop_insn_hoist (const struct loop *, rtx);
697 static void loop_iv_add_mult_emit_before (const struct loop *, rtx, rtx, rtx,
698 rtx, basic_block, rtx);
699 static rtx loop_insn_emit_before (const struct loop *, basic_block,
700 rtx, rtx);
701 static int loop_insn_first_p (rtx, rtx);
702 static rtx get_condition_for_loop (const struct loop *, rtx);
703 static void loop_iv_add_mult_sink (const struct loop *, rtx, rtx, rtx, rtx);
704 static void loop_iv_add_mult_hoist (const struct loop *, rtx, rtx, rtx, rtx);
705 static rtx extend_value_for_giv (struct induction *, rtx);
706 static rtx loop_insn_sink (const struct loop *, rtx);
708 static rtx loop_insn_emit_after (const struct loop *, basic_block, rtx, rtx);
709 static rtx loop_call_insn_emit_before (const struct loop *, basic_block,
710 rtx, rtx);
711 static rtx loop_call_insn_hoist (const struct loop *, rtx);
712 static rtx loop_insn_sink_or_swim (const struct loop *, rtx);
714 static void loop_dump_aux (const struct loop *, FILE *, int);
715 static void loop_delete_insns (rtx, rtx);
716 static HOST_WIDE_INT remove_constant_addition (rtx *);
717 static rtx gen_load_of_final_value (rtx, rtx);
718 void debug_ivs (const struct loop *);
719 void debug_iv_class (const struct iv_class *);
720 void debug_biv (const struct induction *);
721 void debug_giv (const struct induction *);
722 void debug_loop (const struct loop *);
723 void debug_loops (const struct loops *);
725 typedef struct loop_replace_args
727 rtx match;
728 rtx replacement;
729 rtx insn;
730 } loop_replace_args;
732 /* Nonzero iff INSN is between START and END, inclusive. */
733 #define INSN_IN_RANGE_P(INSN, START, END) \
734 (INSN_UID (INSN) < max_uid_for_loop \
735 && INSN_LUID (INSN) >= INSN_LUID (START) \
736 && INSN_LUID (INSN) <= INSN_LUID (END))
738 /* Indirect_jump_in_function is computed once per function. */
739 static int indirect_jump_in_function;
740 static int indirect_jump_in_function_p (rtx);
742 static int compute_luids (rtx, rtx, int);
744 static int biv_elimination_giv_has_0_offset (struct induction *,
745 struct induction *, rtx);
747 /* Benefit penalty, if a giv is not replaceable, i.e. must emit an insn to
748 copy the value of the strength reduced giv to its original register. */
749 static int copy_cost;
751 /* Cost of using a register, to normalize the benefits of a giv. */
752 static int reg_address_cost;
754 void
755 init_loop (void)
757 rtx reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
759 reg_address_cost = address_cost (reg, SImode);
761 copy_cost = COSTS_N_INSNS (1);
764 /* Compute the mapping from uids to luids.
765 LUIDs are numbers assigned to insns, like uids,
766 except that luids increase monotonically through the code.
767 Start at insn START and stop just before END. Assign LUIDs
768 starting with PREV_LUID + 1. Return the last assigned LUID + 1. */
769 static int
770 compute_luids (rtx start, rtx end, int prev_luid)
772 int i;
773 rtx insn;
775 for (insn = start, i = prev_luid; insn != end; insn = NEXT_INSN (insn))
777 if (INSN_UID (insn) >= max_uid_for_loop)
778 continue;
779 /* Don't assign luids to line-number NOTEs, so that the distance in
780 luids between two insns is not affected by -g. */
781 if (!NOTE_P (insn)
782 || NOTE_LINE_NUMBER (insn) <= 0)
783 uid_luid[INSN_UID (insn)] = ++i;
784 else
785 /* Give a line number note the same luid as preceding insn. */
786 uid_luid[INSN_UID (insn)] = i;
788 return i + 1;
791 /* Entry point of this file. Perform loop optimization
792 on the current function. F is the first insn of the function
793 and DUMPFILE is a stream for output of a trace of actions taken
794 (or 0 if none should be output). */
796 void
797 loop_optimize (rtx f, FILE *dumpfile, int flags)
799 rtx insn;
800 int i;
801 struct loops loops_data;
802 struct loops *loops = &loops_data;
803 struct loop_info *loops_info;
805 loop_dump_stream = dumpfile;
807 init_recog_no_volatile ();
809 max_reg_before_loop = max_reg_num ();
810 loop_max_reg = max_reg_before_loop;
812 regs_may_share = 0;
814 /* Count the number of loops. */
816 max_loop_num = 0;
817 for (insn = f; insn; insn = NEXT_INSN (insn))
819 if (NOTE_P (insn)
820 && NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
821 max_loop_num++;
824 /* Don't waste time if no loops. */
825 if (max_loop_num == 0)
826 return;
828 loops->num = max_loop_num;
830 /* Get size to use for tables indexed by uids.
831 Leave some space for labels allocated by find_and_verify_loops. */
832 max_uid_for_loop = get_max_uid () + 1 + max_loop_num * 32;
834 uid_luid = xcalloc (max_uid_for_loop, sizeof (int));
835 uid_loop = xcalloc (max_uid_for_loop, sizeof (struct loop *));
837 /* Allocate storage for array of loops. */
838 loops->array = xcalloc (loops->num, sizeof (struct loop));
840 /* Find and process each loop.
841 First, find them, and record them in order of their beginnings. */
842 find_and_verify_loops (f, loops);
844 /* Allocate and initialize auxiliary loop information. */
845 loops_info = xcalloc (loops->num, sizeof (struct loop_info));
846 for (i = 0; i < (int) loops->num; i++)
847 loops->array[i].aux = loops_info + i;
849 /* Now find all register lifetimes. This must be done after
850 find_and_verify_loops, because it might reorder the insns in the
851 function. */
852 reg_scan (f, max_reg_before_loop, 1);
854 /* This must occur after reg_scan so that registers created by gcse
855 will have entries in the register tables.
857 We could have added a call to reg_scan after gcse_main in toplev.c,
858 but moving this call to init_alias_analysis is more efficient. */
859 init_alias_analysis ();
861 /* See if we went too far. Note that get_max_uid already returns
862 one more that the maximum uid of all insn. */
863 if (get_max_uid () > max_uid_for_loop)
864 abort ();
865 /* Now reset it to the actual size we need. See above. */
866 max_uid_for_loop = get_max_uid ();
868 /* find_and_verify_loops has already called compute_luids, but it
869 might have rearranged code afterwards, so we need to recompute
870 the luids now. */
871 compute_luids (f, NULL_RTX, 0);
873 /* Don't leave gaps in uid_luid for insns that have been
874 deleted. It is possible that the first or last insn
875 using some register has been deleted by cross-jumping.
876 Make sure that uid_luid for that former insn's uid
877 points to the general area where that insn used to be. */
878 for (i = 0; i < max_uid_for_loop; i++)
880 uid_luid[0] = uid_luid[i];
881 if (uid_luid[0] != 0)
882 break;
884 for (i = 0; i < max_uid_for_loop; i++)
885 if (uid_luid[i] == 0)
886 uid_luid[i] = uid_luid[i - 1];
888 /* Determine if the function has indirect jump. On some systems
889 this prevents low overhead loop instructions from being used. */
890 indirect_jump_in_function = indirect_jump_in_function_p (f);
892 /* Now scan the loops, last ones first, since this means inner ones are done
893 before outer ones. */
894 for (i = max_loop_num - 1; i >= 0; i--)
896 struct loop *loop = &loops->array[i];
898 if (! loop->invalid && loop->end)
900 scan_loop (loop, flags);
901 ggc_collect ();
905 end_alias_analysis ();
907 /* Clean up. */
908 for (i = 0; i < (int) loops->num; i++)
909 free (loops_info[i].mems);
911 free (uid_luid);
912 free (uid_loop);
913 free (loops_info);
914 free (loops->array);
917 /* Returns the next insn, in execution order, after INSN. START and
918 END are the NOTE_INSN_LOOP_BEG and NOTE_INSN_LOOP_END for the loop,
919 respectively. LOOP->TOP, if non-NULL, is the top of the loop in the
920 insn-stream; it is used with loops that are entered near the
921 bottom. */
923 static rtx
924 next_insn_in_loop (const struct loop *loop, rtx insn)
926 insn = NEXT_INSN (insn);
928 if (insn == loop->end)
930 if (loop->top)
931 /* Go to the top of the loop, and continue there. */
932 insn = loop->top;
933 else
934 /* We're done. */
935 insn = NULL_RTX;
938 if (insn == loop->scan_start)
939 /* We're done. */
940 insn = NULL_RTX;
942 return insn;
945 /* Find any register references hidden inside X and add them to
946 the dependency list DEPS. This is used to look inside CLOBBER (MEM
947 when checking whether a PARALLEL can be pulled out of a loop. */
949 static rtx
950 find_regs_nested (rtx deps, rtx x)
952 enum rtx_code code = GET_CODE (x);
953 if (code == REG)
954 deps = gen_rtx_EXPR_LIST (VOIDmode, x, deps);
955 else
957 const char *fmt = GET_RTX_FORMAT (code);
958 int i, j;
959 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
961 if (fmt[i] == 'e')
962 deps = find_regs_nested (deps, XEXP (x, i));
963 else if (fmt[i] == 'E')
964 for (j = 0; j < XVECLEN (x, i); j++)
965 deps = find_regs_nested (deps, XVECEXP (x, i, j));
968 return deps;
971 /* Optimize one loop described by LOOP. */
973 /* ??? Could also move memory writes out of loops if the destination address
974 is invariant, the source is invariant, the memory write is not volatile,
975 and if we can prove that no read inside the loop can read this address
976 before the write occurs. If there is a read of this address after the
977 write, then we can also mark the memory read as invariant. */
979 static void
980 scan_loop (struct loop *loop, int flags)
982 struct loop_info *loop_info = LOOP_INFO (loop);
983 struct loop_regs *regs = LOOP_REGS (loop);
984 int i;
985 rtx loop_start = loop->start;
986 rtx loop_end = loop->end;
987 rtx p;
988 /* 1 if we are scanning insns that could be executed zero times. */
989 int maybe_never = 0;
990 /* 1 if we are scanning insns that might never be executed
991 due to a subroutine call which might exit before they are reached. */
992 int call_passed = 0;
993 /* Number of insns in the loop. */
994 int insn_count;
995 int tem;
996 rtx temp, update_start, update_end;
997 /* The SET from an insn, if it is the only SET in the insn. */
998 rtx set, set1;
999 /* Chain describing insns movable in current loop. */
1000 struct loop_movables *movables = LOOP_MOVABLES (loop);
1001 /* Ratio of extra register life span we can justify
1002 for saving an instruction. More if loop doesn't call subroutines
1003 since in that case saving an insn makes more difference
1004 and more registers are available. */
1005 int threshold;
1006 int in_libcall;
1008 loop->top = 0;
1010 movables->head = 0;
1011 movables->last = 0;
1013 /* Determine whether this loop starts with a jump down to a test at
1014 the end. This will occur for a small number of loops with a test
1015 that is too complex to duplicate in front of the loop.
1017 We search for the first insn or label in the loop, skipping NOTEs.
1018 However, we must be careful not to skip past a NOTE_INSN_LOOP_BEG
1019 (because we might have a loop executed only once that contains a
1020 loop which starts with a jump to its exit test) or a NOTE_INSN_LOOP_END
1021 (in case we have a degenerate loop).
1023 Note that if we mistakenly think that a loop is entered at the top
1024 when, in fact, it is entered at the exit test, the only effect will be
1025 slightly poorer optimization. Making the opposite error can generate
1026 incorrect code. Since very few loops now start with a jump to the
1027 exit test, the code here to detect that case is very conservative. */
1029 for (p = NEXT_INSN (loop_start);
1030 p != loop_end
1031 && !LABEL_P (p) && ! INSN_P (p)
1032 && (!NOTE_P (p)
1033 || (NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_BEG
1034 && NOTE_LINE_NUMBER (p) != NOTE_INSN_LOOP_END));
1035 p = NEXT_INSN (p))
1038 loop->scan_start = p;
1040 /* If loop end is the end of the current function, then emit a
1041 NOTE_INSN_DELETED after loop_end and set loop->sink to the dummy
1042 note insn. This is the position we use when sinking insns out of
1043 the loop. */
1044 if (NEXT_INSN (loop->end) != 0)
1045 loop->sink = NEXT_INSN (loop->end);
1046 else
1047 loop->sink = emit_note_after (NOTE_INSN_DELETED, loop->end);
1049 /* Set up variables describing this loop. */
1050 prescan_loop (loop);
1051 threshold = (loop_info->has_call ? 1 : 2) * (1 + n_non_fixed_regs);
1053 /* If loop has a jump before the first label,
1054 the true entry is the target of that jump.
1055 Start scan from there.
1056 But record in LOOP->TOP the place where the end-test jumps
1057 back to so we can scan that after the end of the loop. */
1058 if (JUMP_P (p)
1059 /* Loop entry must be unconditional jump (and not a RETURN) */
1060 && any_uncondjump_p (p)
1061 && JUMP_LABEL (p) != 0
1062 /* Check to see whether the jump actually
1063 jumps out of the loop (meaning it's no loop).
1064 This case can happen for things like
1065 do {..} while (0). If this label was generated previously
1066 by loop, we can't tell anything about it and have to reject
1067 the loop. */
1068 && INSN_IN_RANGE_P (JUMP_LABEL (p), loop_start, loop_end))
1070 loop->top = next_label (loop->scan_start);
1071 loop->scan_start = JUMP_LABEL (p);
1074 /* If LOOP->SCAN_START was an insn created by loop, we don't know its luid
1075 as required by loop_reg_used_before_p. So skip such loops. (This
1076 test may never be true, but it's best to play it safe.)
1078 Also, skip loops where we do not start scanning at a label. This
1079 test also rejects loops starting with a JUMP_INSN that failed the
1080 test above. */
1082 if (INSN_UID (loop->scan_start) >= max_uid_for_loop
1083 || !LABEL_P (loop->scan_start))
1085 if (loop_dump_stream)
1086 fprintf (loop_dump_stream, "\nLoop from %d to %d is phony.\n\n",
1087 INSN_UID (loop_start), INSN_UID (loop_end));
1088 return;
1091 /* Allocate extra space for REGs that might be created by load_mems.
1092 We allocate a little extra slop as well, in the hopes that we
1093 won't have to reallocate the regs array. */
1094 loop_regs_scan (loop, loop_info->mems_idx + 16);
1095 insn_count = count_insns_in_loop (loop);
1097 if (loop_dump_stream)
1098 fprintf (loop_dump_stream, "\nLoop from %d to %d: %d real insns.\n",
1099 INSN_UID (loop_start), INSN_UID (loop_end), insn_count);
1101 /* Scan through the loop finding insns that are safe to move.
1102 Set REGS->ARRAY[I].SET_IN_LOOP negative for the reg I being set, so that
1103 this reg will be considered invariant for subsequent insns.
1104 We consider whether subsequent insns use the reg
1105 in deciding whether it is worth actually moving.
1107 MAYBE_NEVER is nonzero if we have passed a conditional jump insn
1108 and therefore it is possible that the insns we are scanning
1109 would never be executed. At such times, we must make sure
1110 that it is safe to execute the insn once instead of zero times.
1111 When MAYBE_NEVER is 0, all insns will be executed at least once
1112 so that is not a problem. */
1114 for (in_libcall = 0, p = next_insn_in_loop (loop, loop->scan_start);
1115 p != NULL_RTX;
1116 p = next_insn_in_loop (loop, p))
1118 if (in_libcall && INSN_P (p) && find_reg_note (p, REG_RETVAL, NULL_RTX))
1119 in_libcall--;
1120 if (NONJUMP_INSN_P (p))
1122 temp = find_reg_note (p, REG_LIBCALL, NULL_RTX);
1123 if (temp)
1124 in_libcall++;
1125 if (! in_libcall
1126 && (set = single_set (p))
1127 && REG_P (SET_DEST (set))
1128 #ifdef PIC_OFFSET_TABLE_REG_CALL_CLOBBERED
1129 && SET_DEST (set) != pic_offset_table_rtx
1130 #endif
1131 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
1133 int tem1 = 0;
1134 int tem2 = 0;
1135 int move_insn = 0;
1136 int insert_temp = 0;
1137 rtx src = SET_SRC (set);
1138 rtx dependencies = 0;
1140 /* Figure out what to use as a source of this insn. If a
1141 REG_EQUIV note is given or if a REG_EQUAL note with a
1142 constant operand is specified, use it as the source and
1143 mark that we should move this insn by calling
1144 emit_move_insn rather that duplicating the insn.
1146 Otherwise, only use the REG_EQUAL contents if a REG_RETVAL
1147 note is present. */
1148 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
1149 if (temp)
1150 src = XEXP (temp, 0), move_insn = 1;
1151 else
1153 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
1154 if (temp && CONSTANT_P (XEXP (temp, 0)))
1155 src = XEXP (temp, 0), move_insn = 1;
1156 if (temp && find_reg_note (p, REG_RETVAL, NULL_RTX))
1158 src = XEXP (temp, 0);
1159 /* A libcall block can use regs that don't appear in
1160 the equivalent expression. To move the libcall,
1161 we must move those regs too. */
1162 dependencies = libcall_other_reg (p, src);
1166 /* For parallels, add any possible uses to the dependencies, as
1167 we can't move the insn without resolving them first.
1168 MEMs inside CLOBBERs may also reference registers; these
1169 count as implicit uses. */
1170 if (GET_CODE (PATTERN (p)) == PARALLEL)
1172 for (i = 0; i < XVECLEN (PATTERN (p), 0); i++)
1174 rtx x = XVECEXP (PATTERN (p), 0, i);
1175 if (GET_CODE (x) == USE)
1176 dependencies
1177 = gen_rtx_EXPR_LIST (VOIDmode, XEXP (x, 0),
1178 dependencies);
1179 else if (GET_CODE (x) == CLOBBER
1180 && MEM_P (XEXP (x, 0)))
1181 dependencies = find_regs_nested (dependencies,
1182 XEXP (XEXP (x, 0), 0));
1186 if (/* The register is used in basic blocks other
1187 than the one where it is set (meaning that
1188 something after this point in the loop might
1189 depend on its value before the set). */
1190 ! reg_in_basic_block_p (p, SET_DEST (set))
1191 /* And the set is not guaranteed to be executed once
1192 the loop starts, or the value before the set is
1193 needed before the set occurs...
1195 ??? Note we have quadratic behavior here, mitigated
1196 by the fact that the previous test will often fail for
1197 large loops. Rather than re-scanning the entire loop
1198 each time for register usage, we should build tables
1199 of the register usage and use them here instead. */
1200 && (maybe_never
1201 || loop_reg_used_before_p (loop, set, p)))
1202 /* It is unsafe to move the set. However, it may be OK to
1203 move the source into a new pseudo, and substitute a
1204 reg-to-reg copy for the original insn.
1206 This code used to consider it OK to move a set of a variable
1207 which was not created by the user and not used in an exit
1208 test.
1209 That behavior is incorrect and was removed. */
1210 insert_temp = 1;
1212 /* Don't try to optimize a MODE_CC set with a constant
1213 source. It probably will be combined with a conditional
1214 jump. */
1215 if (GET_MODE_CLASS (GET_MODE (SET_DEST (set))) == MODE_CC
1216 && CONSTANT_P (src))
1218 /* Don't try to optimize a register that was made
1219 by loop-optimization for an inner loop.
1220 We don't know its life-span, so we can't compute
1221 the benefit. */
1222 else if (REGNO (SET_DEST (set)) >= max_reg_before_loop)
1224 /* Don't move the source and add a reg-to-reg copy:
1225 - with -Os (this certainly increases size),
1226 - if the mode doesn't support copy operations (obviously),
1227 - if the source is already a reg (the motion will gain nothing),
1228 - if the source is a legitimate constant (likewise). */
1229 else if (insert_temp
1230 && (optimize_size
1231 || ! can_copy_p (GET_MODE (SET_SRC (set)))
1232 || REG_P (SET_SRC (set))
1233 || (CONSTANT_P (SET_SRC (set))
1234 && LEGITIMATE_CONSTANT_P (SET_SRC (set)))))
1236 else if ((tem = loop_invariant_p (loop, src))
1237 && (dependencies == 0
1238 || (tem2
1239 = loop_invariant_p (loop, dependencies)) != 0)
1240 && (regs->array[REGNO (SET_DEST (set))].set_in_loop == 1
1241 || (tem1
1242 = consec_sets_invariant_p
1243 (loop, SET_DEST (set),
1244 regs->array[REGNO (SET_DEST (set))].set_in_loop,
1245 p)))
1246 /* If the insn can cause a trap (such as divide by zero),
1247 can't move it unless it's guaranteed to be executed
1248 once loop is entered. Even a function call might
1249 prevent the trap insn from being reached
1250 (since it might exit!) */
1251 && ! ((maybe_never || call_passed)
1252 && may_trap_p (src)))
1254 struct movable *m;
1255 int regno = REGNO (SET_DEST (set));
1257 /* A potential lossage is where we have a case where two insns
1258 can be combined as long as they are both in the loop, but
1259 we move one of them outside the loop. For large loops,
1260 this can lose. The most common case of this is the address
1261 of a function being called.
1263 Therefore, if this register is marked as being used
1264 exactly once if we are in a loop with calls
1265 (a "large loop"), see if we can replace the usage of
1266 this register with the source of this SET. If we can,
1267 delete this insn.
1269 Don't do this if P has a REG_RETVAL note or if we have
1270 SMALL_REGISTER_CLASSES and SET_SRC is a hard register. */
1272 if (loop_info->has_call
1273 && regs->array[regno].single_usage != 0
1274 && regs->array[regno].single_usage != const0_rtx
1275 && REGNO_FIRST_UID (regno) == INSN_UID (p)
1276 && (REGNO_LAST_UID (regno)
1277 == INSN_UID (regs->array[regno].single_usage))
1278 && regs->array[regno].set_in_loop == 1
1279 && GET_CODE (SET_SRC (set)) != ASM_OPERANDS
1280 && ! side_effects_p (SET_SRC (set))
1281 && ! find_reg_note (p, REG_RETVAL, NULL_RTX)
1282 && (! SMALL_REGISTER_CLASSES
1283 || (! (REG_P (SET_SRC (set))
1284 && (REGNO (SET_SRC (set))
1285 < FIRST_PSEUDO_REGISTER))))
1286 && regno >= FIRST_PSEUDO_REGISTER
1287 /* This test is not redundant; SET_SRC (set) might be
1288 a call-clobbered register and the life of REGNO
1289 might span a call. */
1290 && ! modified_between_p (SET_SRC (set), p,
1291 regs->array[regno].single_usage)
1292 && no_labels_between_p (p,
1293 regs->array[regno].single_usage)
1294 && validate_replace_rtx (SET_DEST (set), SET_SRC (set),
1295 regs->array[regno].single_usage))
1297 /* Replace any usage in a REG_EQUAL note. Must copy
1298 the new source, so that we don't get rtx sharing
1299 between the SET_SOURCE and REG_NOTES of insn p. */
1300 REG_NOTES (regs->array[regno].single_usage)
1301 = (replace_rtx
1302 (REG_NOTES (regs->array[regno].single_usage),
1303 SET_DEST (set), copy_rtx (SET_SRC (set))));
1305 delete_insn (p);
1306 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
1307 i++)
1308 regs->array[regno+i].set_in_loop = 0;
1309 continue;
1312 m = xmalloc (sizeof (struct movable));
1313 m->next = 0;
1314 m->insn = p;
1315 m->set_src = src;
1316 m->dependencies = dependencies;
1317 m->set_dest = SET_DEST (set);
1318 m->force = 0;
1319 m->consec
1320 = regs->array[REGNO (SET_DEST (set))].set_in_loop - 1;
1321 m->done = 0;
1322 m->forces = 0;
1323 m->partial = 0;
1324 m->move_insn = move_insn;
1325 m->move_insn_first = 0;
1326 m->insert_temp = insert_temp;
1327 m->is_equiv = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
1328 m->savemode = VOIDmode;
1329 m->regno = regno;
1330 /* Set M->cond if either loop_invariant_p
1331 or consec_sets_invariant_p returned 2
1332 (only conditionally invariant). */
1333 m->cond = ((tem | tem1 | tem2) > 1);
1334 m->global = LOOP_REG_GLOBAL_P (loop, regno);
1335 m->match = 0;
1336 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
1337 m->savings = regs->array[regno].n_times_set;
1338 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
1339 m->savings += libcall_benefit (p);
1340 for (i = 0; i < LOOP_REGNO_NREGS (regno, SET_DEST (set)); i++)
1341 regs->array[regno+i].set_in_loop = move_insn ? -2 : -1;
1342 /* Add M to the end of the chain MOVABLES. */
1343 loop_movables_add (movables, m);
1345 if (m->consec > 0)
1347 /* It is possible for the first instruction to have a
1348 REG_EQUAL note but a non-invariant SET_SRC, so we must
1349 remember the status of the first instruction in case
1350 the last instruction doesn't have a REG_EQUAL note. */
1351 m->move_insn_first = m->move_insn;
1353 /* Skip this insn, not checking REG_LIBCALL notes. */
1354 p = next_nonnote_insn (p);
1355 /* Skip the consecutive insns, if there are any. */
1356 p = skip_consec_insns (p, m->consec);
1357 /* Back up to the last insn of the consecutive group. */
1358 p = prev_nonnote_insn (p);
1360 /* We must now reset m->move_insn, m->is_equiv, and
1361 possibly m->set_src to correspond to the effects of
1362 all the insns. */
1363 temp = find_reg_note (p, REG_EQUIV, NULL_RTX);
1364 if (temp)
1365 m->set_src = XEXP (temp, 0), m->move_insn = 1;
1366 else
1368 temp = find_reg_note (p, REG_EQUAL, NULL_RTX);
1369 if (temp && CONSTANT_P (XEXP (temp, 0)))
1370 m->set_src = XEXP (temp, 0), m->move_insn = 1;
1371 else
1372 m->move_insn = 0;
1375 m->is_equiv
1376 = (find_reg_note (p, REG_EQUIV, NULL_RTX) != 0);
1379 /* If this register is always set within a STRICT_LOW_PART
1380 or set to zero, then its high bytes are constant.
1381 So clear them outside the loop and within the loop
1382 just load the low bytes.
1383 We must check that the machine has an instruction to do so.
1384 Also, if the value loaded into the register
1385 depends on the same register, this cannot be done. */
1386 else if (SET_SRC (set) == const0_rtx
1387 && NONJUMP_INSN_P (NEXT_INSN (p))
1388 && (set1 = single_set (NEXT_INSN (p)))
1389 && GET_CODE (set1) == SET
1390 && (GET_CODE (SET_DEST (set1)) == STRICT_LOW_PART)
1391 && (GET_CODE (XEXP (SET_DEST (set1), 0)) == SUBREG)
1392 && (SUBREG_REG (XEXP (SET_DEST (set1), 0))
1393 == SET_DEST (set))
1394 && !reg_mentioned_p (SET_DEST (set), SET_SRC (set1)))
1396 int regno = REGNO (SET_DEST (set));
1397 if (regs->array[regno].set_in_loop == 2)
1399 struct movable *m;
1400 m = xmalloc (sizeof (struct movable));
1401 m->next = 0;
1402 m->insn = p;
1403 m->set_dest = SET_DEST (set);
1404 m->dependencies = 0;
1405 m->force = 0;
1406 m->consec = 0;
1407 m->done = 0;
1408 m->forces = 0;
1409 m->move_insn = 0;
1410 m->move_insn_first = 0;
1411 m->insert_temp = insert_temp;
1412 m->partial = 1;
1413 /* If the insn may not be executed on some cycles,
1414 we can't clear the whole reg; clear just high part.
1415 Not even if the reg is used only within this loop.
1416 Consider this:
1417 while (1)
1418 while (s != t) {
1419 if (foo ()) x = *s;
1420 use (x);
1422 Clearing x before the inner loop could clobber a value
1423 being saved from the last time around the outer loop.
1424 However, if the reg is not used outside this loop
1425 and all uses of the register are in the same
1426 basic block as the store, there is no problem.
1428 If this insn was made by loop, we don't know its
1429 INSN_LUID and hence must make a conservative
1430 assumption. */
1431 m->global = (INSN_UID (p) >= max_uid_for_loop
1432 || LOOP_REG_GLOBAL_P (loop, regno)
1433 || (labels_in_range_p
1434 (p, REGNO_FIRST_LUID (regno))));
1435 if (maybe_never && m->global)
1436 m->savemode = GET_MODE (SET_SRC (set1));
1437 else
1438 m->savemode = VOIDmode;
1439 m->regno = regno;
1440 m->cond = 0;
1441 m->match = 0;
1442 m->lifetime = LOOP_REG_LIFETIME (loop, regno);
1443 m->savings = 1;
1444 for (i = 0;
1445 i < LOOP_REGNO_NREGS (regno, SET_DEST (set));
1446 i++)
1447 regs->array[regno+i].set_in_loop = -1;
1448 /* Add M to the end of the chain MOVABLES. */
1449 loop_movables_add (movables, m);
1454 /* Past a call insn, we get to insns which might not be executed
1455 because the call might exit. This matters for insns that trap.
1456 Constant and pure call insns always return, so they don't count. */
1457 else if (CALL_P (p) && ! CONST_OR_PURE_CALL_P (p))
1458 call_passed = 1;
1459 /* Past a label or a jump, we get to insns for which we
1460 can't count on whether or how many times they will be
1461 executed during each iteration. Therefore, we can
1462 only move out sets of trivial variables
1463 (those not used after the loop). */
1464 /* Similar code appears twice in strength_reduce. */
1465 else if ((LABEL_P (p) || JUMP_P (p))
1466 /* If we enter the loop in the middle, and scan around to the
1467 beginning, don't set maybe_never for that. This must be an
1468 unconditional jump, otherwise the code at the top of the
1469 loop might never be executed. Unconditional jumps are
1470 followed by a barrier then the loop_end. */
1471 && ! (JUMP_P (p) && JUMP_LABEL (p) == loop->top
1472 && NEXT_INSN (NEXT_INSN (p)) == loop_end
1473 && any_uncondjump_p (p)))
1474 maybe_never = 1;
1477 /* If one movable subsumes another, ignore that other. */
1479 ignore_some_movables (movables);
1481 /* For each movable insn, see if the reg that it loads
1482 leads when it dies right into another conditionally movable insn.
1483 If so, record that the second insn "forces" the first one,
1484 since the second can be moved only if the first is. */
1486 force_movables (movables);
1488 /* See if there are multiple movable insns that load the same value.
1489 If there are, make all but the first point at the first one
1490 through the `match' field, and add the priorities of them
1491 all together as the priority of the first. */
1493 combine_movables (movables, regs);
1495 /* Now consider each movable insn to decide whether it is worth moving.
1496 Store 0 in regs->array[I].set_in_loop for each reg I that is moved.
1498 For machines with few registers this increases code size, so do not
1499 move moveables when optimizing for code size on such machines.
1500 (The 18 below is the value for i386.) */
1502 if (!optimize_size
1503 || (reg_class_size[GENERAL_REGS] > 18 && !loop_info->has_call))
1505 move_movables (loop, movables, threshold, insn_count);
1507 /* Recalculate regs->array if move_movables has created new
1508 registers. */
1509 if (max_reg_num () > regs->num)
1511 loop_regs_scan (loop, 0);
1512 for (update_start = loop_start;
1513 PREV_INSN (update_start)
1514 && !LABEL_P (PREV_INSN (update_start));
1515 update_start = PREV_INSN (update_start))
1517 update_end = NEXT_INSN (loop_end);
1519 reg_scan_update (update_start, update_end, loop_max_reg);
1520 loop_max_reg = max_reg_num ();
1524 /* Now candidates that still are negative are those not moved.
1525 Change regs->array[I].set_in_loop to indicate that those are not actually
1526 invariant. */
1527 for (i = 0; i < regs->num; i++)
1528 if (regs->array[i].set_in_loop < 0)
1529 regs->array[i].set_in_loop = regs->array[i].n_times_set;
1531 /* Now that we've moved some things out of the loop, we might be able to
1532 hoist even more memory references. */
1533 load_mems (loop);
1535 /* Recalculate regs->array if load_mems has created new registers. */
1536 if (max_reg_num () > regs->num)
1537 loop_regs_scan (loop, 0);
1539 for (update_start = loop_start;
1540 PREV_INSN (update_start)
1541 && !LABEL_P (PREV_INSN (update_start));
1542 update_start = PREV_INSN (update_start))
1544 update_end = NEXT_INSN (loop_end);
1546 reg_scan_update (update_start, update_end, loop_max_reg);
1547 loop_max_reg = max_reg_num ();
1549 if (flag_strength_reduce)
1551 if (update_end && LABEL_P (update_end))
1552 /* Ensure our label doesn't go away. */
1553 LABEL_NUSES (update_end)++;
1555 strength_reduce (loop, flags);
1557 reg_scan_update (update_start, update_end, loop_max_reg);
1558 loop_max_reg = max_reg_num ();
1560 if (update_end && LABEL_P (update_end)
1561 && --LABEL_NUSES (update_end) == 0)
1562 delete_related_insns (update_end);
1566 /* The movable information is required for strength reduction. */
1567 loop_movables_free (movables);
1569 free (regs->array);
1570 regs->array = 0;
1571 regs->num = 0;
1574 /* Add elements to *OUTPUT to record all the pseudo-regs
1575 mentioned in IN_THIS but not mentioned in NOT_IN_THIS. */
1577 static void
1578 record_excess_regs (rtx in_this, rtx not_in_this, rtx *output)
1580 enum rtx_code code;
1581 const char *fmt;
1582 int i;
1584 code = GET_CODE (in_this);
1586 switch (code)
1588 case PC:
1589 case CC0:
1590 case CONST_INT:
1591 case CONST_DOUBLE:
1592 case CONST:
1593 case SYMBOL_REF:
1594 case LABEL_REF:
1595 return;
1597 case REG:
1598 if (REGNO (in_this) >= FIRST_PSEUDO_REGISTER
1599 && ! reg_mentioned_p (in_this, not_in_this))
1600 *output = gen_rtx_EXPR_LIST (VOIDmode, in_this, *output);
1601 return;
1603 default:
1604 break;
1607 fmt = GET_RTX_FORMAT (code);
1608 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
1610 int j;
1612 switch (fmt[i])
1614 case 'E':
1615 for (j = 0; j < XVECLEN (in_this, i); j++)
1616 record_excess_regs (XVECEXP (in_this, i, j), not_in_this, output);
1617 break;
1619 case 'e':
1620 record_excess_regs (XEXP (in_this, i), not_in_this, output);
1621 break;
1626 /* Check what regs are referred to in the libcall block ending with INSN,
1627 aside from those mentioned in the equivalent value.
1628 If there are none, return 0.
1629 If there are one or more, return an EXPR_LIST containing all of them. */
1631 static rtx
1632 libcall_other_reg (rtx insn, rtx equiv)
1634 rtx note = find_reg_note (insn, REG_RETVAL, NULL_RTX);
1635 rtx p = XEXP (note, 0);
1636 rtx output = 0;
1638 /* First, find all the regs used in the libcall block
1639 that are not mentioned as inputs to the result. */
1641 while (p != insn)
1643 if (INSN_P (p))
1644 record_excess_regs (PATTERN (p), equiv, &output);
1645 p = NEXT_INSN (p);
1648 return output;
1651 /* Return 1 if all uses of REG
1652 are between INSN and the end of the basic block. */
1654 static int
1655 reg_in_basic_block_p (rtx insn, rtx reg)
1657 int regno = REGNO (reg);
1658 rtx p;
1660 if (REGNO_FIRST_UID (regno) != INSN_UID (insn))
1661 return 0;
1663 /* Search this basic block for the already recorded last use of the reg. */
1664 for (p = insn; p; p = NEXT_INSN (p))
1666 switch (GET_CODE (p))
1668 case NOTE:
1669 break;
1671 case INSN:
1672 case CALL_INSN:
1673 /* Ordinary insn: if this is the last use, we win. */
1674 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1675 return 1;
1676 break;
1678 case JUMP_INSN:
1679 /* Jump insn: if this is the last use, we win. */
1680 if (REGNO_LAST_UID (regno) == INSN_UID (p))
1681 return 1;
1682 /* Otherwise, it's the end of the basic block, so we lose. */
1683 return 0;
1685 case CODE_LABEL:
1686 case BARRIER:
1687 /* It's the end of the basic block, so we lose. */
1688 return 0;
1690 default:
1691 break;
1695 /* The "last use" that was recorded can't be found after the first
1696 use. This can happen when the last use was deleted while
1697 processing an inner loop, this inner loop was then completely
1698 unrolled, and the outer loop is always exited after the inner loop,
1699 so that everything after the first use becomes a single basic block. */
1700 return 1;
1703 /* Compute the benefit of eliminating the insns in the block whose
1704 last insn is LAST. This may be a group of insns used to compute a
1705 value directly or can contain a library call. */
1707 static int
1708 libcall_benefit (rtx last)
1710 rtx insn;
1711 int benefit = 0;
1713 for (insn = XEXP (find_reg_note (last, REG_RETVAL, NULL_RTX), 0);
1714 insn != last; insn = NEXT_INSN (insn))
1716 if (CALL_P (insn))
1717 benefit += 10; /* Assume at least this many insns in a library
1718 routine. */
1719 else if (NONJUMP_INSN_P (insn)
1720 && GET_CODE (PATTERN (insn)) != USE
1721 && GET_CODE (PATTERN (insn)) != CLOBBER)
1722 benefit++;
1725 return benefit;
1728 /* Skip COUNT insns from INSN, counting library calls as 1 insn. */
1730 static rtx
1731 skip_consec_insns (rtx insn, int count)
1733 for (; count > 0; count--)
1735 rtx temp;
1737 /* If first insn of libcall sequence, skip to end. */
1738 /* Do this at start of loop, since INSN is guaranteed to
1739 be an insn here. */
1740 if (!NOTE_P (insn)
1741 && (temp = find_reg_note (insn, REG_LIBCALL, NULL_RTX)))
1742 insn = XEXP (temp, 0);
1745 insn = NEXT_INSN (insn);
1746 while (NOTE_P (insn));
1749 return insn;
1752 /* Ignore any movable whose insn falls within a libcall
1753 which is part of another movable.
1754 We make use of the fact that the movable for the libcall value
1755 was made later and so appears later on the chain. */
1757 static void
1758 ignore_some_movables (struct loop_movables *movables)
1760 struct movable *m, *m1;
1762 for (m = movables->head; m; m = m->next)
1764 /* Is this a movable for the value of a libcall? */
1765 rtx note = find_reg_note (m->insn, REG_RETVAL, NULL_RTX);
1766 if (note)
1768 rtx insn;
1769 /* Check for earlier movables inside that range,
1770 and mark them invalid. We cannot use LUIDs here because
1771 insns created by loop.c for prior loops don't have LUIDs.
1772 Rather than reject all such insns from movables, we just
1773 explicitly check each insn in the libcall (since invariant
1774 libcalls aren't that common). */
1775 for (insn = XEXP (note, 0); insn != m->insn; insn = NEXT_INSN (insn))
1776 for (m1 = movables->head; m1 != m; m1 = m1->next)
1777 if (m1->insn == insn)
1778 m1->done = 1;
1783 /* For each movable insn, see if the reg that it loads
1784 leads when it dies right into another conditionally movable insn.
1785 If so, record that the second insn "forces" the first one,
1786 since the second can be moved only if the first is. */
1788 static void
1789 force_movables (struct loop_movables *movables)
1791 struct movable *m, *m1;
1793 for (m1 = movables->head; m1; m1 = m1->next)
1794 /* Omit this if moving just the (SET (REG) 0) of a zero-extend. */
1795 if (!m1->partial && !m1->done)
1797 int regno = m1->regno;
1798 for (m = m1->next; m; m = m->next)
1799 /* ??? Could this be a bug? What if CSE caused the
1800 register of M1 to be used after this insn?
1801 Since CSE does not update regno_last_uid,
1802 this insn M->insn might not be where it dies.
1803 But very likely this doesn't matter; what matters is
1804 that M's reg is computed from M1's reg. */
1805 if (INSN_UID (m->insn) == REGNO_LAST_UID (regno)
1806 && !m->done)
1807 break;
1808 if (m != 0 && m->set_src == m1->set_dest
1809 /* If m->consec, m->set_src isn't valid. */
1810 && m->consec == 0)
1811 m = 0;
1813 /* Increase the priority of the moving the first insn
1814 since it permits the second to be moved as well.
1815 Likewise for insns already forced by the first insn. */
1816 if (m != 0)
1818 struct movable *m2;
1820 m->forces = m1;
1821 for (m2 = m1; m2; m2 = m2->forces)
1823 m2->lifetime += m->lifetime;
1824 m2->savings += m->savings;
1830 /* Find invariant expressions that are equal and can be combined into
1831 one register. */
1833 static void
1834 combine_movables (struct loop_movables *movables, struct loop_regs *regs)
1836 struct movable *m;
1837 char *matched_regs = xmalloc (regs->num);
1838 enum machine_mode mode;
1840 /* Regs that are set more than once are not allowed to match
1841 or be matched. I'm no longer sure why not. */
1842 /* Only pseudo registers are allowed to match or be matched,
1843 since move_movables does not validate the change. */
1844 /* Perhaps testing m->consec_sets would be more appropriate here? */
1846 for (m = movables->head; m; m = m->next)
1847 if (m->match == 0 && regs->array[m->regno].n_times_set == 1
1848 && m->regno >= FIRST_PSEUDO_REGISTER
1849 && !m->insert_temp
1850 && !m->partial)
1852 struct movable *m1;
1853 int regno = m->regno;
1855 memset (matched_regs, 0, regs->num);
1856 matched_regs[regno] = 1;
1858 /* We want later insns to match the first one. Don't make the first
1859 one match any later ones. So start this loop at m->next. */
1860 for (m1 = m->next; m1; m1 = m1->next)
1861 if (m != m1 && m1->match == 0
1862 && !m1->insert_temp
1863 && regs->array[m1->regno].n_times_set == 1
1864 && m1->regno >= FIRST_PSEUDO_REGISTER
1865 /* A reg used outside the loop mustn't be eliminated. */
1866 && !m1->global
1867 /* A reg used for zero-extending mustn't be eliminated. */
1868 && !m1->partial
1869 && (matched_regs[m1->regno]
1872 /* Can combine regs with different modes loaded from the
1873 same constant only if the modes are the same or
1874 if both are integer modes with M wider or the same
1875 width as M1. The check for integer is redundant, but
1876 safe, since the only case of differing destination
1877 modes with equal sources is when both sources are
1878 VOIDmode, i.e., CONST_INT. */
1879 (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest)
1880 || (GET_MODE_CLASS (GET_MODE (m->set_dest)) == MODE_INT
1881 && GET_MODE_CLASS (GET_MODE (m1->set_dest)) == MODE_INT
1882 && (GET_MODE_BITSIZE (GET_MODE (m->set_dest))
1883 >= GET_MODE_BITSIZE (GET_MODE (m1->set_dest)))))
1884 /* See if the source of M1 says it matches M. */
1885 && ((REG_P (m1->set_src)
1886 && matched_regs[REGNO (m1->set_src)])
1887 || rtx_equal_for_loop_p (m->set_src, m1->set_src,
1888 movables, regs))))
1889 && ((m->dependencies == m1->dependencies)
1890 || rtx_equal_p (m->dependencies, m1->dependencies)))
1892 m->lifetime += m1->lifetime;
1893 m->savings += m1->savings;
1894 m1->done = 1;
1895 m1->match = m;
1896 matched_regs[m1->regno] = 1;
1900 /* Now combine the regs used for zero-extension.
1901 This can be done for those not marked `global'
1902 provided their lives don't overlap. */
1904 for (mode = GET_CLASS_NARROWEST_MODE (MODE_INT); mode != VOIDmode;
1905 mode = GET_MODE_WIDER_MODE (mode))
1907 struct movable *m0 = 0;
1909 /* Combine all the registers for extension from mode MODE.
1910 Don't combine any that are used outside this loop. */
1911 for (m = movables->head; m; m = m->next)
1912 if (m->partial && ! m->global
1913 && mode == GET_MODE (SET_SRC (PATTERN (NEXT_INSN (m->insn)))))
1915 struct movable *m1;
1917 int first = REGNO_FIRST_LUID (m->regno);
1918 int last = REGNO_LAST_LUID (m->regno);
1920 if (m0 == 0)
1922 /* First one: don't check for overlap, just record it. */
1923 m0 = m;
1924 continue;
1927 /* Make sure they extend to the same mode.
1928 (Almost always true.) */
1929 if (GET_MODE (m->set_dest) != GET_MODE (m0->set_dest))
1930 continue;
1932 /* We already have one: check for overlap with those
1933 already combined together. */
1934 for (m1 = movables->head; m1 != m; m1 = m1->next)
1935 if (m1 == m0 || (m1->partial && m1->match == m0))
1936 if (! (REGNO_FIRST_LUID (m1->regno) > last
1937 || REGNO_LAST_LUID (m1->regno) < first))
1938 goto overlap;
1940 /* No overlap: we can combine this with the others. */
1941 m0->lifetime += m->lifetime;
1942 m0->savings += m->savings;
1943 m->done = 1;
1944 m->match = m0;
1946 overlap:
1951 /* Clean up. */
1952 free (matched_regs);
1955 /* Returns the number of movable instructions in LOOP that were not
1956 moved outside the loop. */
1958 static int
1959 num_unmoved_movables (const struct loop *loop)
1961 int num = 0;
1962 struct movable *m;
1964 for (m = LOOP_MOVABLES (loop)->head; m; m = m->next)
1965 if (!m->done)
1966 ++num;
1968 return num;
1972 /* Return 1 if regs X and Y will become the same if moved. */
1974 static int
1975 regs_match_p (rtx x, rtx y, struct loop_movables *movables)
1977 unsigned int xn = REGNO (x);
1978 unsigned int yn = REGNO (y);
1979 struct movable *mx, *my;
1981 for (mx = movables->head; mx; mx = mx->next)
1982 if (mx->regno == xn)
1983 break;
1985 for (my = movables->head; my; my = my->next)
1986 if (my->regno == yn)
1987 break;
1989 return (mx && my
1990 && ((mx->match == my->match && mx->match != 0)
1991 || mx->match == my
1992 || mx == my->match));
1995 /* Return 1 if X and Y are identical-looking rtx's.
1996 This is the Lisp function EQUAL for rtx arguments.
1998 If two registers are matching movables or a movable register and an
1999 equivalent constant, consider them equal. */
2001 static int
2002 rtx_equal_for_loop_p (rtx x, rtx y, struct loop_movables *movables,
2003 struct loop_regs *regs)
2005 int i;
2006 int j;
2007 struct movable *m;
2008 enum rtx_code code;
2009 const char *fmt;
2011 if (x == y)
2012 return 1;
2013 if (x == 0 || y == 0)
2014 return 0;
2016 code = GET_CODE (x);
2018 /* If we have a register and a constant, they may sometimes be
2019 equal. */
2020 if (REG_P (x) && regs->array[REGNO (x)].set_in_loop == -2
2021 && CONSTANT_P (y))
2023 for (m = movables->head; m; m = m->next)
2024 if (m->move_insn && m->regno == REGNO (x)
2025 && rtx_equal_p (m->set_src, y))
2026 return 1;
2028 else if (REG_P (y) && regs->array[REGNO (y)].set_in_loop == -2
2029 && CONSTANT_P (x))
2031 for (m = movables->head; m; m = m->next)
2032 if (m->move_insn && m->regno == REGNO (y)
2033 && rtx_equal_p (m->set_src, x))
2034 return 1;
2037 /* Otherwise, rtx's of different codes cannot be equal. */
2038 if (code != GET_CODE (y))
2039 return 0;
2041 /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent.
2042 (REG:SI x) and (REG:HI x) are NOT equivalent. */
2044 if (GET_MODE (x) != GET_MODE (y))
2045 return 0;
2047 /* These three types of rtx's can be compared nonrecursively. */
2048 if (code == REG)
2049 return (REGNO (x) == REGNO (y) || regs_match_p (x, y, movables));
2051 if (code == LABEL_REF)
2052 return XEXP (x, 0) == XEXP (y, 0);
2053 if (code == SYMBOL_REF)
2054 return XSTR (x, 0) == XSTR (y, 0);
2056 /* Compare the elements. If any pair of corresponding elements
2057 fail to match, return 0 for the whole things. */
2059 fmt = GET_RTX_FORMAT (code);
2060 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2062 switch (fmt[i])
2064 case 'w':
2065 if (XWINT (x, i) != XWINT (y, i))
2066 return 0;
2067 break;
2069 case 'i':
2070 if (XINT (x, i) != XINT (y, i))
2071 return 0;
2072 break;
2074 case 'E':
2075 /* Two vectors must have the same length. */
2076 if (XVECLEN (x, i) != XVECLEN (y, i))
2077 return 0;
2079 /* And the corresponding elements must match. */
2080 for (j = 0; j < XVECLEN (x, i); j++)
2081 if (rtx_equal_for_loop_p (XVECEXP (x, i, j), XVECEXP (y, i, j),
2082 movables, regs) == 0)
2083 return 0;
2084 break;
2086 case 'e':
2087 if (rtx_equal_for_loop_p (XEXP (x, i), XEXP (y, i), movables, regs)
2088 == 0)
2089 return 0;
2090 break;
2092 case 's':
2093 if (strcmp (XSTR (x, i), XSTR (y, i)))
2094 return 0;
2095 break;
2097 case 'u':
2098 /* These are just backpointers, so they don't matter. */
2099 break;
2101 case '0':
2102 break;
2104 /* It is believed that rtx's at this level will never
2105 contain anything but integers and other rtx's,
2106 except for within LABEL_REFs and SYMBOL_REFs. */
2107 default:
2108 abort ();
2111 return 1;
2114 /* If X contains any LABEL_REF's, add REG_LABEL notes for them to all
2115 insns in INSNS which use the reference. LABEL_NUSES for CODE_LABEL
2116 references is incremented once for each added note. */
2118 static void
2119 add_label_notes (rtx x, rtx insns)
2121 enum rtx_code code = GET_CODE (x);
2122 int i, j;
2123 const char *fmt;
2124 rtx insn;
2126 if (code == LABEL_REF && !LABEL_REF_NONLOCAL_P (x))
2128 /* This code used to ignore labels that referred to dispatch tables to
2129 avoid flow generating (slightly) worse code.
2131 We no longer ignore such label references (see LABEL_REF handling in
2132 mark_jump_label for additional information). */
2133 for (insn = insns; insn; insn = NEXT_INSN (insn))
2134 if (reg_mentioned_p (XEXP (x, 0), insn))
2136 REG_NOTES (insn) = gen_rtx_INSN_LIST (REG_LABEL, XEXP (x, 0),
2137 REG_NOTES (insn));
2138 if (LABEL_P (XEXP (x, 0)))
2139 LABEL_NUSES (XEXP (x, 0))++;
2143 fmt = GET_RTX_FORMAT (code);
2144 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2146 if (fmt[i] == 'e')
2147 add_label_notes (XEXP (x, i), insns);
2148 else if (fmt[i] == 'E')
2149 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
2150 add_label_notes (XVECEXP (x, i, j), insns);
2154 /* Scan MOVABLES, and move the insns that deserve to be moved.
2155 If two matching movables are combined, replace one reg with the
2156 other throughout. */
2158 static void
2159 move_movables (struct loop *loop, struct loop_movables *movables,
2160 int threshold, int insn_count)
2162 struct loop_regs *regs = LOOP_REGS (loop);
2163 int nregs = regs->num;
2164 rtx new_start = 0;
2165 struct movable *m;
2166 rtx p;
2167 rtx loop_start = loop->start;
2168 rtx loop_end = loop->end;
2169 /* Map of pseudo-register replacements to handle combining
2170 when we move several insns that load the same value
2171 into different pseudo-registers. */
2172 rtx *reg_map = xcalloc (nregs, sizeof (rtx));
2173 char *already_moved = xcalloc (nregs, sizeof (char));
2175 for (m = movables->head; m; m = m->next)
2177 /* Describe this movable insn. */
2179 if (loop_dump_stream)
2181 fprintf (loop_dump_stream, "Insn %d: regno %d (life %d), ",
2182 INSN_UID (m->insn), m->regno, m->lifetime);
2183 if (m->consec > 0)
2184 fprintf (loop_dump_stream, "consec %d, ", m->consec);
2185 if (m->cond)
2186 fprintf (loop_dump_stream, "cond ");
2187 if (m->force)
2188 fprintf (loop_dump_stream, "force ");
2189 if (m->global)
2190 fprintf (loop_dump_stream, "global ");
2191 if (m->done)
2192 fprintf (loop_dump_stream, "done ");
2193 if (m->move_insn)
2194 fprintf (loop_dump_stream, "move-insn ");
2195 if (m->match)
2196 fprintf (loop_dump_stream, "matches %d ",
2197 INSN_UID (m->match->insn));
2198 if (m->forces)
2199 fprintf (loop_dump_stream, "forces %d ",
2200 INSN_UID (m->forces->insn));
2203 /* Ignore the insn if it's already done (it matched something else).
2204 Otherwise, see if it is now safe to move. */
2206 if (!m->done
2207 && (! m->cond
2208 || (1 == loop_invariant_p (loop, m->set_src)
2209 && (m->dependencies == 0
2210 || 1 == loop_invariant_p (loop, m->dependencies))
2211 && (m->consec == 0
2212 || 1 == consec_sets_invariant_p (loop, m->set_dest,
2213 m->consec + 1,
2214 m->insn))))
2215 && (! m->forces || m->forces->done))
2217 int regno;
2218 rtx p;
2219 int savings = m->savings;
2221 /* We have an insn that is safe to move.
2222 Compute its desirability. */
2224 p = m->insn;
2225 regno = m->regno;
2227 if (loop_dump_stream)
2228 fprintf (loop_dump_stream, "savings %d ", savings);
2230 if (regs->array[regno].moved_once && loop_dump_stream)
2231 fprintf (loop_dump_stream, "halved since already moved ");
2233 /* An insn MUST be moved if we already moved something else
2234 which is safe only if this one is moved too: that is,
2235 if already_moved[REGNO] is nonzero. */
2237 /* An insn is desirable to move if the new lifetime of the
2238 register is no more than THRESHOLD times the old lifetime.
2239 If it's not desirable, it means the loop is so big
2240 that moving won't speed things up much,
2241 and it is liable to make register usage worse. */
2243 /* It is also desirable to move if it can be moved at no
2244 extra cost because something else was already moved. */
2246 if (already_moved[regno]
2247 || (threshold * savings * m->lifetime) >=
2248 (regs->array[regno].moved_once ? insn_count * 2 : insn_count)
2249 || (m->forces && m->forces->done
2250 && regs->array[m->forces->regno].n_times_set == 1))
2252 int count;
2253 struct movable *m1;
2254 rtx first = NULL_RTX;
2255 rtx newreg = NULL_RTX;
2257 if (m->insert_temp)
2258 newreg = gen_reg_rtx (GET_MODE (m->set_dest));
2260 /* Now move the insns that set the reg. */
2262 if (m->partial && m->match)
2264 rtx newpat, i1;
2265 rtx r1, r2;
2266 /* Find the end of this chain of matching regs.
2267 Thus, we load each reg in the chain from that one reg.
2268 And that reg is loaded with 0 directly,
2269 since it has ->match == 0. */
2270 for (m1 = m; m1->match; m1 = m1->match);
2271 newpat = gen_move_insn (SET_DEST (PATTERN (m->insn)),
2272 SET_DEST (PATTERN (m1->insn)));
2273 i1 = loop_insn_hoist (loop, newpat);
2275 /* Mark the moved, invariant reg as being allowed to
2276 share a hard reg with the other matching invariant. */
2277 REG_NOTES (i1) = REG_NOTES (m->insn);
2278 r1 = SET_DEST (PATTERN (m->insn));
2279 r2 = SET_DEST (PATTERN (m1->insn));
2280 regs_may_share
2281 = gen_rtx_EXPR_LIST (VOIDmode, r1,
2282 gen_rtx_EXPR_LIST (VOIDmode, r2,
2283 regs_may_share));
2284 delete_insn (m->insn);
2286 if (new_start == 0)
2287 new_start = i1;
2289 if (loop_dump_stream)
2290 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
2292 /* If we are to re-generate the item being moved with a
2293 new move insn, first delete what we have and then emit
2294 the move insn before the loop. */
2295 else if (m->move_insn)
2297 rtx i1, temp, seq;
2299 for (count = m->consec; count >= 0; count--)
2301 /* If this is the first insn of a library call sequence,
2302 something is very wrong. */
2303 if (!NOTE_P (p)
2304 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
2305 abort ();
2307 /* If this is the last insn of a libcall sequence, then
2308 delete every insn in the sequence except the last.
2309 The last insn is handled in the normal manner. */
2310 if (!NOTE_P (p)
2311 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
2313 temp = XEXP (temp, 0);
2314 while (temp != p)
2315 temp = delete_insn (temp);
2318 temp = p;
2319 p = delete_insn (p);
2321 /* simplify_giv_expr expects that it can walk the insns
2322 at m->insn forwards and see this old sequence we are
2323 tossing here. delete_insn does preserve the next
2324 pointers, but when we skip over a NOTE we must fix
2325 it up. Otherwise that code walks into the non-deleted
2326 insn stream. */
2327 while (p && NOTE_P (p))
2328 p = NEXT_INSN (temp) = NEXT_INSN (p);
2330 if (m->insert_temp)
2332 /* Replace the original insn with a move from
2333 our newly created temp. */
2334 start_sequence ();
2335 emit_move_insn (m->set_dest, newreg);
2336 seq = get_insns ();
2337 end_sequence ();
2338 emit_insn_before (seq, p);
2342 start_sequence ();
2343 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
2344 m->set_src);
2345 seq = get_insns ();
2346 end_sequence ();
2348 add_label_notes (m->set_src, seq);
2350 i1 = loop_insn_hoist (loop, seq);
2351 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
2352 set_unique_reg_note (i1,
2353 m->is_equiv ? REG_EQUIV : REG_EQUAL,
2354 m->set_src);
2356 if (loop_dump_stream)
2357 fprintf (loop_dump_stream, " moved to %d", INSN_UID (i1));
2359 /* The more regs we move, the less we like moving them. */
2360 threshold -= 3;
2362 else
2364 for (count = m->consec; count >= 0; count--)
2366 rtx i1, temp;
2368 /* If first insn of libcall sequence, skip to end. */
2369 /* Do this at start of loop, since p is guaranteed to
2370 be an insn here. */
2371 if (!NOTE_P (p)
2372 && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
2373 p = XEXP (temp, 0);
2375 /* If last insn of libcall sequence, move all
2376 insns except the last before the loop. The last
2377 insn is handled in the normal manner. */
2378 if (!NOTE_P (p)
2379 && (temp = find_reg_note (p, REG_RETVAL, NULL_RTX)))
2381 rtx fn_address = 0;
2382 rtx fn_reg = 0;
2383 rtx fn_address_insn = 0;
2385 first = 0;
2386 for (temp = XEXP (temp, 0); temp != p;
2387 temp = NEXT_INSN (temp))
2389 rtx body;
2390 rtx n;
2391 rtx next;
2393 if (NOTE_P (temp))
2394 continue;
2396 body = PATTERN (temp);
2398 /* Find the next insn after TEMP,
2399 not counting USE or NOTE insns. */
2400 for (next = NEXT_INSN (temp); next != p;
2401 next = NEXT_INSN (next))
2402 if (! (NONJUMP_INSN_P (next)
2403 && GET_CODE (PATTERN (next)) == USE)
2404 && !NOTE_P (next))
2405 break;
2407 /* If that is the call, this may be the insn
2408 that loads the function address.
2410 Extract the function address from the insn
2411 that loads it into a register.
2412 If this insn was cse'd, we get incorrect code.
2414 So emit a new move insn that copies the
2415 function address into the register that the
2416 call insn will use. flow.c will delete any
2417 redundant stores that we have created. */
2418 if (CALL_P (next)
2419 && GET_CODE (body) == SET
2420 && REG_P (SET_DEST (body))
2421 && (n = find_reg_note (temp, REG_EQUAL,
2422 NULL_RTX)))
2424 fn_reg = SET_SRC (body);
2425 if (!REG_P (fn_reg))
2426 fn_reg = SET_DEST (body);
2427 fn_address = XEXP (n, 0);
2428 fn_address_insn = temp;
2430 /* We have the call insn.
2431 If it uses the register we suspect it might,
2432 load it with the correct address directly. */
2433 if (CALL_P (temp)
2434 && fn_address != 0
2435 && reg_referenced_p (fn_reg, body))
2436 loop_insn_emit_after (loop, 0, fn_address_insn,
2437 gen_move_insn
2438 (fn_reg, fn_address));
2440 if (CALL_P (temp))
2442 i1 = loop_call_insn_hoist (loop, body);
2443 /* Because the USAGE information potentially
2444 contains objects other than hard registers
2445 we need to copy it. */
2446 if (CALL_INSN_FUNCTION_USAGE (temp))
2447 CALL_INSN_FUNCTION_USAGE (i1)
2448 = copy_rtx (CALL_INSN_FUNCTION_USAGE (temp));
2450 else
2451 i1 = loop_insn_hoist (loop, body);
2452 if (first == 0)
2453 first = i1;
2454 if (temp == fn_address_insn)
2455 fn_address_insn = i1;
2456 REG_NOTES (i1) = REG_NOTES (temp);
2457 REG_NOTES (temp) = NULL;
2458 delete_insn (temp);
2460 if (new_start == 0)
2461 new_start = first;
2463 if (m->savemode != VOIDmode)
2465 /* P sets REG to zero; but we should clear only
2466 the bits that are not covered by the mode
2467 m->savemode. */
2468 rtx reg = m->set_dest;
2469 rtx sequence;
2470 rtx tem;
2472 start_sequence ();
2473 tem = expand_simple_binop
2474 (GET_MODE (reg), AND, reg,
2475 GEN_INT ((((HOST_WIDE_INT) 1
2476 << GET_MODE_BITSIZE (m->savemode)))
2477 - 1),
2478 reg, 1, OPTAB_LIB_WIDEN);
2479 if (tem == 0)
2480 abort ();
2481 if (tem != reg)
2482 emit_move_insn (reg, tem);
2483 sequence = get_insns ();
2484 end_sequence ();
2485 i1 = loop_insn_hoist (loop, sequence);
2487 else if (CALL_P (p))
2489 i1 = loop_call_insn_hoist (loop, PATTERN (p));
2490 /* Because the USAGE information potentially
2491 contains objects other than hard registers
2492 we need to copy it. */
2493 if (CALL_INSN_FUNCTION_USAGE (p))
2494 CALL_INSN_FUNCTION_USAGE (i1)
2495 = copy_rtx (CALL_INSN_FUNCTION_USAGE (p));
2497 else if (count == m->consec && m->move_insn_first)
2499 rtx seq;
2500 /* The SET_SRC might not be invariant, so we must
2501 use the REG_EQUAL note. */
2502 start_sequence ();
2503 emit_move_insn (m->insert_temp ? newreg : m->set_dest,
2504 m->set_src);
2505 seq = get_insns ();
2506 end_sequence ();
2508 add_label_notes (m->set_src, seq);
2510 i1 = loop_insn_hoist (loop, seq);
2511 if (! find_reg_note (i1, REG_EQUAL, NULL_RTX))
2512 set_unique_reg_note (i1, m->is_equiv ? REG_EQUIV
2513 : REG_EQUAL, m->set_src);
2515 else if (m->insert_temp)
2517 rtx *reg_map2 = xcalloc (REGNO (newreg),
2518 sizeof(rtx));
2519 reg_map2 [m->regno] = newreg;
2521 i1 = loop_insn_hoist (loop, copy_rtx (PATTERN (p)));
2522 replace_regs (i1, reg_map2, REGNO (newreg), 1);
2523 free (reg_map2);
2525 else
2526 i1 = loop_insn_hoist (loop, PATTERN (p));
2528 if (REG_NOTES (i1) == 0)
2530 REG_NOTES (i1) = REG_NOTES (p);
2531 REG_NOTES (p) = NULL;
2533 /* If there is a REG_EQUAL note present whose value
2534 is not loop invariant, then delete it, since it
2535 may cause problems with later optimization passes.
2536 It is possible for cse to create such notes
2537 like this as a result of record_jump_cond. */
2539 if ((temp = find_reg_note (i1, REG_EQUAL, NULL_RTX))
2540 && ! loop_invariant_p (loop, XEXP (temp, 0)))
2541 remove_note (i1, temp);
2544 if (new_start == 0)
2545 new_start = i1;
2547 if (loop_dump_stream)
2548 fprintf (loop_dump_stream, " moved to %d",
2549 INSN_UID (i1));
2551 /* If library call, now fix the REG_NOTES that contain
2552 insn pointers, namely REG_LIBCALL on FIRST
2553 and REG_RETVAL on I1. */
2554 if ((temp = find_reg_note (i1, REG_RETVAL, NULL_RTX)))
2556 XEXP (temp, 0) = first;
2557 temp = find_reg_note (first, REG_LIBCALL, NULL_RTX);
2558 XEXP (temp, 0) = i1;
2561 temp = p;
2562 delete_insn (p);
2563 p = NEXT_INSN (p);
2565 /* simplify_giv_expr expects that it can walk the insns
2566 at m->insn forwards and see this old sequence we are
2567 tossing here. delete_insn does preserve the next
2568 pointers, but when we skip over a NOTE we must fix
2569 it up. Otherwise that code walks into the non-deleted
2570 insn stream. */
2571 while (p && NOTE_P (p))
2572 p = NEXT_INSN (temp) = NEXT_INSN (p);
2574 if (m->insert_temp)
2576 rtx seq;
2577 /* Replace the original insn with a move from
2578 our newly created temp. */
2579 start_sequence ();
2580 emit_move_insn (m->set_dest, newreg);
2581 seq = get_insns ();
2582 end_sequence ();
2583 emit_insn_before (seq, p);
2587 /* The more regs we move, the less we like moving them. */
2588 threshold -= 3;
2591 m->done = 1;
2593 if (!m->insert_temp)
2595 /* Any other movable that loads the same register
2596 MUST be moved. */
2597 already_moved[regno] = 1;
2599 /* This reg has been moved out of one loop. */
2600 regs->array[regno].moved_once = 1;
2602 /* The reg set here is now invariant. */
2603 if (! m->partial)
2605 int i;
2606 for (i = 0; i < LOOP_REGNO_NREGS (regno, m->set_dest); i++)
2607 regs->array[regno+i].set_in_loop = 0;
2610 /* Change the length-of-life info for the register
2611 to say it lives at least the full length of this loop.
2612 This will help guide optimizations in outer loops. */
2614 if (REGNO_FIRST_LUID (regno) > INSN_LUID (loop_start))
2615 /* This is the old insn before all the moved insns.
2616 We can't use the moved insn because it is out of range
2617 in uid_luid. Only the old insns have luids. */
2618 REGNO_FIRST_UID (regno) = INSN_UID (loop_start);
2619 if (REGNO_LAST_LUID (regno) < INSN_LUID (loop_end))
2620 REGNO_LAST_UID (regno) = INSN_UID (loop_end);
2623 /* Combine with this moved insn any other matching movables. */
2625 if (! m->partial)
2626 for (m1 = movables->head; m1; m1 = m1->next)
2627 if (m1->match == m)
2629 rtx temp;
2631 /* Schedule the reg loaded by M1
2632 for replacement so that shares the reg of M.
2633 If the modes differ (only possible in restricted
2634 circumstances, make a SUBREG.
2636 Note this assumes that the target dependent files
2637 treat REG and SUBREG equally, including within
2638 GO_IF_LEGITIMATE_ADDRESS and in all the
2639 predicates since we never verify that replacing the
2640 original register with a SUBREG results in a
2641 recognizable insn. */
2642 if (GET_MODE (m->set_dest) == GET_MODE (m1->set_dest))
2643 reg_map[m1->regno] = m->set_dest;
2644 else
2645 reg_map[m1->regno]
2646 = gen_lowpart_common (GET_MODE (m1->set_dest),
2647 m->set_dest);
2649 /* Get rid of the matching insn
2650 and prevent further processing of it. */
2651 m1->done = 1;
2653 /* If library call, delete all insns. */
2654 if ((temp = find_reg_note (m1->insn, REG_RETVAL,
2655 NULL_RTX)))
2656 delete_insn_chain (XEXP (temp, 0), m1->insn);
2657 else
2658 delete_insn (m1->insn);
2660 /* Any other movable that loads the same register
2661 MUST be moved. */
2662 already_moved[m1->regno] = 1;
2664 /* The reg merged here is now invariant,
2665 if the reg it matches is invariant. */
2666 if (! m->partial)
2668 int i;
2669 for (i = 0;
2670 i < LOOP_REGNO_NREGS (regno, m1->set_dest);
2671 i++)
2672 regs->array[m1->regno+i].set_in_loop = 0;
2676 else if (loop_dump_stream)
2677 fprintf (loop_dump_stream, "not desirable");
2679 else if (loop_dump_stream && !m->match)
2680 fprintf (loop_dump_stream, "not safe");
2682 if (loop_dump_stream)
2683 fprintf (loop_dump_stream, "\n");
2686 if (new_start == 0)
2687 new_start = loop_start;
2689 /* Go through all the instructions in the loop, making
2690 all the register substitutions scheduled in REG_MAP. */
2691 for (p = new_start; p != loop_end; p = NEXT_INSN (p))
2692 if (INSN_P (p))
2694 replace_regs (PATTERN (p), reg_map, nregs, 0);
2695 replace_regs (REG_NOTES (p), reg_map, nregs, 0);
2696 INSN_CODE (p) = -1;
2699 /* Clean up. */
2700 free (reg_map);
2701 free (already_moved);
2705 static void
2706 loop_movables_add (struct loop_movables *movables, struct movable *m)
2708 if (movables->head == 0)
2709 movables->head = m;
2710 else
2711 movables->last->next = m;
2712 movables->last = m;
2716 static void
2717 loop_movables_free (struct loop_movables *movables)
2719 struct movable *m;
2720 struct movable *m_next;
2722 for (m = movables->head; m; m = m_next)
2724 m_next = m->next;
2725 free (m);
2729 #if 0
2730 /* Scan X and replace the address of any MEM in it with ADDR.
2731 REG is the address that MEM should have before the replacement. */
2733 static void
2734 replace_call_address (rtx x, rtx reg, rtx addr)
2736 enum rtx_code code;
2737 int i;
2738 const char *fmt;
2740 if (x == 0)
2741 return;
2742 code = GET_CODE (x);
2743 switch (code)
2745 case PC:
2746 case CC0:
2747 case CONST_INT:
2748 case CONST_DOUBLE:
2749 case CONST:
2750 case SYMBOL_REF:
2751 case LABEL_REF:
2752 case REG:
2753 return;
2755 case SET:
2756 /* Short cut for very common case. */
2757 replace_call_address (XEXP (x, 1), reg, addr);
2758 return;
2760 case CALL:
2761 /* Short cut for very common case. */
2762 replace_call_address (XEXP (x, 0), reg, addr);
2763 return;
2765 case MEM:
2766 /* If this MEM uses a reg other than the one we expected,
2767 something is wrong. */
2768 if (XEXP (x, 0) != reg)
2769 abort ();
2770 XEXP (x, 0) = addr;
2771 return;
2773 default:
2774 break;
2777 fmt = GET_RTX_FORMAT (code);
2778 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2780 if (fmt[i] == 'e')
2781 replace_call_address (XEXP (x, i), reg, addr);
2782 else if (fmt[i] == 'E')
2784 int j;
2785 for (j = 0; j < XVECLEN (x, i); j++)
2786 replace_call_address (XVECEXP (x, i, j), reg, addr);
2790 #endif
2792 /* Return the number of memory refs to addresses that vary
2793 in the rtx X. */
2795 static int
2796 count_nonfixed_reads (const struct loop *loop, rtx x)
2798 enum rtx_code code;
2799 int i;
2800 const char *fmt;
2801 int value;
2803 if (x == 0)
2804 return 0;
2806 code = GET_CODE (x);
2807 switch (code)
2809 case PC:
2810 case CC0:
2811 case CONST_INT:
2812 case CONST_DOUBLE:
2813 case CONST:
2814 case SYMBOL_REF:
2815 case LABEL_REF:
2816 case REG:
2817 return 0;
2819 case MEM:
2820 return ((loop_invariant_p (loop, XEXP (x, 0)) != 1)
2821 + count_nonfixed_reads (loop, XEXP (x, 0)));
2823 default:
2824 break;
2827 value = 0;
2828 fmt = GET_RTX_FORMAT (code);
2829 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
2831 if (fmt[i] == 'e')
2832 value += count_nonfixed_reads (loop, XEXP (x, i));
2833 if (fmt[i] == 'E')
2835 int j;
2836 for (j = 0; j < XVECLEN (x, i); j++)
2837 value += count_nonfixed_reads (loop, XVECEXP (x, i, j));
2840 return value;
2843 /* Scan a loop setting the elements `loops_enclosed',
2844 `has_call', `has_nonconst_call', `has_volatile', `has_tablejump',
2845 `unknown_address_altered', `unknown_constant_address_altered', and
2846 `num_mem_sets' in LOOP. Also, fill in the array `mems' and the
2847 list `store_mems' in LOOP. */
2849 static void
2850 prescan_loop (struct loop *loop)
2852 int level = 1;
2853 rtx insn;
2854 struct loop_info *loop_info = LOOP_INFO (loop);
2855 rtx start = loop->start;
2856 rtx end = loop->end;
2857 /* The label after END. Jumping here is just like falling off the
2858 end of the loop. We use next_nonnote_insn instead of next_label
2859 as a hedge against the (pathological) case where some actual insn
2860 might end up between the two. */
2861 rtx exit_target = next_nonnote_insn (end);
2863 loop_info->has_indirect_jump = indirect_jump_in_function;
2864 loop_info->pre_header_has_call = 0;
2865 loop_info->has_call = 0;
2866 loop_info->has_nonconst_call = 0;
2867 loop_info->has_prefetch = 0;
2868 loop_info->has_volatile = 0;
2869 loop_info->has_tablejump = 0;
2870 loop_info->has_multiple_exit_targets = 0;
2871 loop->level = 1;
2873 loop_info->unknown_address_altered = 0;
2874 loop_info->unknown_constant_address_altered = 0;
2875 loop_info->store_mems = NULL_RTX;
2876 loop_info->first_loop_store_insn = NULL_RTX;
2877 loop_info->mems_idx = 0;
2878 loop_info->num_mem_sets = 0;
2880 for (insn = start; insn && !LABEL_P (insn);
2881 insn = PREV_INSN (insn))
2883 if (CALL_P (insn))
2885 loop_info->pre_header_has_call = 1;
2886 break;
2890 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
2891 insn = NEXT_INSN (insn))
2893 switch (GET_CODE (insn))
2895 case NOTE:
2896 if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_BEG)
2898 ++level;
2899 /* Count number of loops contained in this one. */
2900 loop->level++;
2902 else if (NOTE_LINE_NUMBER (insn) == NOTE_INSN_LOOP_END)
2903 --level;
2904 break;
2906 case CALL_INSN:
2907 if (! CONST_OR_PURE_CALL_P (insn))
2909 loop_info->unknown_address_altered = 1;
2910 loop_info->has_nonconst_call = 1;
2912 else if (pure_call_p (insn))
2913 loop_info->has_nonconst_call = 1;
2914 loop_info->has_call = 1;
2915 if (can_throw_internal (insn))
2916 loop_info->has_multiple_exit_targets = 1;
2917 break;
2919 case JUMP_INSN:
2920 if (! loop_info->has_multiple_exit_targets)
2922 rtx set = pc_set (insn);
2924 if (set)
2926 rtx src = SET_SRC (set);
2927 rtx label1, label2;
2929 if (GET_CODE (src) == IF_THEN_ELSE)
2931 label1 = XEXP (src, 1);
2932 label2 = XEXP (src, 2);
2934 else
2936 label1 = src;
2937 label2 = NULL_RTX;
2942 if (label1 && label1 != pc_rtx)
2944 if (GET_CODE (label1) != LABEL_REF)
2946 /* Something tricky. */
2947 loop_info->has_multiple_exit_targets = 1;
2948 break;
2950 else if (XEXP (label1, 0) != exit_target
2951 && LABEL_OUTSIDE_LOOP_P (label1))
2953 /* A jump outside the current loop. */
2954 loop_info->has_multiple_exit_targets = 1;
2955 break;
2959 label1 = label2;
2960 label2 = NULL_RTX;
2962 while (label1);
2964 else
2966 /* A return, or something tricky. */
2967 loop_info->has_multiple_exit_targets = 1;
2970 /* Fall through. */
2972 case INSN:
2973 if (volatile_refs_p (PATTERN (insn)))
2974 loop_info->has_volatile = 1;
2976 if (JUMP_P (insn)
2977 && (GET_CODE (PATTERN (insn)) == ADDR_DIFF_VEC
2978 || GET_CODE (PATTERN (insn)) == ADDR_VEC))
2979 loop_info->has_tablejump = 1;
2981 note_stores (PATTERN (insn), note_addr_stored, loop_info);
2982 if (! loop_info->first_loop_store_insn && loop_info->store_mems)
2983 loop_info->first_loop_store_insn = insn;
2985 if (flag_non_call_exceptions && can_throw_internal (insn))
2986 loop_info->has_multiple_exit_targets = 1;
2987 break;
2989 default:
2990 break;
2994 /* Now, rescan the loop, setting up the LOOP_MEMS array. */
2995 if (/* An exception thrown by a called function might land us
2996 anywhere. */
2997 ! loop_info->has_nonconst_call
2998 /* We don't want loads for MEMs moved to a location before the
2999 one at which their stack memory becomes allocated. (Note
3000 that this is not a problem for malloc, etc., since those
3001 require actual function calls. */
3002 && ! current_function_calls_alloca
3003 /* There are ways to leave the loop other than falling off the
3004 end. */
3005 && ! loop_info->has_multiple_exit_targets)
3006 for (insn = NEXT_INSN (start); insn != NEXT_INSN (end);
3007 insn = NEXT_INSN (insn))
3008 for_each_rtx (&insn, insert_loop_mem, loop_info);
3010 /* BLKmode MEMs are added to LOOP_STORE_MEM as necessary so
3011 that loop_invariant_p and load_mems can use true_dependence
3012 to determine what is really clobbered. */
3013 if (loop_info->unknown_address_altered)
3015 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
3017 loop_info->store_mems
3018 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
3020 if (loop_info->unknown_constant_address_altered)
3022 rtx mem = gen_rtx_MEM (BLKmode, const0_rtx);
3023 MEM_READONLY_P (mem) = 1;
3024 loop_info->store_mems
3025 = gen_rtx_EXPR_LIST (VOIDmode, mem, loop_info->store_mems);
3029 /* Invalidate all loops containing LABEL. */
3031 static void
3032 invalidate_loops_containing_label (rtx label)
3034 struct loop *loop;
3035 for (loop = uid_loop[INSN_UID (label)]; loop; loop = loop->outer)
3036 loop->invalid = 1;
3039 /* Scan the function looking for loops. Record the start and end of each loop.
3040 Also mark as invalid loops any loops that contain a setjmp or are branched
3041 to from outside the loop. */
3043 static void
3044 find_and_verify_loops (rtx f, struct loops *loops)
3046 rtx insn;
3047 rtx label;
3048 int num_loops;
3049 struct loop *current_loop;
3050 struct loop *next_loop;
3051 struct loop *loop;
3053 num_loops = loops->num;
3055 compute_luids (f, NULL_RTX, 0);
3057 /* If there are jumps to undefined labels,
3058 treat them as jumps out of any/all loops.
3059 This also avoids writing past end of tables when there are no loops. */
3060 uid_loop[0] = NULL;
3062 /* Find boundaries of loops, mark which loops are contained within
3063 loops, and invalidate loops that have setjmp. */
3065 num_loops = 0;
3066 current_loop = NULL;
3067 for (insn = f; insn; insn = NEXT_INSN (insn))
3069 if (NOTE_P (insn))
3070 switch (NOTE_LINE_NUMBER (insn))
3072 case NOTE_INSN_LOOP_BEG:
3073 next_loop = loops->array + num_loops;
3074 next_loop->num = num_loops;
3075 num_loops++;
3076 next_loop->start = insn;
3077 next_loop->outer = current_loop;
3078 current_loop = next_loop;
3079 break;
3081 case NOTE_INSN_LOOP_END:
3082 if (! current_loop)
3083 abort ();
3085 current_loop->end = insn;
3086 current_loop = current_loop->outer;
3087 break;
3089 default:
3090 break;
3093 if (CALL_P (insn)
3094 && find_reg_note (insn, REG_SETJMP, NULL))
3096 /* In this case, we must invalidate our current loop and any
3097 enclosing loop. */
3098 for (loop = current_loop; loop; loop = loop->outer)
3100 loop->invalid = 1;
3101 if (loop_dump_stream)
3102 fprintf (loop_dump_stream,
3103 "\nLoop at %d ignored due to setjmp.\n",
3104 INSN_UID (loop->start));
3108 /* Note that this will mark the NOTE_INSN_LOOP_END note as being in the
3109 enclosing loop, but this doesn't matter. */
3110 uid_loop[INSN_UID (insn)] = current_loop;
3113 /* Any loop containing a label used in an initializer must be invalidated,
3114 because it can be jumped into from anywhere. */
3115 for (label = forced_labels; label; label = XEXP (label, 1))
3116 invalidate_loops_containing_label (XEXP (label, 0));
3118 /* Any loop containing a label used for an exception handler must be
3119 invalidated, because it can be jumped into from anywhere. */
3120 for_each_eh_label (invalidate_loops_containing_label);
3122 /* Now scan all insn's in the function. If any JUMP_INSN branches into a
3123 loop that it is not contained within, that loop is marked invalid.
3124 If any INSN or CALL_INSN uses a label's address, then the loop containing
3125 that label is marked invalid, because it could be jumped into from
3126 anywhere.
3128 Also look for blocks of code ending in an unconditional branch that
3129 exits the loop. If such a block is surrounded by a conditional
3130 branch around the block, move the block elsewhere (see below) and
3131 invert the jump to point to the code block. This may eliminate a
3132 label in our loop and will simplify processing by both us and a
3133 possible second cse pass. */
3135 for (insn = f; insn; insn = NEXT_INSN (insn))
3136 if (INSN_P (insn))
3138 struct loop *this_loop = uid_loop[INSN_UID (insn)];
3140 if (NONJUMP_INSN_P (insn) || CALL_P (insn))
3142 rtx note = find_reg_note (insn, REG_LABEL, NULL_RTX);
3143 if (note)
3144 invalidate_loops_containing_label (XEXP (note, 0));
3147 if (!JUMP_P (insn))
3148 continue;
3150 mark_loop_jump (PATTERN (insn), this_loop);
3152 /* See if this is an unconditional branch outside the loop. */
3153 if (this_loop
3154 && (GET_CODE (PATTERN (insn)) == RETURN
3155 || (any_uncondjump_p (insn)
3156 && onlyjump_p (insn)
3157 && (uid_loop[INSN_UID (JUMP_LABEL (insn))]
3158 != this_loop)))
3159 && get_max_uid () < max_uid_for_loop)
3161 rtx p;
3162 rtx our_next = next_real_insn (insn);
3163 rtx last_insn_to_move = NEXT_INSN (insn);
3164 struct loop *dest_loop;
3165 struct loop *outer_loop = NULL;
3167 /* Go backwards until we reach the start of the loop, a label,
3168 or a JUMP_INSN. */
3169 for (p = PREV_INSN (insn);
3170 !LABEL_P (p)
3171 && ! (NOTE_P (p)
3172 && NOTE_LINE_NUMBER (p) == NOTE_INSN_LOOP_BEG)
3173 && !JUMP_P (p);
3174 p = PREV_INSN (p))
3177 /* Check for the case where we have a jump to an inner nested
3178 loop, and do not perform the optimization in that case. */
3180 if (JUMP_LABEL (insn))
3182 dest_loop = uid_loop[INSN_UID (JUMP_LABEL (insn))];
3183 if (dest_loop)
3185 for (outer_loop = dest_loop; outer_loop;
3186 outer_loop = outer_loop->outer)
3187 if (outer_loop == this_loop)
3188 break;
3192 /* Make sure that the target of P is within the current loop. */
3194 if (JUMP_P (p) && JUMP_LABEL (p)
3195 && uid_loop[INSN_UID (JUMP_LABEL (p))] != this_loop)
3196 outer_loop = this_loop;
3198 /* If we stopped on a JUMP_INSN to the next insn after INSN,
3199 we have a block of code to try to move.
3201 We look backward and then forward from the target of INSN
3202 to find a BARRIER at the same loop depth as the target.
3203 If we find such a BARRIER, we make a new label for the start
3204 of the block, invert the jump in P and point it to that label,
3205 and move the block of code to the spot we found. */
3207 if (! outer_loop
3208 && JUMP_P (p)
3209 && JUMP_LABEL (p) != 0
3210 /* Just ignore jumps to labels that were never emitted.
3211 These always indicate compilation errors. */
3212 && INSN_UID (JUMP_LABEL (p)) != 0
3213 && any_condjump_p (p) && onlyjump_p (p)
3214 && next_real_insn (JUMP_LABEL (p)) == our_next
3215 /* If it's not safe to move the sequence, then we
3216 mustn't try. */
3217 && insns_safe_to_move_p (p, NEXT_INSN (insn),
3218 &last_insn_to_move))
3220 rtx target
3221 = JUMP_LABEL (insn) ? JUMP_LABEL (insn) : get_last_insn ();
3222 struct loop *target_loop = uid_loop[INSN_UID (target)];
3223 rtx loc, loc2;
3224 rtx tmp;
3226 /* Search for possible garbage past the conditional jumps
3227 and look for the last barrier. */
3228 for (tmp = last_insn_to_move;
3229 tmp && !LABEL_P (tmp); tmp = NEXT_INSN (tmp))
3230 if (BARRIER_P (tmp))
3231 last_insn_to_move = tmp;
3233 for (loc = target; loc; loc = PREV_INSN (loc))
3234 if (BARRIER_P (loc)
3235 /* Don't move things inside a tablejump. */
3236 && ((loc2 = next_nonnote_insn (loc)) == 0
3237 || !LABEL_P (loc2)
3238 || (loc2 = next_nonnote_insn (loc2)) == 0
3239 || !JUMP_P (loc2)
3240 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
3241 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
3242 && uid_loop[INSN_UID (loc)] == target_loop)
3243 break;
3245 if (loc == 0)
3246 for (loc = target; loc; loc = NEXT_INSN (loc))
3247 if (BARRIER_P (loc)
3248 /* Don't move things inside a tablejump. */
3249 && ((loc2 = next_nonnote_insn (loc)) == 0
3250 || !LABEL_P (loc2)
3251 || (loc2 = next_nonnote_insn (loc2)) == 0
3252 || !JUMP_P (loc2)
3253 || (GET_CODE (PATTERN (loc2)) != ADDR_VEC
3254 && GET_CODE (PATTERN (loc2)) != ADDR_DIFF_VEC))
3255 && uid_loop[INSN_UID (loc)] == target_loop)
3256 break;
3258 if (loc)
3260 rtx cond_label = JUMP_LABEL (p);
3261 rtx new_label = get_label_after (p);
3263 /* Ensure our label doesn't go away. */
3264 LABEL_NUSES (cond_label)++;
3266 /* Verify that uid_loop is large enough and that
3267 we can invert P. */
3268 if (invert_jump (p, new_label, 1))
3270 rtx q, r;
3272 /* If no suitable BARRIER was found, create a suitable
3273 one before TARGET. Since TARGET is a fall through
3274 path, we'll need to insert a jump around our block
3275 and add a BARRIER before TARGET.
3277 This creates an extra unconditional jump outside
3278 the loop. However, the benefits of removing rarely
3279 executed instructions from inside the loop usually
3280 outweighs the cost of the extra unconditional jump
3281 outside the loop. */
3282 if (loc == 0)
3284 rtx temp;
3286 temp = gen_jump (JUMP_LABEL (insn));
3287 temp = emit_jump_insn_before (temp, target);
3288 JUMP_LABEL (temp) = JUMP_LABEL (insn);
3289 LABEL_NUSES (JUMP_LABEL (insn))++;
3290 loc = emit_barrier_before (target);
3293 /* Include the BARRIER after INSN and copy the
3294 block after LOC. */
3295 if (squeeze_notes (&new_label, &last_insn_to_move))
3296 abort ();
3297 reorder_insns (new_label, last_insn_to_move, loc);
3299 /* All those insns are now in TARGET_LOOP. */
3300 for (q = new_label;
3301 q != NEXT_INSN (last_insn_to_move);
3302 q = NEXT_INSN (q))
3303 uid_loop[INSN_UID (q)] = target_loop;
3305 /* The label jumped to by INSN is no longer a loop
3306 exit. Unless INSN does not have a label (e.g.,
3307 it is a RETURN insn), search loop->exit_labels
3308 to find its label_ref, and remove it. Also turn
3309 off LABEL_OUTSIDE_LOOP_P bit. */
3310 if (JUMP_LABEL (insn))
3312 for (q = 0, r = this_loop->exit_labels;
3314 q = r, r = LABEL_NEXTREF (r))
3315 if (XEXP (r, 0) == JUMP_LABEL (insn))
3317 LABEL_OUTSIDE_LOOP_P (r) = 0;
3318 if (q)
3319 LABEL_NEXTREF (q) = LABEL_NEXTREF (r);
3320 else
3321 this_loop->exit_labels = LABEL_NEXTREF (r);
3322 break;
3325 for (loop = this_loop; loop && loop != target_loop;
3326 loop = loop->outer)
3327 loop->exit_count--;
3329 /* If we didn't find it, then something is
3330 wrong. */
3331 if (! r)
3332 abort ();
3335 /* P is now a jump outside the loop, so it must be put
3336 in loop->exit_labels, and marked as such.
3337 The easiest way to do this is to just call
3338 mark_loop_jump again for P. */
3339 mark_loop_jump (PATTERN (p), this_loop);
3341 /* If INSN now jumps to the insn after it,
3342 delete INSN. */
3343 if (JUMP_LABEL (insn) != 0
3344 && (next_real_insn (JUMP_LABEL (insn))
3345 == next_real_insn (insn)))
3346 delete_related_insns (insn);
3349 /* Continue the loop after where the conditional
3350 branch used to jump, since the only branch insn
3351 in the block (if it still remains) is an inter-loop
3352 branch and hence needs no processing. */
3353 insn = NEXT_INSN (cond_label);
3355 if (--LABEL_NUSES (cond_label) == 0)
3356 delete_related_insns (cond_label);
3358 /* This loop will be continued with NEXT_INSN (insn). */
3359 insn = PREV_INSN (insn);
3366 /* If any label in X jumps to a loop different from LOOP_NUM and any of the
3367 loops it is contained in, mark the target loop invalid.
3369 For speed, we assume that X is part of a pattern of a JUMP_INSN. */
3371 static void
3372 mark_loop_jump (rtx x, struct loop *loop)
3374 struct loop *dest_loop;
3375 struct loop *outer_loop;
3376 int i;
3378 switch (GET_CODE (x))
3380 case PC:
3381 case USE:
3382 case CLOBBER:
3383 case REG:
3384 case MEM:
3385 case CONST_INT:
3386 case CONST_DOUBLE:
3387 case RETURN:
3388 return;
3390 case CONST:
3391 /* There could be a label reference in here. */
3392 mark_loop_jump (XEXP (x, 0), loop);
3393 return;
3395 case PLUS:
3396 case MINUS:
3397 case MULT:
3398 mark_loop_jump (XEXP (x, 0), loop);
3399 mark_loop_jump (XEXP (x, 1), loop);
3400 return;
3402 case LO_SUM:
3403 /* This may refer to a LABEL_REF or SYMBOL_REF. */
3404 mark_loop_jump (XEXP (x, 1), loop);
3405 return;
3407 case SIGN_EXTEND:
3408 case ZERO_EXTEND:
3409 mark_loop_jump (XEXP (x, 0), loop);
3410 return;
3412 case LABEL_REF:
3413 dest_loop = uid_loop[INSN_UID (XEXP (x, 0))];
3415 /* Link together all labels that branch outside the loop. This
3416 is used by final_[bg]iv_value and the loop unrolling code. Also
3417 mark this LABEL_REF so we know that this branch should predict
3418 false. */
3420 /* A check to make sure the label is not in an inner nested loop,
3421 since this does not count as a loop exit. */
3422 if (dest_loop)
3424 for (outer_loop = dest_loop; outer_loop;
3425 outer_loop = outer_loop->outer)
3426 if (outer_loop == loop)
3427 break;
3429 else
3430 outer_loop = NULL;
3432 if (loop && ! outer_loop)
3434 LABEL_OUTSIDE_LOOP_P (x) = 1;
3435 LABEL_NEXTREF (x) = loop->exit_labels;
3436 loop->exit_labels = x;
3438 for (outer_loop = loop;
3439 outer_loop && outer_loop != dest_loop;
3440 outer_loop = outer_loop->outer)
3441 outer_loop->exit_count++;
3444 /* If this is inside a loop, but not in the current loop or one enclosed
3445 by it, it invalidates at least one loop. */
3447 if (! dest_loop)
3448 return;
3450 /* We must invalidate every nested loop containing the target of this
3451 label, except those that also contain the jump insn. */
3453 for (; dest_loop; dest_loop = dest_loop->outer)
3455 /* Stop when we reach a loop that also contains the jump insn. */
3456 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3457 if (dest_loop == outer_loop)
3458 return;
3460 /* If we get here, we know we need to invalidate a loop. */
3461 if (loop_dump_stream && ! dest_loop->invalid)
3462 fprintf (loop_dump_stream,
3463 "\nLoop at %d ignored due to multiple entry points.\n",
3464 INSN_UID (dest_loop->start));
3466 dest_loop->invalid = 1;
3468 return;
3470 case SET:
3471 /* If this is not setting pc, ignore. */
3472 if (SET_DEST (x) == pc_rtx)
3473 mark_loop_jump (SET_SRC (x), loop);
3474 return;
3476 case IF_THEN_ELSE:
3477 mark_loop_jump (XEXP (x, 1), loop);
3478 mark_loop_jump (XEXP (x, 2), loop);
3479 return;
3481 case PARALLEL:
3482 case ADDR_VEC:
3483 for (i = 0; i < XVECLEN (x, 0); i++)
3484 mark_loop_jump (XVECEXP (x, 0, i), loop);
3485 return;
3487 case ADDR_DIFF_VEC:
3488 for (i = 0; i < XVECLEN (x, 1); i++)
3489 mark_loop_jump (XVECEXP (x, 1, i), loop);
3490 return;
3492 default:
3493 /* Strictly speaking this is not a jump into the loop, only a possible
3494 jump out of the loop. However, we have no way to link the destination
3495 of this jump onto the list of exit labels. To be safe we mark this
3496 loop and any containing loops as invalid. */
3497 if (loop)
3499 for (outer_loop = loop; outer_loop; outer_loop = outer_loop->outer)
3501 if (loop_dump_stream && ! outer_loop->invalid)
3502 fprintf (loop_dump_stream,
3503 "\nLoop at %d ignored due to unknown exit jump.\n",
3504 INSN_UID (outer_loop->start));
3505 outer_loop->invalid = 1;
3508 return;
3512 /* Return nonzero if there is a label in the range from
3513 insn INSN to and including the insn whose luid is END
3514 INSN must have an assigned luid (i.e., it must not have
3515 been previously created by loop.c). */
3517 static int
3518 labels_in_range_p (rtx insn, int end)
3520 while (insn && INSN_LUID (insn) <= end)
3522 if (LABEL_P (insn))
3523 return 1;
3524 insn = NEXT_INSN (insn);
3527 return 0;
3530 /* Record that a memory reference X is being set. */
3532 static void
3533 note_addr_stored (rtx x, rtx y ATTRIBUTE_UNUSED,
3534 void *data ATTRIBUTE_UNUSED)
3536 struct loop_info *loop_info = data;
3538 if (x == 0 || !MEM_P (x))
3539 return;
3541 /* Count number of memory writes.
3542 This affects heuristics in strength_reduce. */
3543 loop_info->num_mem_sets++;
3545 /* BLKmode MEM means all memory is clobbered. */
3546 if (GET_MODE (x) == BLKmode)
3548 if (MEM_READONLY_P (x))
3549 loop_info->unknown_constant_address_altered = 1;
3550 else
3551 loop_info->unknown_address_altered = 1;
3553 return;
3556 loop_info->store_mems = gen_rtx_EXPR_LIST (VOIDmode, x,
3557 loop_info->store_mems);
3560 /* X is a value modified by an INSN that references a biv inside a loop
3561 exit test (i.e., X is somehow related to the value of the biv). If X
3562 is a pseudo that is used more than once, then the biv is (effectively)
3563 used more than once. DATA is a pointer to a loop_regs structure. */
3565 static void
3566 note_set_pseudo_multiple_uses (rtx x, rtx y ATTRIBUTE_UNUSED, void *data)
3568 struct loop_regs *regs = (struct loop_regs *) data;
3570 if (x == 0)
3571 return;
3573 while (GET_CODE (x) == STRICT_LOW_PART
3574 || GET_CODE (x) == SIGN_EXTRACT
3575 || GET_CODE (x) == ZERO_EXTRACT
3576 || GET_CODE (x) == SUBREG)
3577 x = XEXP (x, 0);
3579 if (!REG_P (x) || REGNO (x) < FIRST_PSEUDO_REGISTER)
3580 return;
3582 /* If we do not have usage information, or if we know the register
3583 is used more than once, note that fact for check_dbra_loop. */
3584 if (REGNO (x) >= max_reg_before_loop
3585 || ! regs->array[REGNO (x)].single_usage
3586 || regs->array[REGNO (x)].single_usage == const0_rtx)
3587 regs->multiple_uses = 1;
3590 /* Return nonzero if the rtx X is invariant over the current loop.
3592 The value is 2 if we refer to something only conditionally invariant.
3594 A memory ref is invariant if it is not volatile and does not conflict
3595 with anything stored in `loop_info->store_mems'. */
3597 static int
3598 loop_invariant_p (const struct loop *loop, rtx x)
3600 struct loop_info *loop_info = LOOP_INFO (loop);
3601 struct loop_regs *regs = LOOP_REGS (loop);
3602 int i;
3603 enum rtx_code code;
3604 const char *fmt;
3605 int conditional = 0;
3606 rtx mem_list_entry;
3608 if (x == 0)
3609 return 1;
3610 code = GET_CODE (x);
3611 switch (code)
3613 case CONST_INT:
3614 case CONST_DOUBLE:
3615 case SYMBOL_REF:
3616 case CONST:
3617 return 1;
3619 case LABEL_REF:
3620 return 1;
3622 case PC:
3623 case CC0:
3624 case UNSPEC_VOLATILE:
3625 return 0;
3627 case REG:
3628 if ((x == frame_pointer_rtx || x == hard_frame_pointer_rtx
3629 || x == arg_pointer_rtx || x == pic_offset_table_rtx)
3630 && ! current_function_has_nonlocal_goto)
3631 return 1;
3633 if (LOOP_INFO (loop)->has_call
3634 && REGNO (x) < FIRST_PSEUDO_REGISTER && call_used_regs[REGNO (x)])
3635 return 0;
3637 /* Out-of-range regs can occur when we are called from unrolling.
3638 These registers created by the unroller are set in the loop,
3639 hence are never invariant.
3640 Other out-of-range regs can be generated by load_mems; those that
3641 are written to in the loop are not invariant, while those that are
3642 not written to are invariant. It would be easy for load_mems
3643 to set n_times_set correctly for these registers, however, there
3644 is no easy way to distinguish them from registers created by the
3645 unroller. */
3647 if (REGNO (x) >= (unsigned) regs->num)
3648 return 0;
3650 if (regs->array[REGNO (x)].set_in_loop < 0)
3651 return 2;
3653 return regs->array[REGNO (x)].set_in_loop == 0;
3655 case MEM:
3656 /* Volatile memory references must be rejected. Do this before
3657 checking for read-only items, so that volatile read-only items
3658 will be rejected also. */
3659 if (MEM_VOLATILE_P (x))
3660 return 0;
3662 /* See if there is any dependence between a store and this load. */
3663 mem_list_entry = loop_info->store_mems;
3664 while (mem_list_entry)
3666 if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
3667 x, rtx_varies_p))
3668 return 0;
3670 mem_list_entry = XEXP (mem_list_entry, 1);
3673 /* It's not invalidated by a store in memory
3674 but we must still verify the address is invariant. */
3675 break;
3677 case ASM_OPERANDS:
3678 /* Don't mess with insns declared volatile. */
3679 if (MEM_VOLATILE_P (x))
3680 return 0;
3681 break;
3683 default:
3684 break;
3687 fmt = GET_RTX_FORMAT (code);
3688 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3690 if (fmt[i] == 'e')
3692 int tem = loop_invariant_p (loop, XEXP (x, i));
3693 if (tem == 0)
3694 return 0;
3695 if (tem == 2)
3696 conditional = 1;
3698 else if (fmt[i] == 'E')
3700 int j;
3701 for (j = 0; j < XVECLEN (x, i); j++)
3703 int tem = loop_invariant_p (loop, XVECEXP (x, i, j));
3704 if (tem == 0)
3705 return 0;
3706 if (tem == 2)
3707 conditional = 1;
3713 return 1 + conditional;
3716 /* Return nonzero if all the insns in the loop that set REG
3717 are INSN and the immediately following insns,
3718 and if each of those insns sets REG in an invariant way
3719 (not counting uses of REG in them).
3721 The value is 2 if some of these insns are only conditionally invariant.
3723 We assume that INSN itself is the first set of REG
3724 and that its source is invariant. */
3726 static int
3727 consec_sets_invariant_p (const struct loop *loop, rtx reg, int n_sets,
3728 rtx insn)
3730 struct loop_regs *regs = LOOP_REGS (loop);
3731 rtx p = insn;
3732 unsigned int regno = REGNO (reg);
3733 rtx temp;
3734 /* Number of sets we have to insist on finding after INSN. */
3735 int count = n_sets - 1;
3736 int old = regs->array[regno].set_in_loop;
3737 int value = 0;
3738 int this;
3740 /* If N_SETS hit the limit, we can't rely on its value. */
3741 if (n_sets == 127)
3742 return 0;
3744 regs->array[regno].set_in_loop = 0;
3746 while (count > 0)
3748 enum rtx_code code;
3749 rtx set;
3751 p = NEXT_INSN (p);
3752 code = GET_CODE (p);
3754 /* If library call, skip to end of it. */
3755 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
3756 p = XEXP (temp, 0);
3758 this = 0;
3759 if (code == INSN
3760 && (set = single_set (p))
3761 && REG_P (SET_DEST (set))
3762 && REGNO (SET_DEST (set)) == regno)
3764 this = loop_invariant_p (loop, SET_SRC (set));
3765 if (this != 0)
3766 value |= this;
3767 else if ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX)))
3769 /* If this is a libcall, then any invariant REG_EQUAL note is OK.
3770 If this is an ordinary insn, then only CONSTANT_P REG_EQUAL
3771 notes are OK. */
3772 this = (CONSTANT_P (XEXP (temp, 0))
3773 || (find_reg_note (p, REG_RETVAL, NULL_RTX)
3774 && loop_invariant_p (loop, XEXP (temp, 0))));
3775 if (this != 0)
3776 value |= this;
3779 if (this != 0)
3780 count--;
3781 else if (code != NOTE)
3783 regs->array[regno].set_in_loop = old;
3784 return 0;
3788 regs->array[regno].set_in_loop = old;
3789 /* If loop_invariant_p ever returned 2, we return 2. */
3790 return 1 + (value & 2);
3793 /* Look at all uses (not sets) of registers in X. For each, if it is
3794 the single use, set USAGE[REGNO] to INSN; if there was a previous use in
3795 a different insn, set USAGE[REGNO] to const0_rtx. */
3797 static void
3798 find_single_use_in_loop (struct loop_regs *regs, rtx insn, rtx x)
3800 enum rtx_code code = GET_CODE (x);
3801 const char *fmt = GET_RTX_FORMAT (code);
3802 int i, j;
3804 if (code == REG)
3805 regs->array[REGNO (x)].single_usage
3806 = (regs->array[REGNO (x)].single_usage != 0
3807 && regs->array[REGNO (x)].single_usage != insn)
3808 ? const0_rtx : insn;
3810 else if (code == SET)
3812 /* Don't count SET_DEST if it is a REG; otherwise count things
3813 in SET_DEST because if a register is partially modified, it won't
3814 show up as a potential movable so we don't care how USAGE is set
3815 for it. */
3816 if (!REG_P (SET_DEST (x)))
3817 find_single_use_in_loop (regs, insn, SET_DEST (x));
3818 find_single_use_in_loop (regs, insn, SET_SRC (x));
3820 else
3821 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3823 if (fmt[i] == 'e' && XEXP (x, i) != 0)
3824 find_single_use_in_loop (regs, insn, XEXP (x, i));
3825 else if (fmt[i] == 'E')
3826 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
3827 find_single_use_in_loop (regs, insn, XVECEXP (x, i, j));
3831 /* Count and record any set in X which is contained in INSN. Update
3832 REGS->array[I].MAY_NOT_OPTIMIZE and LAST_SET for any register I set
3833 in X. */
3835 static void
3836 count_one_set (struct loop_regs *regs, rtx insn, rtx x, rtx *last_set)
3838 if (GET_CODE (x) == CLOBBER && REG_P (XEXP (x, 0)))
3839 /* Don't move a reg that has an explicit clobber.
3840 It's not worth the pain to try to do it correctly. */
3841 regs->array[REGNO (XEXP (x, 0))].may_not_optimize = 1;
3843 if (GET_CODE (x) == SET || GET_CODE (x) == CLOBBER)
3845 rtx dest = SET_DEST (x);
3846 while (GET_CODE (dest) == SUBREG
3847 || GET_CODE (dest) == ZERO_EXTRACT
3848 || GET_CODE (dest) == SIGN_EXTRACT
3849 || GET_CODE (dest) == STRICT_LOW_PART)
3850 dest = XEXP (dest, 0);
3851 if (REG_P (dest))
3853 int i;
3854 int regno = REGNO (dest);
3855 for (i = 0; i < LOOP_REGNO_NREGS (regno, dest); i++)
3857 /* If this is the first setting of this reg
3858 in current basic block, and it was set before,
3859 it must be set in two basic blocks, so it cannot
3860 be moved out of the loop. */
3861 if (regs->array[regno].set_in_loop > 0
3862 && last_set[regno] == 0)
3863 regs->array[regno+i].may_not_optimize = 1;
3864 /* If this is not first setting in current basic block,
3865 see if reg was used in between previous one and this.
3866 If so, neither one can be moved. */
3867 if (last_set[regno] != 0
3868 && reg_used_between_p (dest, last_set[regno], insn))
3869 regs->array[regno+i].may_not_optimize = 1;
3870 if (regs->array[regno+i].set_in_loop < 127)
3871 ++regs->array[regno+i].set_in_loop;
3872 last_set[regno+i] = insn;
3878 /* Given a loop that is bounded by LOOP->START and LOOP->END and that
3879 is entered at LOOP->SCAN_START, return 1 if the register set in SET
3880 contained in insn INSN is used by any insn that precedes INSN in
3881 cyclic order starting from the loop entry point.
3883 We don't want to use INSN_LUID here because if we restrict INSN to those
3884 that have a valid INSN_LUID, it means we cannot move an invariant out
3885 from an inner loop past two loops. */
3887 static int
3888 loop_reg_used_before_p (const struct loop *loop, rtx set, rtx insn)
3890 rtx reg = SET_DEST (set);
3891 rtx p;
3893 /* Scan forward checking for register usage. If we hit INSN, we
3894 are done. Otherwise, if we hit LOOP->END, wrap around to LOOP->START. */
3895 for (p = loop->scan_start; p != insn; p = NEXT_INSN (p))
3897 if (INSN_P (p) && reg_overlap_mentioned_p (reg, PATTERN (p)))
3898 return 1;
3900 if (p == loop->end)
3901 p = loop->start;
3904 return 0;
3908 /* Information we collect about arrays that we might want to prefetch. */
3909 struct prefetch_info
3911 struct iv_class *class; /* Class this prefetch is based on. */
3912 struct induction *giv; /* GIV this prefetch is based on. */
3913 rtx base_address; /* Start prefetching from this address plus
3914 index. */
3915 HOST_WIDE_INT index;
3916 HOST_WIDE_INT stride; /* Prefetch stride in bytes in each
3917 iteration. */
3918 unsigned int bytes_accessed; /* Sum of sizes of all accesses to this
3919 prefetch area in one iteration. */
3920 unsigned int total_bytes; /* Total bytes loop will access in this block.
3921 This is set only for loops with known
3922 iteration counts and is 0xffffffff
3923 otherwise. */
3924 int prefetch_in_loop; /* Number of prefetch insns in loop. */
3925 int prefetch_before_loop; /* Number of prefetch insns before loop. */
3926 unsigned int write : 1; /* 1 for read/write prefetches. */
3929 /* Data used by check_store function. */
3930 struct check_store_data
3932 rtx mem_address;
3933 int mem_write;
3936 static void check_store (rtx, rtx, void *);
3937 static void emit_prefetch_instructions (struct loop *);
3938 static int rtx_equal_for_prefetch_p (rtx, rtx);
3940 /* Set mem_write when mem_address is found. Used as callback to
3941 note_stores. */
3942 static void
3943 check_store (rtx x, rtx pat ATTRIBUTE_UNUSED, void *data)
3945 struct check_store_data *d = (struct check_store_data *) data;
3947 if ((MEM_P (x)) && rtx_equal_p (d->mem_address, XEXP (x, 0)))
3948 d->mem_write = 1;
3951 /* Like rtx_equal_p, but attempts to swap commutative operands. This is
3952 important to get some addresses combined. Later more sophisticated
3953 transformations can be added when necessary.
3955 ??? Same trick with swapping operand is done at several other places.
3956 It can be nice to develop some common way to handle this. */
3958 static int
3959 rtx_equal_for_prefetch_p (rtx x, rtx y)
3961 int i;
3962 int j;
3963 enum rtx_code code = GET_CODE (x);
3964 const char *fmt;
3966 if (x == y)
3967 return 1;
3968 if (code != GET_CODE (y))
3969 return 0;
3971 if (COMMUTATIVE_ARITH_P (x))
3973 return ((rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 0))
3974 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 1)))
3975 || (rtx_equal_for_prefetch_p (XEXP (x, 0), XEXP (y, 1))
3976 && rtx_equal_for_prefetch_p (XEXP (x, 1), XEXP (y, 0))));
3979 /* Compare the elements. If any pair of corresponding elements fails to
3980 match, return 0 for the whole thing. */
3982 fmt = GET_RTX_FORMAT (code);
3983 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
3985 switch (fmt[i])
3987 case 'w':
3988 if (XWINT (x, i) != XWINT (y, i))
3989 return 0;
3990 break;
3992 case 'i':
3993 if (XINT (x, i) != XINT (y, i))
3994 return 0;
3995 break;
3997 case 'E':
3998 /* Two vectors must have the same length. */
3999 if (XVECLEN (x, i) != XVECLEN (y, i))
4000 return 0;
4002 /* And the corresponding elements must match. */
4003 for (j = 0; j < XVECLEN (x, i); j++)
4004 if (rtx_equal_for_prefetch_p (XVECEXP (x, i, j),
4005 XVECEXP (y, i, j)) == 0)
4006 return 0;
4007 break;
4009 case 'e':
4010 if (rtx_equal_for_prefetch_p (XEXP (x, i), XEXP (y, i)) == 0)
4011 return 0;
4012 break;
4014 case 's':
4015 if (strcmp (XSTR (x, i), XSTR (y, i)))
4016 return 0;
4017 break;
4019 case 'u':
4020 /* These are just backpointers, so they don't matter. */
4021 break;
4023 case '0':
4024 break;
4026 /* It is believed that rtx's at this level will never
4027 contain anything but integers and other rtx's,
4028 except for within LABEL_REFs and SYMBOL_REFs. */
4029 default:
4030 abort ();
4033 return 1;
4036 /* Remove constant addition value from the expression X (when present)
4037 and return it. */
4039 static HOST_WIDE_INT
4040 remove_constant_addition (rtx *x)
4042 HOST_WIDE_INT addval = 0;
4043 rtx exp = *x;
4045 /* Avoid clobbering a shared CONST expression. */
4046 if (GET_CODE (exp) == CONST)
4048 if (GET_CODE (XEXP (exp, 0)) == PLUS
4049 && GET_CODE (XEXP (XEXP (exp, 0), 0)) == SYMBOL_REF
4050 && GET_CODE (XEXP (XEXP (exp, 0), 1)) == CONST_INT)
4052 *x = XEXP (XEXP (exp, 0), 0);
4053 return INTVAL (XEXP (XEXP (exp, 0), 1));
4055 return 0;
4058 if (GET_CODE (exp) == CONST_INT)
4060 addval = INTVAL (exp);
4061 *x = const0_rtx;
4064 /* For plus expression recurse on ourself. */
4065 else if (GET_CODE (exp) == PLUS)
4067 addval += remove_constant_addition (&XEXP (exp, 0));
4068 addval += remove_constant_addition (&XEXP (exp, 1));
4070 /* In case our parameter was constant, remove extra zero from the
4071 expression. */
4072 if (XEXP (exp, 0) == const0_rtx)
4073 *x = XEXP (exp, 1);
4074 else if (XEXP (exp, 1) == const0_rtx)
4075 *x = XEXP (exp, 0);
4078 return addval;
4081 /* Attempt to identify accesses to arrays that are most likely to cause cache
4082 misses, and emit prefetch instructions a few prefetch blocks forward.
4084 To detect the arrays we use the GIV information that was collected by the
4085 strength reduction pass.
4087 The prefetch instructions are generated after the GIV information is done
4088 and before the strength reduction process. The new GIVs are injected into
4089 the strength reduction tables, so the prefetch addresses are optimized as
4090 well.
4092 GIVs are split into base address, stride, and constant addition values.
4093 GIVs with the same address, stride and close addition values are combined
4094 into a single prefetch. Also writes to GIVs are detected, so that prefetch
4095 for write instructions can be used for the block we write to, on machines
4096 that support write prefetches.
4098 Several heuristics are used to determine when to prefetch. They are
4099 controlled by defined symbols that can be overridden for each target. */
4101 static void
4102 emit_prefetch_instructions (struct loop *loop)
4104 int num_prefetches = 0;
4105 int num_real_prefetches = 0;
4106 int num_real_write_prefetches = 0;
4107 int num_prefetches_before = 0;
4108 int num_write_prefetches_before = 0;
4109 int ahead = 0;
4110 int i;
4111 struct iv_class *bl;
4112 struct induction *iv;
4113 struct prefetch_info info[MAX_PREFETCHES];
4114 struct loop_ivs *ivs = LOOP_IVS (loop);
4116 if (!HAVE_prefetch)
4117 return;
4119 /* Consider only loops w/o calls. When a call is done, the loop is probably
4120 slow enough to read the memory. */
4121 if (PREFETCH_NO_CALL && LOOP_INFO (loop)->has_call)
4123 if (loop_dump_stream)
4124 fprintf (loop_dump_stream, "Prefetch: ignoring loop: has call.\n");
4126 return;
4129 /* Don't prefetch in loops known to have few iterations. */
4130 if (PREFETCH_NO_LOW_LOOPCNT
4131 && LOOP_INFO (loop)->n_iterations
4132 && LOOP_INFO (loop)->n_iterations <= PREFETCH_LOW_LOOPCNT)
4134 if (loop_dump_stream)
4135 fprintf (loop_dump_stream,
4136 "Prefetch: ignoring loop: not enough iterations.\n");
4137 return;
4140 /* Search all induction variables and pick those interesting for the prefetch
4141 machinery. */
4142 for (bl = ivs->list; bl; bl = bl->next)
4144 struct induction *biv = bl->biv, *biv1;
4145 int basestride = 0;
4147 biv1 = biv;
4149 /* Expect all BIVs to be executed in each iteration. This makes our
4150 analysis more conservative. */
4151 while (biv1)
4153 /* Discard non-constant additions that we can't handle well yet, and
4154 BIVs that are executed multiple times; such BIVs ought to be
4155 handled in the nested loop. We accept not_every_iteration BIVs,
4156 since these only result in larger strides and make our
4157 heuristics more conservative. */
4158 if (GET_CODE (biv->add_val) != CONST_INT)
4160 if (loop_dump_stream)
4162 fprintf (loop_dump_stream,
4163 "Prefetch: ignoring biv %d: non-constant addition at insn %d:",
4164 REGNO (biv->src_reg), INSN_UID (biv->insn));
4165 print_rtl (loop_dump_stream, biv->add_val);
4166 fprintf (loop_dump_stream, "\n");
4168 break;
4171 if (biv->maybe_multiple)
4173 if (loop_dump_stream)
4175 fprintf (loop_dump_stream,
4176 "Prefetch: ignoring biv %d: maybe_multiple at insn %i:",
4177 REGNO (biv->src_reg), INSN_UID (biv->insn));
4178 print_rtl (loop_dump_stream, biv->add_val);
4179 fprintf (loop_dump_stream, "\n");
4181 break;
4184 basestride += INTVAL (biv1->add_val);
4185 biv1 = biv1->next_iv;
4188 if (biv1 || !basestride)
4189 continue;
4191 for (iv = bl->giv; iv; iv = iv->next_iv)
4193 rtx address;
4194 rtx temp;
4195 HOST_WIDE_INT index = 0;
4196 int add = 1;
4197 HOST_WIDE_INT stride = 0;
4198 int stride_sign = 1;
4199 struct check_store_data d;
4200 const char *ignore_reason = NULL;
4201 int size = GET_MODE_SIZE (GET_MODE (iv));
4203 /* See whether an induction variable is interesting to us and if
4204 not, report the reason. */
4205 if (iv->giv_type != DEST_ADDR)
4206 ignore_reason = "giv is not a destination address";
4208 /* We are interested only in constant stride memory references
4209 in order to be able to compute density easily. */
4210 else if (GET_CODE (iv->mult_val) != CONST_INT)
4211 ignore_reason = "stride is not constant";
4213 else
4215 stride = INTVAL (iv->mult_val) * basestride;
4216 if (stride < 0)
4218 stride = -stride;
4219 stride_sign = -1;
4222 /* On some targets, reversed order prefetches are not
4223 worthwhile. */
4224 if (PREFETCH_NO_REVERSE_ORDER && stride_sign < 0)
4225 ignore_reason = "reversed order stride";
4227 /* Prefetch of accesses with an extreme stride might not be
4228 worthwhile, either. */
4229 else if (PREFETCH_NO_EXTREME_STRIDE
4230 && stride > PREFETCH_EXTREME_STRIDE)
4231 ignore_reason = "extreme stride";
4233 /* Ignore GIVs with varying add values; we can't predict the
4234 value for the next iteration. */
4235 else if (!loop_invariant_p (loop, iv->add_val))
4236 ignore_reason = "giv has varying add value";
4238 /* Ignore GIVs in the nested loops; they ought to have been
4239 handled already. */
4240 else if (iv->maybe_multiple)
4241 ignore_reason = "giv is in nested loop";
4244 if (ignore_reason != NULL)
4246 if (loop_dump_stream)
4247 fprintf (loop_dump_stream,
4248 "Prefetch: ignoring giv at %d: %s.\n",
4249 INSN_UID (iv->insn), ignore_reason);
4250 continue;
4253 /* Determine the pointer to the basic array we are examining. It is
4254 the sum of the BIV's initial value and the GIV's add_val. */
4255 address = copy_rtx (iv->add_val);
4256 temp = copy_rtx (bl->initial_value);
4258 address = simplify_gen_binary (PLUS, Pmode, temp, address);
4259 index = remove_constant_addition (&address);
4261 d.mem_write = 0;
4262 d.mem_address = *iv->location;
4264 /* When the GIV is not always executed, we might be better off by
4265 not dirtying the cache pages. */
4266 if (PREFETCH_CONDITIONAL || iv->always_executed)
4267 note_stores (PATTERN (iv->insn), check_store, &d);
4268 else
4270 if (loop_dump_stream)
4271 fprintf (loop_dump_stream, "Prefetch: Ignoring giv at %d: %s\n",
4272 INSN_UID (iv->insn), "in conditional code.");
4273 continue;
4276 /* Attempt to find another prefetch to the same array and see if we
4277 can merge this one. */
4278 for (i = 0; i < num_prefetches; i++)
4279 if (rtx_equal_for_prefetch_p (address, info[i].base_address)
4280 && stride == info[i].stride)
4282 /* In case both access same array (same location
4283 just with small difference in constant indexes), merge
4284 the prefetches. Just do the later and the earlier will
4285 get prefetched from previous iteration.
4286 The artificial threshold should not be too small,
4287 but also not bigger than small portion of memory usually
4288 traversed by single loop. */
4289 if (index >= info[i].index
4290 && index - info[i].index < PREFETCH_EXTREME_DIFFERENCE)
4292 info[i].write |= d.mem_write;
4293 info[i].bytes_accessed += size;
4294 info[i].index = index;
4295 info[i].giv = iv;
4296 info[i].class = bl;
4297 info[num_prefetches].base_address = address;
4298 add = 0;
4299 break;
4302 if (index < info[i].index
4303 && info[i].index - index < PREFETCH_EXTREME_DIFFERENCE)
4305 info[i].write |= d.mem_write;
4306 info[i].bytes_accessed += size;
4307 add = 0;
4308 break;
4312 /* Merging failed. */
4313 if (add)
4315 info[num_prefetches].giv = iv;
4316 info[num_prefetches].class = bl;
4317 info[num_prefetches].index = index;
4318 info[num_prefetches].stride = stride;
4319 info[num_prefetches].base_address = address;
4320 info[num_prefetches].write = d.mem_write;
4321 info[num_prefetches].bytes_accessed = size;
4322 num_prefetches++;
4323 if (num_prefetches >= MAX_PREFETCHES)
4325 if (loop_dump_stream)
4326 fprintf (loop_dump_stream,
4327 "Maximal number of prefetches exceeded.\n");
4328 return;
4334 for (i = 0; i < num_prefetches; i++)
4336 int density;
4338 /* Attempt to calculate the total number of bytes fetched by all
4339 iterations of the loop. Avoid overflow. */
4340 if (LOOP_INFO (loop)->n_iterations
4341 && ((unsigned HOST_WIDE_INT) (0xffffffff / info[i].stride)
4342 >= LOOP_INFO (loop)->n_iterations))
4343 info[i].total_bytes = info[i].stride * LOOP_INFO (loop)->n_iterations;
4344 else
4345 info[i].total_bytes = 0xffffffff;
4347 density = info[i].bytes_accessed * 100 / info[i].stride;
4349 /* Prefetch might be worthwhile only when the loads/stores are dense. */
4350 if (PREFETCH_ONLY_DENSE_MEM)
4351 if (density * 256 > PREFETCH_DENSE_MEM * 100
4352 && (info[i].total_bytes / PREFETCH_BLOCK
4353 >= PREFETCH_BLOCKS_BEFORE_LOOP_MIN))
4355 info[i].prefetch_before_loop = 1;
4356 info[i].prefetch_in_loop
4357 = (info[i].total_bytes / PREFETCH_BLOCK
4358 > PREFETCH_BLOCKS_BEFORE_LOOP_MAX);
4360 else
4362 info[i].prefetch_in_loop = 0, info[i].prefetch_before_loop = 0;
4363 if (loop_dump_stream)
4364 fprintf (loop_dump_stream,
4365 "Prefetch: ignoring giv at %d: %d%% density is too low.\n",
4366 INSN_UID (info[i].giv->insn), density);
4368 else
4369 info[i].prefetch_in_loop = 1, info[i].prefetch_before_loop = 1;
4371 /* Find how many prefetch instructions we'll use within the loop. */
4372 if (info[i].prefetch_in_loop != 0)
4374 info[i].prefetch_in_loop = ((info[i].stride + PREFETCH_BLOCK - 1)
4375 / PREFETCH_BLOCK);
4376 num_real_prefetches += info[i].prefetch_in_loop;
4377 if (info[i].write)
4378 num_real_write_prefetches += info[i].prefetch_in_loop;
4382 /* Determine how many iterations ahead to prefetch within the loop, based
4383 on how many prefetches we currently expect to do within the loop. */
4384 if (num_real_prefetches != 0)
4386 if ((ahead = SIMULTANEOUS_PREFETCHES / num_real_prefetches) == 0)
4388 if (loop_dump_stream)
4389 fprintf (loop_dump_stream,
4390 "Prefetch: ignoring prefetches within loop: ahead is zero; %d < %d\n",
4391 SIMULTANEOUS_PREFETCHES, num_real_prefetches);
4392 num_real_prefetches = 0, num_real_write_prefetches = 0;
4395 /* We'll also use AHEAD to determine how many prefetch instructions to
4396 emit before a loop, so don't leave it zero. */
4397 if (ahead == 0)
4398 ahead = PREFETCH_BLOCKS_BEFORE_LOOP_MAX;
4400 for (i = 0; i < num_prefetches; i++)
4402 /* Update if we've decided not to prefetch anything within the loop. */
4403 if (num_real_prefetches == 0)
4404 info[i].prefetch_in_loop = 0;
4406 /* Find how many prefetch instructions we'll use before the loop. */
4407 if (info[i].prefetch_before_loop != 0)
4409 int n = info[i].total_bytes / PREFETCH_BLOCK;
4410 if (n > ahead)
4411 n = ahead;
4412 info[i].prefetch_before_loop = n;
4413 num_prefetches_before += n;
4414 if (info[i].write)
4415 num_write_prefetches_before += n;
4418 if (loop_dump_stream)
4420 if (info[i].prefetch_in_loop == 0
4421 && info[i].prefetch_before_loop == 0)
4422 continue;
4423 fprintf (loop_dump_stream, "Prefetch insn: %d",
4424 INSN_UID (info[i].giv->insn));
4425 fprintf (loop_dump_stream,
4426 "; in loop: %d; before: %d; %s\n",
4427 info[i].prefetch_in_loop,
4428 info[i].prefetch_before_loop,
4429 info[i].write ? "read/write" : "read only");
4430 fprintf (loop_dump_stream,
4431 " density: %d%%; bytes_accessed: %u; total_bytes: %u\n",
4432 (int) (info[i].bytes_accessed * 100 / info[i].stride),
4433 info[i].bytes_accessed, info[i].total_bytes);
4434 fprintf (loop_dump_stream, " index: " HOST_WIDE_INT_PRINT_DEC
4435 "; stride: " HOST_WIDE_INT_PRINT_DEC "; address: ",
4436 info[i].index, info[i].stride);
4437 print_rtl (loop_dump_stream, info[i].base_address);
4438 fprintf (loop_dump_stream, "\n");
4442 if (num_real_prefetches + num_prefetches_before > 0)
4444 /* Record that this loop uses prefetch instructions. */
4445 LOOP_INFO (loop)->has_prefetch = 1;
4447 if (loop_dump_stream)
4449 fprintf (loop_dump_stream, "Real prefetches needed within loop: %d (write: %d)\n",
4450 num_real_prefetches, num_real_write_prefetches);
4451 fprintf (loop_dump_stream, "Real prefetches needed before loop: %d (write: %d)\n",
4452 num_prefetches_before, num_write_prefetches_before);
4456 for (i = 0; i < num_prefetches; i++)
4458 int y;
4460 for (y = 0; y < info[i].prefetch_in_loop; y++)
4462 rtx loc = copy_rtx (*info[i].giv->location);
4463 rtx insn;
4464 int bytes_ahead = PREFETCH_BLOCK * (ahead + y);
4465 rtx before_insn = info[i].giv->insn;
4466 rtx prev_insn = PREV_INSN (info[i].giv->insn);
4467 rtx seq;
4469 /* We can save some effort by offsetting the address on
4470 architectures with offsettable memory references. */
4471 if (offsettable_address_p (0, VOIDmode, loc))
4472 loc = plus_constant (loc, bytes_ahead);
4473 else
4475 rtx reg = gen_reg_rtx (Pmode);
4476 loop_iv_add_mult_emit_before (loop, loc, const1_rtx,
4477 GEN_INT (bytes_ahead), reg,
4478 0, before_insn);
4479 loc = reg;
4482 start_sequence ();
4483 /* Make sure the address operand is valid for prefetch. */
4484 if (! (*insn_data[(int)CODE_FOR_prefetch].operand[0].predicate)
4485 (loc, insn_data[(int)CODE_FOR_prefetch].operand[0].mode))
4486 loc = force_reg (Pmode, loc);
4487 emit_insn (gen_prefetch (loc, GEN_INT (info[i].write),
4488 GEN_INT (3)));
4489 seq = get_insns ();
4490 end_sequence ();
4491 emit_insn_before (seq, before_insn);
4493 /* Check all insns emitted and record the new GIV
4494 information. */
4495 insn = NEXT_INSN (prev_insn);
4496 while (insn != before_insn)
4498 insn = check_insn_for_givs (loop, insn,
4499 info[i].giv->always_executed,
4500 info[i].giv->maybe_multiple);
4501 insn = NEXT_INSN (insn);
4505 if (PREFETCH_BEFORE_LOOP)
4507 /* Emit insns before the loop to fetch the first cache lines or,
4508 if we're not prefetching within the loop, everything we expect
4509 to need. */
4510 for (y = 0; y < info[i].prefetch_before_loop; y++)
4512 rtx reg = gen_reg_rtx (Pmode);
4513 rtx loop_start = loop->start;
4514 rtx init_val = info[i].class->initial_value;
4515 rtx add_val = simplify_gen_binary (PLUS, Pmode,
4516 info[i].giv->add_val,
4517 GEN_INT (y * PREFETCH_BLOCK));
4519 /* Functions called by LOOP_IV_ADD_EMIT_BEFORE expect a
4520 non-constant INIT_VAL to have the same mode as REG, which
4521 in this case we know to be Pmode. */
4522 if (GET_MODE (init_val) != Pmode && !CONSTANT_P (init_val))
4524 rtx seq;
4526 start_sequence ();
4527 init_val = convert_to_mode (Pmode, init_val, 0);
4528 seq = get_insns ();
4529 end_sequence ();
4530 loop_insn_emit_before (loop, 0, loop_start, seq);
4532 loop_iv_add_mult_emit_before (loop, init_val,
4533 info[i].giv->mult_val,
4534 add_val, reg, 0, loop_start);
4535 emit_insn_before (gen_prefetch (reg, GEN_INT (info[i].write),
4536 GEN_INT (3)),
4537 loop_start);
4542 return;
4545 /* Communication with routines called via `note_stores'. */
4547 static rtx note_insn;
4549 /* Dummy register to have nonzero DEST_REG for DEST_ADDR type givs. */
4551 static rtx addr_placeholder;
4553 /* ??? Unfinished optimizations, and possible future optimizations,
4554 for the strength reduction code. */
4556 /* ??? The interaction of biv elimination, and recognition of 'constant'
4557 bivs, may cause problems. */
4559 /* ??? Add heuristics so that DEST_ADDR strength reduction does not cause
4560 performance problems.
4562 Perhaps don't eliminate things that can be combined with an addressing
4563 mode. Find all givs that have the same biv, mult_val, and add_val;
4564 then for each giv, check to see if its only use dies in a following
4565 memory address. If so, generate a new memory address and check to see
4566 if it is valid. If it is valid, then store the modified memory address,
4567 otherwise, mark the giv as not done so that it will get its own iv. */
4569 /* ??? Could try to optimize branches when it is known that a biv is always
4570 positive. */
4572 /* ??? When replace a biv in a compare insn, we should replace with closest
4573 giv so that an optimized branch can still be recognized by the combiner,
4574 e.g. the VAX acb insn. */
4576 /* ??? Many of the checks involving uid_luid could be simplified if regscan
4577 was rerun in loop_optimize whenever a register was added or moved.
4578 Also, some of the optimizations could be a little less conservative. */
4580 /* Searches the insns between INSN and LOOP->END. Returns 1 if there
4581 is a backward branch in that range that branches to somewhere between
4582 LOOP->START and INSN. Returns 0 otherwise. */
4584 /* ??? This is quadratic algorithm. Could be rewritten to be linear.
4585 In practice, this is not a problem, because this function is seldom called,
4586 and uses a negligible amount of CPU time on average. */
4588 static int
4589 back_branch_in_range_p (const struct loop *loop, rtx insn)
4591 rtx p, q, target_insn;
4592 rtx loop_start = loop->start;
4593 rtx loop_end = loop->end;
4594 rtx orig_loop_end = loop->end;
4596 /* Stop before we get to the backward branch at the end of the loop. */
4597 loop_end = prev_nonnote_insn (loop_end);
4598 if (BARRIER_P (loop_end))
4599 loop_end = PREV_INSN (loop_end);
4601 /* Check in case insn has been deleted, search forward for first non
4602 deleted insn following it. */
4603 while (INSN_DELETED_P (insn))
4604 insn = NEXT_INSN (insn);
4606 /* Check for the case where insn is the last insn in the loop. Deal
4607 with the case where INSN was a deleted loop test insn, in which case
4608 it will now be the NOTE_LOOP_END. */
4609 if (insn == loop_end || insn == orig_loop_end)
4610 return 0;
4612 for (p = NEXT_INSN (insn); p != loop_end; p = NEXT_INSN (p))
4614 if (JUMP_P (p))
4616 target_insn = JUMP_LABEL (p);
4618 /* Search from loop_start to insn, to see if one of them is
4619 the target_insn. We can't use INSN_LUID comparisons here,
4620 since insn may not have an LUID entry. */
4621 for (q = loop_start; q != insn; q = NEXT_INSN (q))
4622 if (q == target_insn)
4623 return 1;
4627 return 0;
4630 /* Scan the loop body and call FNCALL for each insn. In the addition to the
4631 LOOP and INSN parameters pass MAYBE_MULTIPLE and NOT_EVERY_ITERATION to the
4632 callback.
4634 NOT_EVERY_ITERATION is 1 if current insn is not known to be executed at
4635 least once for every loop iteration except for the last one.
4637 MAYBE_MULTIPLE is 1 if current insn may be executed more than once for every
4638 loop iteration.
4640 typedef rtx (*loop_insn_callback) (struct loop *, rtx, int, int);
4641 static void
4642 for_each_insn_in_loop (struct loop *loop, loop_insn_callback fncall)
4644 int not_every_iteration = 0;
4645 int maybe_multiple = 0;
4646 int past_loop_latch = 0;
4647 rtx p;
4649 /* If loop_scan_start points to the loop exit test, we have to be wary of
4650 subversive use of gotos inside expression statements. */
4651 if (prev_nonnote_insn (loop->scan_start) != prev_nonnote_insn (loop->start))
4652 maybe_multiple = back_branch_in_range_p (loop, loop->scan_start);
4654 /* Scan through loop and update NOT_EVERY_ITERATION and MAYBE_MULTIPLE. */
4655 for (p = next_insn_in_loop (loop, loop->scan_start);
4656 p != NULL_RTX;
4657 p = next_insn_in_loop (loop, p))
4659 p = fncall (loop, p, not_every_iteration, maybe_multiple);
4661 /* Past CODE_LABEL, we get to insns that may be executed multiple
4662 times. The only way we can be sure that they can't is if every
4663 jump insn between here and the end of the loop either
4664 returns, exits the loop, is a jump to a location that is still
4665 behind the label, or is a jump to the loop start. */
4667 if (LABEL_P (p))
4669 rtx insn = p;
4671 maybe_multiple = 0;
4673 while (1)
4675 insn = NEXT_INSN (insn);
4676 if (insn == loop->scan_start)
4677 break;
4678 if (insn == loop->end)
4680 if (loop->top != 0)
4681 insn = loop->top;
4682 else
4683 break;
4684 if (insn == loop->scan_start)
4685 break;
4688 if (JUMP_P (insn)
4689 && GET_CODE (PATTERN (insn)) != RETURN
4690 && (!any_condjump_p (insn)
4691 || (JUMP_LABEL (insn) != 0
4692 && JUMP_LABEL (insn) != loop->scan_start
4693 && !loop_insn_first_p (p, JUMP_LABEL (insn)))))
4695 maybe_multiple = 1;
4696 break;
4701 /* Past a jump, we get to insns for which we can't count
4702 on whether they will be executed during each iteration. */
4703 /* This code appears twice in strength_reduce. There is also similar
4704 code in scan_loop. */
4705 if (JUMP_P (p)
4706 /* If we enter the loop in the middle, and scan around to the
4707 beginning, don't set not_every_iteration for that.
4708 This can be any kind of jump, since we want to know if insns
4709 will be executed if the loop is executed. */
4710 && !(JUMP_LABEL (p) == loop->top
4711 && ((NEXT_INSN (NEXT_INSN (p)) == loop->end
4712 && any_uncondjump_p (p))
4713 || (NEXT_INSN (p) == loop->end && any_condjump_p (p)))))
4715 rtx label = 0;
4717 /* If this is a jump outside the loop, then it also doesn't
4718 matter. Check to see if the target of this branch is on the
4719 loop->exits_labels list. */
4721 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
4722 if (XEXP (label, 0) == JUMP_LABEL (p))
4723 break;
4725 if (!label)
4726 not_every_iteration = 1;
4729 /* Note if we pass a loop latch. If we do, then we can not clear
4730 NOT_EVERY_ITERATION below when we pass the last CODE_LABEL in
4731 a loop since a jump before the last CODE_LABEL may have started
4732 a new loop iteration.
4734 Note that LOOP_TOP is only set for rotated loops and we need
4735 this check for all loops, so compare against the CODE_LABEL
4736 which immediately follows LOOP_START. */
4737 if (JUMP_P (p)
4738 && JUMP_LABEL (p) == NEXT_INSN (loop->start))
4739 past_loop_latch = 1;
4741 /* Unlike in the code motion pass where MAYBE_NEVER indicates that
4742 an insn may never be executed, NOT_EVERY_ITERATION indicates whether
4743 or not an insn is known to be executed each iteration of the
4744 loop, whether or not any iterations are known to occur.
4746 Therefore, if we have just passed a label and have no more labels
4747 between here and the test insn of the loop, and we have not passed
4748 a jump to the top of the loop, then we know these insns will be
4749 executed each iteration. */
4751 if (not_every_iteration
4752 && !past_loop_latch
4753 && LABEL_P (p)
4754 && no_labels_between_p (p, loop->end))
4755 not_every_iteration = 0;
4759 static void
4760 loop_bivs_find (struct loop *loop)
4762 struct loop_regs *regs = LOOP_REGS (loop);
4763 struct loop_ivs *ivs = LOOP_IVS (loop);
4764 /* Temporary list pointers for traversing ivs->list. */
4765 struct iv_class *bl, **backbl;
4767 ivs->list = 0;
4769 for_each_insn_in_loop (loop, check_insn_for_bivs);
4771 /* Scan ivs->list to remove all regs that proved not to be bivs.
4772 Make a sanity check against regs->n_times_set. */
4773 for (backbl = &ivs->list, bl = *backbl; bl; bl = bl->next)
4775 if (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4776 /* Above happens if register modified by subreg, etc. */
4777 /* Make sure it is not recognized as a basic induction var: */
4778 || regs->array[bl->regno].n_times_set != bl->biv_count
4779 /* If never incremented, it is invariant that we decided not to
4780 move. So leave it alone. */
4781 || ! bl->incremented)
4783 if (loop_dump_stream)
4784 fprintf (loop_dump_stream, "Biv %d: discarded, %s\n",
4785 bl->regno,
4786 (REG_IV_TYPE (ivs, bl->regno) != BASIC_INDUCT
4787 ? "not induction variable"
4788 : (! bl->incremented ? "never incremented"
4789 : "count error")));
4791 REG_IV_TYPE (ivs, bl->regno) = NOT_BASIC_INDUCT;
4792 *backbl = bl->next;
4794 else
4796 backbl = &bl->next;
4798 if (loop_dump_stream)
4799 fprintf (loop_dump_stream, "Biv %d: verified\n", bl->regno);
4805 /* Determine how BIVS are initialized by looking through pre-header
4806 extended basic block. */
4807 static void
4808 loop_bivs_init_find (struct loop *loop)
4810 struct loop_ivs *ivs = LOOP_IVS (loop);
4811 /* Temporary list pointers for traversing ivs->list. */
4812 struct iv_class *bl;
4813 int call_seen;
4814 rtx p;
4816 /* Find initial value for each biv by searching backwards from loop_start,
4817 halting at first label. Also record any test condition. */
4819 call_seen = 0;
4820 for (p = loop->start; p && !LABEL_P (p); p = PREV_INSN (p))
4822 rtx test;
4824 note_insn = p;
4826 if (CALL_P (p))
4827 call_seen = 1;
4829 if (INSN_P (p))
4830 note_stores (PATTERN (p), record_initial, ivs);
4832 /* Record any test of a biv that branches around the loop if no store
4833 between it and the start of loop. We only care about tests with
4834 constants and registers and only certain of those. */
4835 if (JUMP_P (p)
4836 && JUMP_LABEL (p) != 0
4837 && next_real_insn (JUMP_LABEL (p)) == next_real_insn (loop->end)
4838 && (test = get_condition_for_loop (loop, p)) != 0
4839 && REG_P (XEXP (test, 0))
4840 && REGNO (XEXP (test, 0)) < max_reg_before_loop
4841 && (bl = REG_IV_CLASS (ivs, REGNO (XEXP (test, 0)))) != 0
4842 && valid_initial_value_p (XEXP (test, 1), p, call_seen, loop->start)
4843 && bl->init_insn == 0)
4845 /* If an NE test, we have an initial value! */
4846 if (GET_CODE (test) == NE)
4848 bl->init_insn = p;
4849 bl->init_set = gen_rtx_SET (VOIDmode,
4850 XEXP (test, 0), XEXP (test, 1));
4852 else
4853 bl->initial_test = test;
4859 /* Look at the each biv and see if we can say anything better about its
4860 initial value from any initializing insns set up above. (This is done
4861 in two passes to avoid missing SETs in a PARALLEL.) */
4862 static void
4863 loop_bivs_check (struct loop *loop)
4865 struct loop_ivs *ivs = LOOP_IVS (loop);
4866 /* Temporary list pointers for traversing ivs->list. */
4867 struct iv_class *bl;
4868 struct iv_class **backbl;
4870 for (backbl = &ivs->list; (bl = *backbl); backbl = &bl->next)
4872 rtx src;
4873 rtx note;
4875 if (! bl->init_insn)
4876 continue;
4878 /* IF INIT_INSN has a REG_EQUAL or REG_EQUIV note and the value
4879 is a constant, use the value of that. */
4880 if (((note = find_reg_note (bl->init_insn, REG_EQUAL, 0)) != NULL
4881 && CONSTANT_P (XEXP (note, 0)))
4882 || ((note = find_reg_note (bl->init_insn, REG_EQUIV, 0)) != NULL
4883 && CONSTANT_P (XEXP (note, 0))))
4884 src = XEXP (note, 0);
4885 else
4886 src = SET_SRC (bl->init_set);
4888 if (loop_dump_stream)
4889 fprintf (loop_dump_stream,
4890 "Biv %d: initialized at insn %d: initial value ",
4891 bl->regno, INSN_UID (bl->init_insn));
4893 if ((GET_MODE (src) == GET_MODE (regno_reg_rtx[bl->regno])
4894 || GET_MODE (src) == VOIDmode)
4895 && valid_initial_value_p (src, bl->init_insn,
4896 LOOP_INFO (loop)->pre_header_has_call,
4897 loop->start))
4899 bl->initial_value = src;
4901 if (loop_dump_stream)
4903 print_simple_rtl (loop_dump_stream, src);
4904 fputc ('\n', loop_dump_stream);
4907 /* If we can't make it a giv,
4908 let biv keep initial value of "itself". */
4909 else if (loop_dump_stream)
4910 fprintf (loop_dump_stream, "is complex\n");
4915 /* Search the loop for general induction variables. */
4917 static void
4918 loop_givs_find (struct loop* loop)
4920 for_each_insn_in_loop (loop, check_insn_for_givs);
4924 /* For each giv for which we still don't know whether or not it is
4925 replaceable, check to see if it is replaceable because its final value
4926 can be calculated. */
4928 static void
4929 loop_givs_check (struct loop *loop)
4931 struct loop_ivs *ivs = LOOP_IVS (loop);
4932 struct iv_class *bl;
4934 for (bl = ivs->list; bl; bl = bl->next)
4936 struct induction *v;
4938 for (v = bl->giv; v; v = v->next_iv)
4939 if (! v->replaceable && ! v->not_replaceable)
4940 check_final_value (loop, v);
4944 /* Try to generate the simplest rtx for the expression
4945 (PLUS (MULT mult1 mult2) add1). This is used to calculate the initial
4946 value of giv's. */
4948 static rtx
4949 fold_rtx_mult_add (rtx mult1, rtx mult2, rtx add1, enum machine_mode mode)
4951 rtx temp, mult_res;
4952 rtx result;
4954 /* The modes must all be the same. This should always be true. For now,
4955 check to make sure. */
4956 if ((GET_MODE (mult1) != mode && GET_MODE (mult1) != VOIDmode)
4957 || (GET_MODE (mult2) != mode && GET_MODE (mult2) != VOIDmode)
4958 || (GET_MODE (add1) != mode && GET_MODE (add1) != VOIDmode))
4959 abort ();
4961 /* Ensure that if at least one of mult1/mult2 are constant, then mult2
4962 will be a constant. */
4963 if (GET_CODE (mult1) == CONST_INT)
4965 temp = mult2;
4966 mult2 = mult1;
4967 mult1 = temp;
4970 mult_res = simplify_binary_operation (MULT, mode, mult1, mult2);
4971 if (! mult_res)
4972 mult_res = gen_rtx_MULT (mode, mult1, mult2);
4974 /* Again, put the constant second. */
4975 if (GET_CODE (add1) == CONST_INT)
4977 temp = add1;
4978 add1 = mult_res;
4979 mult_res = temp;
4982 result = simplify_binary_operation (PLUS, mode, add1, mult_res);
4983 if (! result)
4984 result = gen_rtx_PLUS (mode, add1, mult_res);
4986 return result;
4989 /* Searches the list of induction struct's for the biv BL, to try to calculate
4990 the total increment value for one iteration of the loop as a constant.
4992 Returns the increment value as an rtx, simplified as much as possible,
4993 if it can be calculated. Otherwise, returns 0. */
4995 static rtx
4996 biv_total_increment (const struct iv_class *bl)
4998 struct induction *v;
4999 rtx result;
5001 /* For increment, must check every instruction that sets it. Each
5002 instruction must be executed only once each time through the loop.
5003 To verify this, we check that the insn is always executed, and that
5004 there are no backward branches after the insn that branch to before it.
5005 Also, the insn must have a mult_val of one (to make sure it really is
5006 an increment). */
5008 result = const0_rtx;
5009 for (v = bl->biv; v; v = v->next_iv)
5011 if (v->always_computable && v->mult_val == const1_rtx
5012 && ! v->maybe_multiple
5013 && SCALAR_INT_MODE_P (v->mode))
5015 /* If we have already counted it, skip it. */
5016 if (v->same)
5017 continue;
5019 result = fold_rtx_mult_add (result, const1_rtx, v->add_val, v->mode);
5021 else
5022 return 0;
5025 return result;
5028 /* Try to prove that the register is dead after the loop exits. Trace every
5029 loop exit looking for an insn that will always be executed, which sets
5030 the register to some value, and appears before the first use of the register
5031 is found. If successful, then return 1, otherwise return 0. */
5033 /* ?? Could be made more intelligent in the handling of jumps, so that
5034 it can search past if statements and other similar structures. */
5036 static int
5037 reg_dead_after_loop (const struct loop *loop, rtx reg)
5039 rtx insn, label;
5040 int jump_count = 0;
5041 int label_count = 0;
5043 /* In addition to checking all exits of this loop, we must also check
5044 all exits of inner nested loops that would exit this loop. We don't
5045 have any way to identify those, so we just give up if there are any
5046 such inner loop exits. */
5048 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
5049 label_count++;
5051 if (label_count != loop->exit_count)
5052 return 0;
5054 /* HACK: Must also search the loop fall through exit, create a label_ref
5055 here which points to the loop->end, and append the loop_number_exit_labels
5056 list to it. */
5057 label = gen_rtx_LABEL_REF (VOIDmode, loop->end);
5058 LABEL_NEXTREF (label) = loop->exit_labels;
5060 for (; label; label = LABEL_NEXTREF (label))
5062 /* Succeed if find an insn which sets the biv or if reach end of
5063 function. Fail if find an insn that uses the biv, or if come to
5064 a conditional jump. */
5066 insn = NEXT_INSN (XEXP (label, 0));
5067 while (insn)
5069 if (INSN_P (insn))
5071 rtx set, note;
5073 if (reg_referenced_p (reg, PATTERN (insn)))
5074 return 0;
5076 note = find_reg_equal_equiv_note (insn);
5077 if (note && reg_overlap_mentioned_p (reg, XEXP (note, 0)))
5078 return 0;
5080 set = single_set (insn);
5081 if (set && rtx_equal_p (SET_DEST (set), reg))
5082 break;
5084 if (JUMP_P (insn))
5086 if (GET_CODE (PATTERN (insn)) == RETURN)
5087 break;
5088 else if (!any_uncondjump_p (insn)
5089 /* Prevent infinite loop following infinite loops. */
5090 || jump_count++ > 20)
5091 return 0;
5092 else
5093 insn = JUMP_LABEL (insn);
5097 insn = NEXT_INSN (insn);
5101 /* Success, the register is dead on all loop exits. */
5102 return 1;
5105 /* Try to calculate the final value of the biv, the value it will have at
5106 the end of the loop. If we can do it, return that value. */
5108 static rtx
5109 final_biv_value (const struct loop *loop, struct iv_class *bl)
5111 unsigned HOST_WIDE_INT n_iterations = LOOP_INFO (loop)->n_iterations;
5112 rtx increment, tem;
5114 /* ??? This only works for MODE_INT biv's. Reject all others for now. */
5116 if (GET_MODE_CLASS (bl->biv->mode) != MODE_INT)
5117 return 0;
5119 /* The final value for reversed bivs must be calculated differently than
5120 for ordinary bivs. In this case, there is already an insn after the
5121 loop which sets this biv's final value (if necessary), and there are
5122 no other loop exits, so we can return any value. */
5123 if (bl->reversed)
5125 if (loop_dump_stream)
5126 fprintf (loop_dump_stream,
5127 "Final biv value for %d, reversed biv.\n", bl->regno);
5129 return const0_rtx;
5132 /* Try to calculate the final value as initial value + (number of iterations
5133 * increment). For this to work, increment must be invariant, the only
5134 exit from the loop must be the fall through at the bottom (otherwise
5135 it may not have its final value when the loop exits), and the initial
5136 value of the biv must be invariant. */
5138 if (n_iterations != 0
5139 && ! loop->exit_count
5140 && loop_invariant_p (loop, bl->initial_value))
5142 increment = biv_total_increment (bl);
5144 if (increment && loop_invariant_p (loop, increment))
5146 /* Can calculate the loop exit value, emit insns after loop
5147 end to calculate this value into a temporary register in
5148 case it is needed later. */
5150 tem = gen_reg_rtx (bl->biv->mode);
5151 record_base_value (REGNO (tem), bl->biv->add_val, 0);
5152 loop_iv_add_mult_sink (loop, increment, GEN_INT (n_iterations),
5153 bl->initial_value, tem);
5155 if (loop_dump_stream)
5156 fprintf (loop_dump_stream,
5157 "Final biv value for %d, calculated.\n", bl->regno);
5159 return tem;
5163 /* Check to see if the biv is dead at all loop exits. */
5164 if (reg_dead_after_loop (loop, bl->biv->src_reg))
5166 if (loop_dump_stream)
5167 fprintf (loop_dump_stream,
5168 "Final biv value for %d, biv dead after loop exit.\n",
5169 bl->regno);
5171 return const0_rtx;
5174 return 0;
5177 /* Return nonzero if it is possible to eliminate the biv BL provided
5178 all givs are reduced. This is possible if either the reg is not
5179 used outside the loop, or we can compute what its final value will
5180 be. */
5182 static int
5183 loop_biv_eliminable_p (struct loop *loop, struct iv_class *bl,
5184 int threshold, int insn_count)
5186 /* For architectures with a decrement_and_branch_until_zero insn,
5187 don't do this if we put a REG_NONNEG note on the endtest for this
5188 biv. */
5190 #ifdef HAVE_decrement_and_branch_until_zero
5191 if (bl->nonneg)
5193 if (loop_dump_stream)
5194 fprintf (loop_dump_stream,
5195 "Cannot eliminate nonneg biv %d.\n", bl->regno);
5196 return 0;
5198 #endif
5200 /* Check that biv is used outside loop or if it has a final value.
5201 Compare against bl->init_insn rather than loop->start. We aren't
5202 concerned with any uses of the biv between init_insn and
5203 loop->start since these won't be affected by the value of the biv
5204 elsewhere in the function, so long as init_insn doesn't use the
5205 biv itself. */
5207 if ((REGNO_LAST_LUID (bl->regno) < INSN_LUID (loop->end)
5208 && bl->init_insn
5209 && INSN_UID (bl->init_insn) < max_uid_for_loop
5210 && REGNO_FIRST_LUID (bl->regno) >= INSN_LUID (bl->init_insn)
5211 && ! reg_mentioned_p (bl->biv->dest_reg, SET_SRC (bl->init_set)))
5212 || (bl->final_value = final_biv_value (loop, bl)))
5213 return maybe_eliminate_biv (loop, bl, 0, threshold, insn_count);
5215 if (loop_dump_stream)
5217 fprintf (loop_dump_stream,
5218 "Cannot eliminate biv %d.\n",
5219 bl->regno);
5220 fprintf (loop_dump_stream,
5221 "First use: insn %d, last use: insn %d.\n",
5222 REGNO_FIRST_UID (bl->regno),
5223 REGNO_LAST_UID (bl->regno));
5225 return 0;
5229 /* Reduce each giv of BL that we have decided to reduce. */
5231 static void
5232 loop_givs_reduce (struct loop *loop, struct iv_class *bl)
5234 struct induction *v;
5236 for (v = bl->giv; v; v = v->next_iv)
5238 struct induction *tv;
5239 if (! v->ignore && v->same == 0)
5241 int auto_inc_opt = 0;
5243 /* If the code for derived givs immediately below has already
5244 allocated a new_reg, we must keep it. */
5245 if (! v->new_reg)
5246 v->new_reg = gen_reg_rtx (v->mode);
5248 #ifdef AUTO_INC_DEC
5249 /* If the target has auto-increment addressing modes, and
5250 this is an address giv, then try to put the increment
5251 immediately after its use, so that flow can create an
5252 auto-increment addressing mode. */
5253 /* Don't do this for loops entered at the bottom, to avoid
5254 this invalid transformation:
5255 jmp L; -> jmp L;
5256 TOP: TOP:
5257 use giv use giv
5258 L: inc giv
5259 inc biv L:
5260 test biv test giv
5261 cbr TOP cbr TOP
5263 if (v->giv_type == DEST_ADDR && bl->biv_count == 1
5264 && bl->biv->always_executed && ! bl->biv->maybe_multiple
5265 /* We don't handle reversed biv's because bl->biv->insn
5266 does not have a valid INSN_LUID. */
5267 && ! bl->reversed
5268 && v->always_executed && ! v->maybe_multiple
5269 && INSN_UID (v->insn) < max_uid_for_loop
5270 && !loop->top)
5272 /* If other giv's have been combined with this one, then
5273 this will work only if all uses of the other giv's occur
5274 before this giv's insn. This is difficult to check.
5276 We simplify this by looking for the common case where
5277 there is one DEST_REG giv, and this giv's insn is the
5278 last use of the dest_reg of that DEST_REG giv. If the
5279 increment occurs after the address giv, then we can
5280 perform the optimization. (Otherwise, the increment
5281 would have to go before other_giv, and we would not be
5282 able to combine it with the address giv to get an
5283 auto-inc address.) */
5284 if (v->combined_with)
5286 struct induction *other_giv = 0;
5288 for (tv = bl->giv; tv; tv = tv->next_iv)
5289 if (tv->same == v)
5291 if (other_giv)
5292 break;
5293 else
5294 other_giv = tv;
5296 if (! tv && other_giv
5297 && REGNO (other_giv->dest_reg) < max_reg_before_loop
5298 && (REGNO_LAST_UID (REGNO (other_giv->dest_reg))
5299 == INSN_UID (v->insn))
5300 && INSN_LUID (v->insn) < INSN_LUID (bl->biv->insn))
5301 auto_inc_opt = 1;
5303 /* Check for case where increment is before the address
5304 giv. Do this test in "loop order". */
5305 else if ((INSN_LUID (v->insn) > INSN_LUID (bl->biv->insn)
5306 && (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
5307 || (INSN_LUID (bl->biv->insn)
5308 > INSN_LUID (loop->scan_start))))
5309 || (INSN_LUID (v->insn) < INSN_LUID (loop->scan_start)
5310 && (INSN_LUID (loop->scan_start)
5311 < INSN_LUID (bl->biv->insn))))
5312 auto_inc_opt = -1;
5313 else
5314 auto_inc_opt = 1;
5316 #ifdef HAVE_cc0
5318 rtx prev;
5320 /* We can't put an insn immediately after one setting
5321 cc0, or immediately before one using cc0. */
5322 if ((auto_inc_opt == 1 && sets_cc0_p (PATTERN (v->insn)))
5323 || (auto_inc_opt == -1
5324 && (prev = prev_nonnote_insn (v->insn)) != 0
5325 && INSN_P (prev)
5326 && sets_cc0_p (PATTERN (prev))))
5327 auto_inc_opt = 0;
5329 #endif
5331 if (auto_inc_opt)
5332 v->auto_inc_opt = 1;
5334 #endif
5336 /* For each place where the biv is incremented, add an insn
5337 to increment the new, reduced reg for the giv. */
5338 for (tv = bl->biv; tv; tv = tv->next_iv)
5340 rtx insert_before;
5342 /* Skip if location is the same as a previous one. */
5343 if (tv->same)
5344 continue;
5345 if (! auto_inc_opt)
5346 insert_before = NEXT_INSN (tv->insn);
5347 else if (auto_inc_opt == 1)
5348 insert_before = NEXT_INSN (v->insn);
5349 else
5350 insert_before = v->insn;
5352 if (tv->mult_val == const1_rtx)
5353 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
5354 v->new_reg, v->new_reg,
5355 0, insert_before);
5356 else /* tv->mult_val == const0_rtx */
5357 /* A multiply is acceptable here
5358 since this is presumed to be seldom executed. */
5359 loop_iv_add_mult_emit_before (loop, tv->add_val, v->mult_val,
5360 v->add_val, v->new_reg,
5361 0, insert_before);
5364 /* Add code at loop start to initialize giv's reduced reg. */
5366 loop_iv_add_mult_hoist (loop,
5367 extend_value_for_giv (v, bl->initial_value),
5368 v->mult_val, v->add_val, v->new_reg);
5374 /* Check for givs whose first use is their definition and whose
5375 last use is the definition of another giv. If so, it is likely
5376 dead and should not be used to derive another giv nor to
5377 eliminate a biv. */
5379 static void
5380 loop_givs_dead_check (struct loop *loop ATTRIBUTE_UNUSED, struct iv_class *bl)
5382 struct induction *v;
5384 for (v = bl->giv; v; v = v->next_iv)
5386 if (v->ignore
5387 || (v->same && v->same->ignore))
5388 continue;
5390 if (v->giv_type == DEST_REG
5391 && REGNO_FIRST_UID (REGNO (v->dest_reg)) == INSN_UID (v->insn))
5393 struct induction *v1;
5395 for (v1 = bl->giv; v1; v1 = v1->next_iv)
5396 if (REGNO_LAST_UID (REGNO (v->dest_reg)) == INSN_UID (v1->insn))
5397 v->maybe_dead = 1;
5403 static void
5404 loop_givs_rescan (struct loop *loop, struct iv_class *bl, rtx *reg_map)
5406 struct induction *v;
5408 for (v = bl->giv; v; v = v->next_iv)
5410 if (v->same && v->same->ignore)
5411 v->ignore = 1;
5413 if (v->ignore)
5414 continue;
5416 /* Update expression if this was combined, in case other giv was
5417 replaced. */
5418 if (v->same)
5419 v->new_reg = replace_rtx (v->new_reg,
5420 v->same->dest_reg, v->same->new_reg);
5422 /* See if this register is known to be a pointer to something. If
5423 so, see if we can find the alignment. First see if there is a
5424 destination register that is a pointer. If so, this shares the
5425 alignment too. Next see if we can deduce anything from the
5426 computational information. If not, and this is a DEST_ADDR
5427 giv, at least we know that it's a pointer, though we don't know
5428 the alignment. */
5429 if (REG_P (v->new_reg)
5430 && v->giv_type == DEST_REG
5431 && REG_POINTER (v->dest_reg))
5432 mark_reg_pointer (v->new_reg,
5433 REGNO_POINTER_ALIGN (REGNO (v->dest_reg)));
5434 else if (REG_P (v->new_reg)
5435 && REG_POINTER (v->src_reg))
5437 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->src_reg));
5439 if (align == 0
5440 || GET_CODE (v->add_val) != CONST_INT
5441 || INTVAL (v->add_val) % (align / BITS_PER_UNIT) != 0)
5442 align = 0;
5444 mark_reg_pointer (v->new_reg, align);
5446 else if (REG_P (v->new_reg)
5447 && REG_P (v->add_val)
5448 && REG_POINTER (v->add_val))
5450 unsigned int align = REGNO_POINTER_ALIGN (REGNO (v->add_val));
5452 if (align == 0 || GET_CODE (v->mult_val) != CONST_INT
5453 || INTVAL (v->mult_val) % (align / BITS_PER_UNIT) != 0)
5454 align = 0;
5456 mark_reg_pointer (v->new_reg, align);
5458 else if (REG_P (v->new_reg) && v->giv_type == DEST_ADDR)
5459 mark_reg_pointer (v->new_reg, 0);
5461 if (v->giv_type == DEST_ADDR)
5462 /* Store reduced reg as the address in the memref where we found
5463 this giv. */
5464 validate_change (v->insn, v->location, v->new_reg, 0);
5465 else if (v->replaceable)
5467 reg_map[REGNO (v->dest_reg)] = v->new_reg;
5469 else
5471 rtx original_insn = v->insn;
5472 rtx note;
5474 /* Not replaceable; emit an insn to set the original giv reg from
5475 the reduced giv, same as above. */
5476 v->insn = loop_insn_emit_after (loop, 0, original_insn,
5477 gen_move_insn (v->dest_reg,
5478 v->new_reg));
5480 /* The original insn may have a REG_EQUAL note. This note is
5481 now incorrect and may result in invalid substitutions later.
5482 The original insn is dead, but may be part of a libcall
5483 sequence, which doesn't seem worth the bother of handling. */
5484 note = find_reg_note (original_insn, REG_EQUAL, NULL_RTX);
5485 if (note)
5486 remove_note (original_insn, note);
5489 /* When a loop is reversed, givs which depend on the reversed
5490 biv, and which are live outside the loop, must be set to their
5491 correct final value. This insn is only needed if the giv is
5492 not replaceable. The correct final value is the same as the
5493 value that the giv starts the reversed loop with. */
5494 if (bl->reversed && ! v->replaceable)
5495 loop_iv_add_mult_sink (loop,
5496 extend_value_for_giv (v, bl->initial_value),
5497 v->mult_val, v->add_val, v->dest_reg);
5498 else if (v->final_value)
5499 loop_insn_sink_or_swim (loop,
5500 gen_load_of_final_value (v->dest_reg,
5501 v->final_value));
5503 if (loop_dump_stream)
5505 fprintf (loop_dump_stream, "giv at %d reduced to ",
5506 INSN_UID (v->insn));
5507 print_simple_rtl (loop_dump_stream, v->new_reg);
5508 fprintf (loop_dump_stream, "\n");
5514 static int
5515 loop_giv_reduce_benefit (struct loop *loop ATTRIBUTE_UNUSED,
5516 struct iv_class *bl, struct induction *v,
5517 rtx test_reg)
5519 int add_cost;
5520 int benefit;
5522 benefit = v->benefit;
5523 PUT_MODE (test_reg, v->mode);
5524 add_cost = iv_add_mult_cost (bl->biv->add_val, v->mult_val,
5525 test_reg, test_reg);
5527 /* Reduce benefit if not replaceable, since we will insert a
5528 move-insn to replace the insn that calculates this giv. Don't do
5529 this unless the giv is a user variable, since it will often be
5530 marked non-replaceable because of the duplication of the exit
5531 code outside the loop. In such a case, the copies we insert are
5532 dead and will be deleted. So they don't have a cost. Similar
5533 situations exist. */
5534 /* ??? The new final_[bg]iv_value code does a much better job of
5535 finding replaceable giv's, and hence this code may no longer be
5536 necessary. */
5537 if (! v->replaceable && ! bl->eliminable
5538 && REG_USERVAR_P (v->dest_reg))
5539 benefit -= copy_cost;
5541 /* Decrease the benefit to count the add-insns that we will insert
5542 to increment the reduced reg for the giv. ??? This can
5543 overestimate the run-time cost of the additional insns, e.g. if
5544 there are multiple basic blocks that increment the biv, but only
5545 one of these blocks is executed during each iteration. There is
5546 no good way to detect cases like this with the current structure
5547 of the loop optimizer. This code is more accurate for
5548 determining code size than run-time benefits. */
5549 benefit -= add_cost * bl->biv_count;
5551 /* Decide whether to strength-reduce this giv or to leave the code
5552 unchanged (recompute it from the biv each time it is used). This
5553 decision can be made independently for each giv. */
5555 #ifdef AUTO_INC_DEC
5556 /* Attempt to guess whether autoincrement will handle some of the
5557 new add insns; if so, increase BENEFIT (undo the subtraction of
5558 add_cost that was done above). */
5559 if (v->giv_type == DEST_ADDR
5560 /* Increasing the benefit is risky, since this is only a guess.
5561 Avoid increasing register pressure in cases where there would
5562 be no other benefit from reducing this giv. */
5563 && benefit > 0
5564 && GET_CODE (v->mult_val) == CONST_INT)
5566 int size = GET_MODE_SIZE (GET_MODE (v->mem));
5568 if (HAVE_POST_INCREMENT
5569 && INTVAL (v->mult_val) == size)
5570 benefit += add_cost * bl->biv_count;
5571 else if (HAVE_PRE_INCREMENT
5572 && INTVAL (v->mult_val) == size)
5573 benefit += add_cost * bl->biv_count;
5574 else if (HAVE_POST_DECREMENT
5575 && -INTVAL (v->mult_val) == size)
5576 benefit += add_cost * bl->biv_count;
5577 else if (HAVE_PRE_DECREMENT
5578 && -INTVAL (v->mult_val) == size)
5579 benefit += add_cost * bl->biv_count;
5581 #endif
5583 return benefit;
5587 /* Free IV structures for LOOP. */
5589 static void
5590 loop_ivs_free (struct loop *loop)
5592 struct loop_ivs *ivs = LOOP_IVS (loop);
5593 struct iv_class *iv = ivs->list;
5595 free (ivs->regs);
5597 while (iv)
5599 struct iv_class *next = iv->next;
5600 struct induction *induction;
5601 struct induction *next_induction;
5603 for (induction = iv->biv; induction; induction = next_induction)
5605 next_induction = induction->next_iv;
5606 free (induction);
5608 for (induction = iv->giv; induction; induction = next_induction)
5610 next_induction = induction->next_iv;
5611 free (induction);
5614 free (iv);
5615 iv = next;
5619 /* Look back before LOOP->START for the insn that sets REG and return
5620 the equivalent constant if there is a REG_EQUAL note otherwise just
5621 the SET_SRC of REG. */
5623 static rtx
5624 loop_find_equiv_value (const struct loop *loop, rtx reg)
5626 rtx loop_start = loop->start;
5627 rtx insn, set;
5628 rtx ret;
5630 ret = reg;
5631 for (insn = PREV_INSN (loop_start); insn; insn = PREV_INSN (insn))
5633 if (LABEL_P (insn))
5634 break;
5636 else if (INSN_P (insn) && reg_set_p (reg, insn))
5638 /* We found the last insn before the loop that sets the register.
5639 If it sets the entire register, and has a REG_EQUAL note,
5640 then use the value of the REG_EQUAL note. */
5641 if ((set = single_set (insn))
5642 && (SET_DEST (set) == reg))
5644 rtx note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
5646 /* Only use the REG_EQUAL note if it is a constant.
5647 Other things, divide in particular, will cause
5648 problems later if we use them. */
5649 if (note && GET_CODE (XEXP (note, 0)) != EXPR_LIST
5650 && CONSTANT_P (XEXP (note, 0)))
5651 ret = XEXP (note, 0);
5652 else
5653 ret = SET_SRC (set);
5655 /* We cannot do this if it changes between the
5656 assignment and loop start though. */
5657 if (modified_between_p (ret, insn, loop_start))
5658 ret = reg;
5660 break;
5663 return ret;
5666 /* Find and return register term common to both expressions OP0 and
5667 OP1 or NULL_RTX if no such term exists. Each expression must be a
5668 REG or a PLUS of a REG. */
5670 static rtx
5671 find_common_reg_term (rtx op0, rtx op1)
5673 if ((REG_P (op0) || GET_CODE (op0) == PLUS)
5674 && (REG_P (op1) || GET_CODE (op1) == PLUS))
5676 rtx op00;
5677 rtx op01;
5678 rtx op10;
5679 rtx op11;
5681 if (GET_CODE (op0) == PLUS)
5682 op01 = XEXP (op0, 1), op00 = XEXP (op0, 0);
5683 else
5684 op01 = const0_rtx, op00 = op0;
5686 if (GET_CODE (op1) == PLUS)
5687 op11 = XEXP (op1, 1), op10 = XEXP (op1, 0);
5688 else
5689 op11 = const0_rtx, op10 = op1;
5691 /* Find and return common register term if present. */
5692 if (REG_P (op00) && (op00 == op10 || op00 == op11))
5693 return op00;
5694 else if (REG_P (op01) && (op01 == op10 || op01 == op11))
5695 return op01;
5698 /* No common register term found. */
5699 return NULL_RTX;
5702 /* Determine the loop iterator and calculate the number of loop
5703 iterations. Returns the exact number of loop iterations if it can
5704 be calculated, otherwise returns zero. */
5706 static unsigned HOST_WIDE_INT
5707 loop_iterations (struct loop *loop)
5709 struct loop_info *loop_info = LOOP_INFO (loop);
5710 struct loop_ivs *ivs = LOOP_IVS (loop);
5711 rtx comparison, comparison_value;
5712 rtx iteration_var, initial_value, increment, final_value;
5713 enum rtx_code comparison_code;
5714 HOST_WIDE_INT inc;
5715 unsigned HOST_WIDE_INT abs_inc;
5716 unsigned HOST_WIDE_INT abs_diff;
5717 int off_by_one;
5718 int increment_dir;
5719 int unsigned_p, compare_dir, final_larger;
5720 rtx last_loop_insn;
5721 struct iv_class *bl;
5723 loop_info->n_iterations = 0;
5724 loop_info->initial_value = 0;
5725 loop_info->initial_equiv_value = 0;
5726 loop_info->comparison_value = 0;
5727 loop_info->final_value = 0;
5728 loop_info->final_equiv_value = 0;
5729 loop_info->increment = 0;
5730 loop_info->iteration_var = 0;
5731 loop_info->iv = 0;
5733 /* We used to use prev_nonnote_insn here, but that fails because it might
5734 accidentally get the branch for a contained loop if the branch for this
5735 loop was deleted. We can only trust branches immediately before the
5736 loop_end. */
5737 last_loop_insn = PREV_INSN (loop->end);
5739 /* ??? We should probably try harder to find the jump insn
5740 at the end of the loop. The following code assumes that
5741 the last loop insn is a jump to the top of the loop. */
5742 if (!JUMP_P (last_loop_insn))
5744 if (loop_dump_stream)
5745 fprintf (loop_dump_stream,
5746 "Loop iterations: No final conditional branch found.\n");
5747 return 0;
5750 /* If there is a more than a single jump to the top of the loop
5751 we cannot (easily) determine the iteration count. */
5752 if (LABEL_NUSES (JUMP_LABEL (last_loop_insn)) > 1)
5754 if (loop_dump_stream)
5755 fprintf (loop_dump_stream,
5756 "Loop iterations: Loop has multiple back edges.\n");
5757 return 0;
5760 /* Find the iteration variable. If the last insn is a conditional
5761 branch, and the insn before tests a register value, make that the
5762 iteration variable. */
5764 comparison = get_condition_for_loop (loop, last_loop_insn);
5765 if (comparison == 0)
5767 if (loop_dump_stream)
5768 fprintf (loop_dump_stream,
5769 "Loop iterations: No final comparison found.\n");
5770 return 0;
5773 /* ??? Get_condition may switch position of induction variable and
5774 invariant register when it canonicalizes the comparison. */
5776 comparison_code = GET_CODE (comparison);
5777 iteration_var = XEXP (comparison, 0);
5778 comparison_value = XEXP (comparison, 1);
5780 if (!REG_P (iteration_var))
5782 if (loop_dump_stream)
5783 fprintf (loop_dump_stream,
5784 "Loop iterations: Comparison not against register.\n");
5785 return 0;
5788 /* The only new registers that are created before loop iterations
5789 are givs made from biv increments or registers created by
5790 load_mems. In the latter case, it is possible that try_copy_prop
5791 will propagate a new pseudo into the old iteration register but
5792 this will be marked by having the REG_USERVAR_P bit set. */
5794 if ((unsigned) REGNO (iteration_var) >= ivs->n_regs
5795 && ! REG_USERVAR_P (iteration_var))
5796 abort ();
5798 /* Determine the initial value of the iteration variable, and the amount
5799 that it is incremented each loop. Use the tables constructed by
5800 the strength reduction pass to calculate these values. */
5802 /* Clear the result values, in case no answer can be found. */
5803 initial_value = 0;
5804 increment = 0;
5806 /* The iteration variable can be either a giv or a biv. Check to see
5807 which it is, and compute the variable's initial value, and increment
5808 value if possible. */
5810 /* If this is a new register, can't handle it since we don't have any
5811 reg_iv_type entry for it. */
5812 if ((unsigned) REGNO (iteration_var) >= ivs->n_regs)
5814 if (loop_dump_stream)
5815 fprintf (loop_dump_stream,
5816 "Loop iterations: No reg_iv_type entry for iteration var.\n");
5817 return 0;
5820 /* Reject iteration variables larger than the host wide int size, since they
5821 could result in a number of iterations greater than the range of our
5822 `unsigned HOST_WIDE_INT' variable loop_info->n_iterations. */
5823 else if ((GET_MODE_BITSIZE (GET_MODE (iteration_var))
5824 > HOST_BITS_PER_WIDE_INT))
5826 if (loop_dump_stream)
5827 fprintf (loop_dump_stream,
5828 "Loop iterations: Iteration var rejected because mode too large.\n");
5829 return 0;
5831 else if (GET_MODE_CLASS (GET_MODE (iteration_var)) != MODE_INT)
5833 if (loop_dump_stream)
5834 fprintf (loop_dump_stream,
5835 "Loop iterations: Iteration var not an integer.\n");
5836 return 0;
5839 /* Try swapping the comparison to identify a suitable iv. */
5840 if (REG_IV_TYPE (ivs, REGNO (iteration_var)) != BASIC_INDUCT
5841 && REG_IV_TYPE (ivs, REGNO (iteration_var)) != GENERAL_INDUCT
5842 && REG_P (comparison_value)
5843 && REGNO (comparison_value) < ivs->n_regs)
5845 rtx temp = comparison_value;
5846 comparison_code = swap_condition (comparison_code);
5847 comparison_value = iteration_var;
5848 iteration_var = temp;
5851 if (REG_IV_TYPE (ivs, REGNO (iteration_var)) == BASIC_INDUCT)
5853 if (REGNO (iteration_var) >= ivs->n_regs)
5854 abort ();
5856 /* Grab initial value, only useful if it is a constant. */
5857 bl = REG_IV_CLASS (ivs, REGNO (iteration_var));
5858 initial_value = bl->initial_value;
5859 if (!bl->biv->always_executed || bl->biv->maybe_multiple)
5861 if (loop_dump_stream)
5862 fprintf (loop_dump_stream,
5863 "Loop iterations: Basic induction var not set once in each iteration.\n");
5864 return 0;
5867 increment = biv_total_increment (bl);
5869 else if (REG_IV_TYPE (ivs, REGNO (iteration_var)) == GENERAL_INDUCT)
5871 HOST_WIDE_INT offset = 0;
5872 struct induction *v = REG_IV_INFO (ivs, REGNO (iteration_var));
5873 rtx biv_initial_value;
5875 if (REGNO (v->src_reg) >= ivs->n_regs)
5876 abort ();
5878 if (!v->always_executed || v->maybe_multiple)
5880 if (loop_dump_stream)
5881 fprintf (loop_dump_stream,
5882 "Loop iterations: General induction var not set once in each iteration.\n");
5883 return 0;
5886 bl = REG_IV_CLASS (ivs, REGNO (v->src_reg));
5888 /* Increment value is mult_val times the increment value of the biv. */
5890 increment = biv_total_increment (bl);
5891 if (increment)
5893 struct induction *biv_inc;
5895 increment = fold_rtx_mult_add (v->mult_val,
5896 extend_value_for_giv (v, increment),
5897 const0_rtx, v->mode);
5898 /* The caller assumes that one full increment has occurred at the
5899 first loop test. But that's not true when the biv is incremented
5900 after the giv is set (which is the usual case), e.g.:
5901 i = 6; do {;} while (i++ < 9) .
5902 Therefore, we bias the initial value by subtracting the amount of
5903 the increment that occurs between the giv set and the giv test. */
5904 for (biv_inc = bl->biv; biv_inc; biv_inc = biv_inc->next_iv)
5906 if (loop_insn_first_p (v->insn, biv_inc->insn))
5908 if (REG_P (biv_inc->add_val))
5910 if (loop_dump_stream)
5911 fprintf (loop_dump_stream,
5912 "Loop iterations: Basic induction var add_val is REG %d.\n",
5913 REGNO (biv_inc->add_val));
5914 return 0;
5917 /* If we have already counted it, skip it. */
5918 if (biv_inc->same)
5919 continue;
5921 offset -= INTVAL (biv_inc->add_val);
5925 if (loop_dump_stream)
5926 fprintf (loop_dump_stream,
5927 "Loop iterations: Giv iterator, initial value bias %ld.\n",
5928 (long) offset);
5930 /* Initial value is mult_val times the biv's initial value plus
5931 add_val. Only useful if it is a constant. */
5932 biv_initial_value = extend_value_for_giv (v, bl->initial_value);
5933 initial_value
5934 = fold_rtx_mult_add (v->mult_val,
5935 plus_constant (biv_initial_value, offset),
5936 v->add_val, v->mode);
5938 else
5940 if (loop_dump_stream)
5941 fprintf (loop_dump_stream,
5942 "Loop iterations: Not basic or general induction var.\n");
5943 return 0;
5946 if (initial_value == 0)
5947 return 0;
5949 unsigned_p = 0;
5950 off_by_one = 0;
5951 switch (comparison_code)
5953 case LEU:
5954 unsigned_p = 1;
5955 case LE:
5956 compare_dir = 1;
5957 off_by_one = 1;
5958 break;
5959 case GEU:
5960 unsigned_p = 1;
5961 case GE:
5962 compare_dir = -1;
5963 off_by_one = -1;
5964 break;
5965 case EQ:
5966 /* Cannot determine loop iterations with this case. */
5967 compare_dir = 0;
5968 break;
5969 case LTU:
5970 unsigned_p = 1;
5971 case LT:
5972 compare_dir = 1;
5973 break;
5974 case GTU:
5975 unsigned_p = 1;
5976 case GT:
5977 compare_dir = -1;
5978 break;
5979 case NE:
5980 compare_dir = 0;
5981 break;
5982 default:
5983 abort ();
5986 /* If the comparison value is an invariant register, then try to find
5987 its value from the insns before the start of the loop. */
5989 final_value = comparison_value;
5990 if (REG_P (comparison_value)
5991 && loop_invariant_p (loop, comparison_value))
5993 final_value = loop_find_equiv_value (loop, comparison_value);
5995 /* If we don't get an invariant final value, we are better
5996 off with the original register. */
5997 if (! loop_invariant_p (loop, final_value))
5998 final_value = comparison_value;
6001 /* Calculate the approximate final value of the induction variable
6002 (on the last successful iteration). The exact final value
6003 depends on the branch operator, and increment sign. It will be
6004 wrong if the iteration variable is not incremented by one each
6005 time through the loop and (comparison_value + off_by_one -
6006 initial_value) % increment != 0.
6007 ??? Note that the final_value may overflow and thus final_larger
6008 will be bogus. A potentially infinite loop will be classified
6009 as immediate, e.g. for (i = 0x7ffffff0; i <= 0x7fffffff; i++) */
6010 if (off_by_one)
6011 final_value = plus_constant (final_value, off_by_one);
6013 /* Save the calculated values describing this loop's bounds, in case
6014 precondition_loop_p will need them later. These values can not be
6015 recalculated inside precondition_loop_p because strength reduction
6016 optimizations may obscure the loop's structure.
6018 These values are only required by precondition_loop_p and insert_bct
6019 whenever the number of iterations cannot be computed at compile time.
6020 Only the difference between final_value and initial_value is
6021 important. Note that final_value is only approximate. */
6022 loop_info->initial_value = initial_value;
6023 loop_info->comparison_value = comparison_value;
6024 loop_info->final_value = plus_constant (comparison_value, off_by_one);
6025 loop_info->increment = increment;
6026 loop_info->iteration_var = iteration_var;
6027 loop_info->comparison_code = comparison_code;
6028 loop_info->iv = bl;
6030 /* Try to determine the iteration count for loops such
6031 as (for i = init; i < init + const; i++). When running the
6032 loop optimization twice, the first pass often converts simple
6033 loops into this form. */
6035 if (REG_P (initial_value))
6037 rtx reg1;
6038 rtx reg2;
6039 rtx const2;
6041 reg1 = initial_value;
6042 if (GET_CODE (final_value) == PLUS)
6043 reg2 = XEXP (final_value, 0), const2 = XEXP (final_value, 1);
6044 else
6045 reg2 = final_value, const2 = const0_rtx;
6047 /* Check for initial_value = reg1, final_value = reg2 + const2,
6048 where reg1 != reg2. */
6049 if (REG_P (reg2) && reg2 != reg1)
6051 rtx temp;
6053 /* Find what reg1 is equivalent to. Hopefully it will
6054 either be reg2 or reg2 plus a constant. */
6055 temp = loop_find_equiv_value (loop, reg1);
6057 if (find_common_reg_term (temp, reg2))
6058 initial_value = temp;
6059 else if (loop_invariant_p (loop, reg2))
6061 /* Find what reg2 is equivalent to. Hopefully it will
6062 either be reg1 or reg1 plus a constant. Let's ignore
6063 the latter case for now since it is not so common. */
6064 temp = loop_find_equiv_value (loop, reg2);
6066 if (temp == loop_info->iteration_var)
6067 temp = initial_value;
6068 if (temp == reg1)
6069 final_value = (const2 == const0_rtx)
6070 ? reg1 : gen_rtx_PLUS (GET_MODE (reg1), reg1, const2);
6075 loop_info->initial_equiv_value = initial_value;
6076 loop_info->final_equiv_value = final_value;
6078 /* For EQ comparison loops, we don't have a valid final value.
6079 Check this now so that we won't leave an invalid value if we
6080 return early for any other reason. */
6081 if (comparison_code == EQ)
6082 loop_info->final_equiv_value = loop_info->final_value = 0;
6084 if (increment == 0)
6086 if (loop_dump_stream)
6087 fprintf (loop_dump_stream,
6088 "Loop iterations: Increment value can't be calculated.\n");
6089 return 0;
6092 if (GET_CODE (increment) != CONST_INT)
6094 /* If we have a REG, check to see if REG holds a constant value. */
6095 /* ??? Other RTL, such as (neg (reg)) is possible here, but it isn't
6096 clear if it is worthwhile to try to handle such RTL. */
6097 if (REG_P (increment) || GET_CODE (increment) == SUBREG)
6098 increment = loop_find_equiv_value (loop, increment);
6100 if (GET_CODE (increment) != CONST_INT)
6102 if (loop_dump_stream)
6104 fprintf (loop_dump_stream,
6105 "Loop iterations: Increment value not constant ");
6106 print_simple_rtl (loop_dump_stream, increment);
6107 fprintf (loop_dump_stream, ".\n");
6109 return 0;
6111 loop_info->increment = increment;
6114 if (GET_CODE (initial_value) != CONST_INT)
6116 if (loop_dump_stream)
6118 fprintf (loop_dump_stream,
6119 "Loop iterations: Initial value not constant ");
6120 print_simple_rtl (loop_dump_stream, initial_value);
6121 fprintf (loop_dump_stream, ".\n");
6123 return 0;
6125 else if (GET_CODE (final_value) != CONST_INT)
6127 if (loop_dump_stream)
6129 fprintf (loop_dump_stream,
6130 "Loop iterations: Final value not constant ");
6131 print_simple_rtl (loop_dump_stream, final_value);
6132 fprintf (loop_dump_stream, ".\n");
6134 return 0;
6136 else if (comparison_code == EQ)
6138 rtx inc_once;
6140 if (loop_dump_stream)
6141 fprintf (loop_dump_stream, "Loop iterations: EQ comparison loop.\n");
6143 inc_once = gen_int_mode (INTVAL (initial_value) + INTVAL (increment),
6144 GET_MODE (iteration_var));
6146 if (inc_once == final_value)
6148 /* The iterator value once through the loop is equal to the
6149 comparison value. Either we have an infinite loop, or
6150 we'll loop twice. */
6151 if (increment == const0_rtx)
6152 return 0;
6153 loop_info->n_iterations = 2;
6155 else
6156 loop_info->n_iterations = 1;
6158 if (GET_CODE (loop_info->initial_value) == CONST_INT)
6159 loop_info->final_value
6160 = gen_int_mode ((INTVAL (loop_info->initial_value)
6161 + loop_info->n_iterations * INTVAL (increment)),
6162 GET_MODE (iteration_var));
6163 else
6164 loop_info->final_value
6165 = plus_constant (loop_info->initial_value,
6166 loop_info->n_iterations * INTVAL (increment));
6167 loop_info->final_equiv_value
6168 = gen_int_mode ((INTVAL (initial_value)
6169 + loop_info->n_iterations * INTVAL (increment)),
6170 GET_MODE (iteration_var));
6171 return loop_info->n_iterations;
6174 /* Final_larger is 1 if final larger, 0 if they are equal, otherwise -1. */
6175 if (unsigned_p)
6176 final_larger
6177 = ((unsigned HOST_WIDE_INT) INTVAL (final_value)
6178 > (unsigned HOST_WIDE_INT) INTVAL (initial_value))
6179 - ((unsigned HOST_WIDE_INT) INTVAL (final_value)
6180 < (unsigned HOST_WIDE_INT) INTVAL (initial_value));
6181 else
6182 final_larger = (INTVAL (final_value) > INTVAL (initial_value))
6183 - (INTVAL (final_value) < INTVAL (initial_value));
6185 if (INTVAL (increment) > 0)
6186 increment_dir = 1;
6187 else if (INTVAL (increment) == 0)
6188 increment_dir = 0;
6189 else
6190 increment_dir = -1;
6192 /* There are 27 different cases: compare_dir = -1, 0, 1;
6193 final_larger = -1, 0, 1; increment_dir = -1, 0, 1.
6194 There are 4 normal cases, 4 reverse cases (where the iteration variable
6195 will overflow before the loop exits), 4 infinite loop cases, and 15
6196 immediate exit (0 or 1 iteration depending on loop type) cases.
6197 Only try to optimize the normal cases. */
6199 /* (compare_dir/final_larger/increment_dir)
6200 Normal cases: (0/-1/-1), (0/1/1), (-1/-1/-1), (1/1/1)
6201 Reverse cases: (0/-1/1), (0/1/-1), (-1/-1/1), (1/1/-1)
6202 Infinite loops: (0/-1/0), (0/1/0), (-1/-1/0), (1/1/0)
6203 Immediate exit: (0/0/X), (-1/0/X), (-1/1/X), (1/0/X), (1/-1/X) */
6205 /* ?? If the meaning of reverse loops (where the iteration variable
6206 will overflow before the loop exits) is undefined, then could
6207 eliminate all of these special checks, and just always assume
6208 the loops are normal/immediate/infinite. Note that this means
6209 the sign of increment_dir does not have to be known. Also,
6210 since it does not really hurt if immediate exit loops or infinite loops
6211 are optimized, then that case could be ignored also, and hence all
6212 loops can be optimized.
6214 According to ANSI Spec, the reverse loop case result is undefined,
6215 because the action on overflow is undefined.
6217 See also the special test for NE loops below. */
6219 if (final_larger == increment_dir && final_larger != 0
6220 && (final_larger == compare_dir || compare_dir == 0))
6221 /* Normal case. */
6223 else
6225 if (loop_dump_stream)
6226 fprintf (loop_dump_stream, "Loop iterations: Not normal loop.\n");
6227 return 0;
6230 /* Calculate the number of iterations, final_value is only an approximation,
6231 so correct for that. Note that abs_diff and n_iterations are
6232 unsigned, because they can be as large as 2^n - 1. */
6234 inc = INTVAL (increment);
6235 if (inc > 0)
6237 abs_diff = INTVAL (final_value) - INTVAL (initial_value);
6238 abs_inc = inc;
6240 else if (inc < 0)
6242 abs_diff = INTVAL (initial_value) - INTVAL (final_value);
6243 abs_inc = -inc;
6245 else
6246 abort ();
6248 /* Given that iteration_var is going to iterate over its own mode,
6249 not HOST_WIDE_INT, disregard higher bits that might have come
6250 into the picture due to sign extension of initial and final
6251 values. */
6252 abs_diff &= ((unsigned HOST_WIDE_INT) 1
6253 << (GET_MODE_BITSIZE (GET_MODE (iteration_var)) - 1)
6254 << 1) - 1;
6256 /* For NE tests, make sure that the iteration variable won't miss
6257 the final value. If abs_diff mod abs_incr is not zero, then the
6258 iteration variable will overflow before the loop exits, and we
6259 can not calculate the number of iterations. */
6260 if (compare_dir == 0 && (abs_diff % abs_inc) != 0)
6261 return 0;
6263 /* Note that the number of iterations could be calculated using
6264 (abs_diff + abs_inc - 1) / abs_inc, provided care was taken to
6265 handle potential overflow of the summation. */
6266 loop_info->n_iterations = abs_diff / abs_inc + ((abs_diff % abs_inc) != 0);
6267 return loop_info->n_iterations;
6270 /* Perform strength reduction and induction variable elimination.
6272 Pseudo registers created during this function will be beyond the
6273 last valid index in several tables including
6274 REGS->ARRAY[I].N_TIMES_SET and REGNO_LAST_UID. This does not cause a
6275 problem here, because the added registers cannot be givs outside of
6276 their loop, and hence will never be reconsidered. But scan_loop
6277 must check regnos to make sure they are in bounds. */
6279 static void
6280 strength_reduce (struct loop *loop, int flags)
6282 struct loop_info *loop_info = LOOP_INFO (loop);
6283 struct loop_regs *regs = LOOP_REGS (loop);
6284 struct loop_ivs *ivs = LOOP_IVS (loop);
6285 rtx p;
6286 /* Temporary list pointer for traversing ivs->list. */
6287 struct iv_class *bl;
6288 /* Ratio of extra register life span we can justify
6289 for saving an instruction. More if loop doesn't call subroutines
6290 since in that case saving an insn makes more difference
6291 and more registers are available. */
6292 /* ??? could set this to last value of threshold in move_movables */
6293 int threshold = (loop_info->has_call ? 1 : 2) * (3 + n_non_fixed_regs);
6294 /* Map of pseudo-register replacements. */
6295 rtx *reg_map = NULL;
6296 int reg_map_size;
6297 rtx test_reg = gen_rtx_REG (word_mode, LAST_VIRTUAL_REGISTER + 1);
6298 int insn_count = count_insns_in_loop (loop);
6300 addr_placeholder = gen_reg_rtx (Pmode);
6302 ivs->n_regs = max_reg_before_loop;
6303 ivs->regs = xcalloc (ivs->n_regs, sizeof (struct iv));
6305 /* Find all BIVs in loop. */
6306 loop_bivs_find (loop);
6308 /* Exit if there are no bivs. */
6309 if (! ivs->list)
6311 loop_ivs_free (loop);
6312 return;
6315 /* Determine how BIVS are initialized by looking through pre-header
6316 extended basic block. */
6317 loop_bivs_init_find (loop);
6319 /* Look at the each biv and see if we can say anything better about its
6320 initial value from any initializing insns set up above. */
6321 loop_bivs_check (loop);
6323 /* Search the loop for general induction variables. */
6324 loop_givs_find (loop);
6326 /* Try to calculate and save the number of loop iterations. This is
6327 set to zero if the actual number can not be calculated. This must
6328 be called after all giv's have been identified, since otherwise it may
6329 fail if the iteration variable is a giv. */
6330 loop_iterations (loop);
6332 #ifdef HAVE_prefetch
6333 if (flags & LOOP_PREFETCH)
6334 emit_prefetch_instructions (loop);
6335 #endif
6337 /* Now for each giv for which we still don't know whether or not it is
6338 replaceable, check to see if it is replaceable because its final value
6339 can be calculated. This must be done after loop_iterations is called,
6340 so that final_giv_value will work correctly. */
6341 loop_givs_check (loop);
6343 /* Try to prove that the loop counter variable (if any) is always
6344 nonnegative; if so, record that fact with a REG_NONNEG note
6345 so that "decrement and branch until zero" insn can be used. */
6346 check_dbra_loop (loop, insn_count);
6348 /* Create reg_map to hold substitutions for replaceable giv regs.
6349 Some givs might have been made from biv increments, so look at
6350 ivs->reg_iv_type for a suitable size. */
6351 reg_map_size = ivs->n_regs;
6352 reg_map = xcalloc (reg_map_size, sizeof (rtx));
6354 /* Examine each iv class for feasibility of strength reduction/induction
6355 variable elimination. */
6357 for (bl = ivs->list; bl; bl = bl->next)
6359 struct induction *v;
6360 int benefit;
6362 /* Test whether it will be possible to eliminate this biv
6363 provided all givs are reduced. */
6364 bl->eliminable = loop_biv_eliminable_p (loop, bl, threshold, insn_count);
6366 /* This will be true at the end, if all givs which depend on this
6367 biv have been strength reduced.
6368 We can't (currently) eliminate the biv unless this is so. */
6369 bl->all_reduced = 1;
6371 /* Check each extension dependent giv in this class to see if its
6372 root biv is safe from wrapping in the interior mode. */
6373 check_ext_dependent_givs (loop, bl);
6375 /* Combine all giv's for this iv_class. */
6376 combine_givs (regs, bl);
6378 for (v = bl->giv; v; v = v->next_iv)
6380 struct induction *tv;
6382 if (v->ignore || v->same)
6383 continue;
6385 benefit = loop_giv_reduce_benefit (loop, bl, v, test_reg);
6387 /* If an insn is not to be strength reduced, then set its ignore
6388 flag, and clear bl->all_reduced. */
6390 /* A giv that depends on a reversed biv must be reduced if it is
6391 used after the loop exit, otherwise, it would have the wrong
6392 value after the loop exit. To make it simple, just reduce all
6393 of such giv's whether or not we know they are used after the loop
6394 exit. */
6396 if (v->lifetime * threshold * benefit < insn_count
6397 && ! bl->reversed)
6399 if (loop_dump_stream)
6400 fprintf (loop_dump_stream,
6401 "giv of insn %d not worth while, %d vs %d.\n",
6402 INSN_UID (v->insn),
6403 v->lifetime * threshold * benefit, insn_count);
6404 v->ignore = 1;
6405 bl->all_reduced = 0;
6407 else
6409 /* Check that we can increment the reduced giv without a
6410 multiply insn. If not, reject it. */
6412 for (tv = bl->biv; tv; tv = tv->next_iv)
6413 if (tv->mult_val == const1_rtx
6414 && ! product_cheap_p (tv->add_val, v->mult_val))
6416 if (loop_dump_stream)
6417 fprintf (loop_dump_stream,
6418 "giv of insn %d: would need a multiply.\n",
6419 INSN_UID (v->insn));
6420 v->ignore = 1;
6421 bl->all_reduced = 0;
6422 break;
6427 /* Check for givs whose first use is their definition and whose
6428 last use is the definition of another giv. If so, it is likely
6429 dead and should not be used to derive another giv nor to
6430 eliminate a biv. */
6431 loop_givs_dead_check (loop, bl);
6433 /* Reduce each giv that we decided to reduce. */
6434 loop_givs_reduce (loop, bl);
6436 /* Rescan all givs. If a giv is the same as a giv not reduced, mark it
6437 as not reduced.
6439 For each giv register that can be reduced now: if replaceable,
6440 substitute reduced reg wherever the old giv occurs;
6441 else add new move insn "giv_reg = reduced_reg". */
6442 loop_givs_rescan (loop, bl, reg_map);
6444 /* All the givs based on the biv bl have been reduced if they
6445 merit it. */
6447 /* For each giv not marked as maybe dead that has been combined with a
6448 second giv, clear any "maybe dead" mark on that second giv.
6449 v->new_reg will either be or refer to the register of the giv it
6450 combined with.
6452 Doing this clearing avoids problems in biv elimination where
6453 a giv's new_reg is a complex value that can't be put in the
6454 insn but the giv combined with (with a reg as new_reg) is
6455 marked maybe_dead. Since the register will be used in either
6456 case, we'd prefer it be used from the simpler giv. */
6458 for (v = bl->giv; v; v = v->next_iv)
6459 if (! v->maybe_dead && v->same)
6460 v->same->maybe_dead = 0;
6462 /* Try to eliminate the biv, if it is a candidate.
6463 This won't work if ! bl->all_reduced,
6464 since the givs we planned to use might not have been reduced.
6466 We have to be careful that we didn't initially think we could
6467 eliminate this biv because of a giv that we now think may be
6468 dead and shouldn't be used as a biv replacement.
6470 Also, there is the possibility that we may have a giv that looks
6471 like it can be used to eliminate a biv, but the resulting insn
6472 isn't valid. This can happen, for example, on the 88k, where a
6473 JUMP_INSN can compare a register only with zero. Attempts to
6474 replace it with a compare with a constant will fail.
6476 Note that in cases where this call fails, we may have replaced some
6477 of the occurrences of the biv with a giv, but no harm was done in
6478 doing so in the rare cases where it can occur. */
6480 if (bl->all_reduced == 1 && bl->eliminable
6481 && maybe_eliminate_biv (loop, bl, 1, threshold, insn_count))
6483 /* ?? If we created a new test to bypass the loop entirely,
6484 or otherwise drop straight in, based on this test, then
6485 we might want to rewrite it also. This way some later
6486 pass has more hope of removing the initialization of this
6487 biv entirely. */
6489 /* If final_value != 0, then the biv may be used after loop end
6490 and we must emit an insn to set it just in case.
6492 Reversed bivs already have an insn after the loop setting their
6493 value, so we don't need another one. We can't calculate the
6494 proper final value for such a biv here anyways. */
6495 if (bl->final_value && ! bl->reversed)
6496 loop_insn_sink_or_swim (loop,
6497 gen_load_of_final_value (bl->biv->dest_reg,
6498 bl->final_value));
6500 if (loop_dump_stream)
6501 fprintf (loop_dump_stream, "Reg %d: biv eliminated\n",
6502 bl->regno);
6504 /* See above note wrt final_value. But since we couldn't eliminate
6505 the biv, we must set the value after the loop instead of before. */
6506 else if (bl->final_value && ! bl->reversed)
6507 loop_insn_sink (loop, gen_load_of_final_value (bl->biv->dest_reg,
6508 bl->final_value));
6511 /* Go through all the instructions in the loop, making all the
6512 register substitutions scheduled in REG_MAP. */
6514 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
6515 if (INSN_P (p))
6517 replace_regs (PATTERN (p), reg_map, reg_map_size, 0);
6518 replace_regs (REG_NOTES (p), reg_map, reg_map_size, 0);
6519 INSN_CODE (p) = -1;
6522 if (loop_dump_stream)
6523 fprintf (loop_dump_stream, "\n");
6525 loop_ivs_free (loop);
6526 if (reg_map)
6527 free (reg_map);
6530 /*Record all basic induction variables calculated in the insn. */
6531 static rtx
6532 check_insn_for_bivs (struct loop *loop, rtx p, int not_every_iteration,
6533 int maybe_multiple)
6535 struct loop_ivs *ivs = LOOP_IVS (loop);
6536 rtx set;
6537 rtx dest_reg;
6538 rtx inc_val;
6539 rtx mult_val;
6540 rtx *location;
6542 if (NONJUMP_INSN_P (p)
6543 && (set = single_set (p))
6544 && REG_P (SET_DEST (set)))
6546 dest_reg = SET_DEST (set);
6547 if (REGNO (dest_reg) < max_reg_before_loop
6548 && REGNO (dest_reg) >= FIRST_PSEUDO_REGISTER
6549 && REG_IV_TYPE (ivs, REGNO (dest_reg)) != NOT_BASIC_INDUCT)
6551 if (basic_induction_var (loop, SET_SRC (set),
6552 GET_MODE (SET_SRC (set)),
6553 dest_reg, p, &inc_val, &mult_val,
6554 &location))
6556 /* It is a possible basic induction variable.
6557 Create and initialize an induction structure for it. */
6559 struct induction *v = xmalloc (sizeof (struct induction));
6561 record_biv (loop, v, p, dest_reg, inc_val, mult_val, location,
6562 not_every_iteration, maybe_multiple);
6563 REG_IV_TYPE (ivs, REGNO (dest_reg)) = BASIC_INDUCT;
6565 else if (REGNO (dest_reg) < ivs->n_regs)
6566 REG_IV_TYPE (ivs, REGNO (dest_reg)) = NOT_BASIC_INDUCT;
6569 return p;
6572 /* Record all givs calculated in the insn.
6573 A register is a giv if: it is only set once, it is a function of a
6574 biv and a constant (or invariant), and it is not a biv. */
6575 static rtx
6576 check_insn_for_givs (struct loop *loop, rtx p, int not_every_iteration,
6577 int maybe_multiple)
6579 struct loop_regs *regs = LOOP_REGS (loop);
6581 rtx set;
6582 /* Look for a general induction variable in a register. */
6583 if (NONJUMP_INSN_P (p)
6584 && (set = single_set (p))
6585 && REG_P (SET_DEST (set))
6586 && ! regs->array[REGNO (SET_DEST (set))].may_not_optimize)
6588 rtx src_reg;
6589 rtx dest_reg;
6590 rtx add_val;
6591 rtx mult_val;
6592 rtx ext_val;
6593 int benefit;
6594 rtx regnote = 0;
6595 rtx last_consec_insn;
6597 dest_reg = SET_DEST (set);
6598 if (REGNO (dest_reg) < FIRST_PSEUDO_REGISTER)
6599 return p;
6601 if (/* SET_SRC is a giv. */
6602 (general_induction_var (loop, SET_SRC (set), &src_reg, &add_val,
6603 &mult_val, &ext_val, 0, &benefit, VOIDmode)
6604 /* Equivalent expression is a giv. */
6605 || ((regnote = find_reg_note (p, REG_EQUAL, NULL_RTX))
6606 && general_induction_var (loop, XEXP (regnote, 0), &src_reg,
6607 &add_val, &mult_val, &ext_val, 0,
6608 &benefit, VOIDmode)))
6609 /* Don't try to handle any regs made by loop optimization.
6610 We have nothing on them in regno_first_uid, etc. */
6611 && REGNO (dest_reg) < max_reg_before_loop
6612 /* Don't recognize a BASIC_INDUCT_VAR here. */
6613 && dest_reg != src_reg
6614 /* This must be the only place where the register is set. */
6615 && (regs->array[REGNO (dest_reg)].n_times_set == 1
6616 /* or all sets must be consecutive and make a giv. */
6617 || (benefit = consec_sets_giv (loop, benefit, p,
6618 src_reg, dest_reg,
6619 &add_val, &mult_val, &ext_val,
6620 &last_consec_insn))))
6622 struct induction *v = xmalloc (sizeof (struct induction));
6624 /* If this is a library call, increase benefit. */
6625 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
6626 benefit += libcall_benefit (p);
6628 /* Skip the consecutive insns, if there are any. */
6629 if (regs->array[REGNO (dest_reg)].n_times_set != 1)
6630 p = last_consec_insn;
6632 record_giv (loop, v, p, src_reg, dest_reg, mult_val, add_val,
6633 ext_val, benefit, DEST_REG, not_every_iteration,
6634 maybe_multiple, (rtx*) 0);
6639 /* Look for givs which are memory addresses. */
6640 if (NONJUMP_INSN_P (p))
6641 find_mem_givs (loop, PATTERN (p), p, not_every_iteration,
6642 maybe_multiple);
6644 /* Update the status of whether giv can derive other givs. This can
6645 change when we pass a label or an insn that updates a biv. */
6646 if (INSN_P (p) || LABEL_P (p))
6647 update_giv_derive (loop, p);
6648 return p;
6651 /* Return 1 if X is a valid source for an initial value (or as value being
6652 compared against in an initial test).
6654 X must be either a register or constant and must not be clobbered between
6655 the current insn and the start of the loop.
6657 INSN is the insn containing X. */
6659 static int
6660 valid_initial_value_p (rtx x, rtx insn, int call_seen, rtx loop_start)
6662 if (CONSTANT_P (x))
6663 return 1;
6665 /* Only consider pseudos we know about initialized in insns whose luids
6666 we know. */
6667 if (!REG_P (x)
6668 || REGNO (x) >= max_reg_before_loop)
6669 return 0;
6671 /* Don't use call-clobbered registers across a call which clobbers it. On
6672 some machines, don't use any hard registers at all. */
6673 if (REGNO (x) < FIRST_PSEUDO_REGISTER
6674 && (SMALL_REGISTER_CLASSES
6675 || (call_used_regs[REGNO (x)] && call_seen)))
6676 return 0;
6678 /* Don't use registers that have been clobbered before the start of the
6679 loop. */
6680 if (reg_set_between_p (x, insn, loop_start))
6681 return 0;
6683 return 1;
6686 /* Scan X for memory refs and check each memory address
6687 as a possible giv. INSN is the insn whose pattern X comes from.
6688 NOT_EVERY_ITERATION is 1 if the insn might not be executed during
6689 every loop iteration. MAYBE_MULTIPLE is 1 if the insn might be executed
6690 more than once in each loop iteration. */
6692 static void
6693 find_mem_givs (const struct loop *loop, rtx x, rtx insn,
6694 int not_every_iteration, int maybe_multiple)
6696 int i, j;
6697 enum rtx_code code;
6698 const char *fmt;
6700 if (x == 0)
6701 return;
6703 code = GET_CODE (x);
6704 switch (code)
6706 case REG:
6707 case CONST_INT:
6708 case CONST:
6709 case CONST_DOUBLE:
6710 case SYMBOL_REF:
6711 case LABEL_REF:
6712 case PC:
6713 case CC0:
6714 case ADDR_VEC:
6715 case ADDR_DIFF_VEC:
6716 case USE:
6717 case CLOBBER:
6718 return;
6720 case MEM:
6722 rtx src_reg;
6723 rtx add_val;
6724 rtx mult_val;
6725 rtx ext_val;
6726 int benefit;
6728 /* This code used to disable creating GIVs with mult_val == 1 and
6729 add_val == 0. However, this leads to lost optimizations when
6730 it comes time to combine a set of related DEST_ADDR GIVs, since
6731 this one would not be seen. */
6733 if (general_induction_var (loop, XEXP (x, 0), &src_reg, &add_val,
6734 &mult_val, &ext_val, 1, &benefit,
6735 GET_MODE (x)))
6737 /* Found one; record it. */
6738 struct induction *v = xmalloc (sizeof (struct induction));
6740 record_giv (loop, v, insn, src_reg, addr_placeholder, mult_val,
6741 add_val, ext_val, benefit, DEST_ADDR,
6742 not_every_iteration, maybe_multiple, &XEXP (x, 0));
6744 v->mem = x;
6747 return;
6749 default:
6750 break;
6753 /* Recursively scan the subexpressions for other mem refs. */
6755 fmt = GET_RTX_FORMAT (code);
6756 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
6757 if (fmt[i] == 'e')
6758 find_mem_givs (loop, XEXP (x, i), insn, not_every_iteration,
6759 maybe_multiple);
6760 else if (fmt[i] == 'E')
6761 for (j = 0; j < XVECLEN (x, i); j++)
6762 find_mem_givs (loop, XVECEXP (x, i, j), insn, not_every_iteration,
6763 maybe_multiple);
6766 /* Fill in the data about one biv update.
6767 V is the `struct induction' in which we record the biv. (It is
6768 allocated by the caller, with alloca.)
6769 INSN is the insn that sets it.
6770 DEST_REG is the biv's reg.
6772 MULT_VAL is const1_rtx if the biv is being incremented here, in which case
6773 INC_VAL is the increment. Otherwise, MULT_VAL is const0_rtx and the biv is
6774 being set to INC_VAL.
6776 NOT_EVERY_ITERATION is nonzero if this biv update is not know to be
6777 executed every iteration; MAYBE_MULTIPLE is nonzero if this biv update
6778 can be executed more than once per iteration. If MAYBE_MULTIPLE
6779 and NOT_EVERY_ITERATION are both zero, we know that the biv update is
6780 executed exactly once per iteration. */
6782 static void
6783 record_biv (struct loop *loop, struct induction *v, rtx insn, rtx dest_reg,
6784 rtx inc_val, rtx mult_val, rtx *location,
6785 int not_every_iteration, int maybe_multiple)
6787 struct loop_ivs *ivs = LOOP_IVS (loop);
6788 struct iv_class *bl;
6790 v->insn = insn;
6791 v->src_reg = dest_reg;
6792 v->dest_reg = dest_reg;
6793 v->mult_val = mult_val;
6794 v->add_val = inc_val;
6795 v->ext_dependent = NULL_RTX;
6796 v->location = location;
6797 v->mode = GET_MODE (dest_reg);
6798 v->always_computable = ! not_every_iteration;
6799 v->always_executed = ! not_every_iteration;
6800 v->maybe_multiple = maybe_multiple;
6801 v->same = 0;
6803 /* Add this to the reg's iv_class, creating a class
6804 if this is the first incrementation of the reg. */
6806 bl = REG_IV_CLASS (ivs, REGNO (dest_reg));
6807 if (bl == 0)
6809 /* Create and initialize new iv_class. */
6811 bl = xmalloc (sizeof (struct iv_class));
6813 bl->regno = REGNO (dest_reg);
6814 bl->biv = 0;
6815 bl->giv = 0;
6816 bl->biv_count = 0;
6817 bl->giv_count = 0;
6819 /* Set initial value to the reg itself. */
6820 bl->initial_value = dest_reg;
6821 bl->final_value = 0;
6822 /* We haven't seen the initializing insn yet. */
6823 bl->init_insn = 0;
6824 bl->init_set = 0;
6825 bl->initial_test = 0;
6826 bl->incremented = 0;
6827 bl->eliminable = 0;
6828 bl->nonneg = 0;
6829 bl->reversed = 0;
6830 bl->total_benefit = 0;
6832 /* Add this class to ivs->list. */
6833 bl->next = ivs->list;
6834 ivs->list = bl;
6836 /* Put it in the array of biv register classes. */
6837 REG_IV_CLASS (ivs, REGNO (dest_reg)) = bl;
6839 else
6841 /* Check if location is the same as a previous one. */
6842 struct induction *induction;
6843 for (induction = bl->biv; induction; induction = induction->next_iv)
6844 if (location == induction->location)
6846 v->same = induction;
6847 break;
6851 /* Update IV_CLASS entry for this biv. */
6852 v->next_iv = bl->biv;
6853 bl->biv = v;
6854 bl->biv_count++;
6855 if (mult_val == const1_rtx)
6856 bl->incremented = 1;
6858 if (loop_dump_stream)
6859 loop_biv_dump (v, loop_dump_stream, 0);
6862 /* Fill in the data about one giv.
6863 V is the `struct induction' in which we record the giv. (It is
6864 allocated by the caller, with alloca.)
6865 INSN is the insn that sets it.
6866 BENEFIT estimates the savings from deleting this insn.
6867 TYPE is DEST_REG or DEST_ADDR; it says whether the giv is computed
6868 into a register or is used as a memory address.
6870 SRC_REG is the biv reg which the giv is computed from.
6871 DEST_REG is the giv's reg (if the giv is stored in a reg).
6872 MULT_VAL and ADD_VAL are the coefficients used to compute the giv.
6873 LOCATION points to the place where this giv's value appears in INSN. */
6875 static void
6876 record_giv (const struct loop *loop, struct induction *v, rtx insn,
6877 rtx src_reg, rtx dest_reg, rtx mult_val, rtx add_val,
6878 rtx ext_val, int benefit, enum g_types type,
6879 int not_every_iteration, int maybe_multiple, rtx *location)
6881 struct loop_ivs *ivs = LOOP_IVS (loop);
6882 struct induction *b;
6883 struct iv_class *bl;
6884 rtx set = single_set (insn);
6885 rtx temp;
6887 /* Attempt to prove constantness of the values. Don't let simplify_rtx
6888 undo the MULT canonicalization that we performed earlier. */
6889 temp = simplify_rtx (add_val);
6890 if (temp
6891 && ! (GET_CODE (add_val) == MULT
6892 && GET_CODE (temp) == ASHIFT))
6893 add_val = temp;
6895 v->insn = insn;
6896 v->src_reg = src_reg;
6897 v->giv_type = type;
6898 v->dest_reg = dest_reg;
6899 v->mult_val = mult_val;
6900 v->add_val = add_val;
6901 v->ext_dependent = ext_val;
6902 v->benefit = benefit;
6903 v->location = location;
6904 v->cant_derive = 0;
6905 v->combined_with = 0;
6906 v->maybe_multiple = maybe_multiple;
6907 v->maybe_dead = 0;
6908 v->derive_adjustment = 0;
6909 v->same = 0;
6910 v->ignore = 0;
6911 v->new_reg = 0;
6912 v->final_value = 0;
6913 v->same_insn = 0;
6914 v->auto_inc_opt = 0;
6915 v->shared = 0;
6917 /* The v->always_computable field is used in update_giv_derive, to
6918 determine whether a giv can be used to derive another giv. For a
6919 DEST_REG giv, INSN computes a new value for the giv, so its value
6920 isn't computable if INSN insn't executed every iteration.
6921 However, for a DEST_ADDR giv, INSN merely uses the value of the giv;
6922 it does not compute a new value. Hence the value is always computable
6923 regardless of whether INSN is executed each iteration. */
6925 if (type == DEST_ADDR)
6926 v->always_computable = 1;
6927 else
6928 v->always_computable = ! not_every_iteration;
6930 v->always_executed = ! not_every_iteration;
6932 if (type == DEST_ADDR)
6934 v->mode = GET_MODE (*location);
6935 v->lifetime = 1;
6937 else /* type == DEST_REG */
6939 v->mode = GET_MODE (SET_DEST (set));
6941 v->lifetime = LOOP_REG_LIFETIME (loop, REGNO (dest_reg));
6943 /* If the lifetime is zero, it means that this register is
6944 really a dead store. So mark this as a giv that can be
6945 ignored. This will not prevent the biv from being eliminated. */
6946 if (v->lifetime == 0)
6947 v->ignore = 1;
6949 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
6950 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
6953 /* Add the giv to the class of givs computed from one biv. */
6955 bl = REG_IV_CLASS (ivs, REGNO (src_reg));
6956 if (bl)
6958 v->next_iv = bl->giv;
6959 bl->giv = v;
6960 /* Don't count DEST_ADDR. This is supposed to count the number of
6961 insns that calculate givs. */
6962 if (type == DEST_REG)
6963 bl->giv_count++;
6964 bl->total_benefit += benefit;
6966 else
6967 /* Fatal error, biv missing for this giv? */
6968 abort ();
6970 if (type == DEST_ADDR)
6972 v->replaceable = 1;
6973 v->not_replaceable = 0;
6975 else
6977 /* The giv can be replaced outright by the reduced register only if all
6978 of the following conditions are true:
6979 - the insn that sets the giv is always executed on any iteration
6980 on which the giv is used at all
6981 (there are two ways to deduce this:
6982 either the insn is executed on every iteration,
6983 or all uses follow that insn in the same basic block),
6984 - the giv is not used outside the loop
6985 - no assignments to the biv occur during the giv's lifetime. */
6987 if (REGNO_FIRST_UID (REGNO (dest_reg)) == INSN_UID (insn)
6988 /* Previous line always fails if INSN was moved by loop opt. */
6989 && REGNO_LAST_LUID (REGNO (dest_reg))
6990 < INSN_LUID (loop->end)
6991 && (! not_every_iteration
6992 || last_use_this_basic_block (dest_reg, insn)))
6994 /* Now check that there are no assignments to the biv within the
6995 giv's lifetime. This requires two separate checks. */
6997 /* Check each biv update, and fail if any are between the first
6998 and last use of the giv.
7000 If this loop contains an inner loop that was unrolled, then
7001 the insn modifying the biv may have been emitted by the loop
7002 unrolling code, and hence does not have a valid luid. Just
7003 mark the biv as not replaceable in this case. It is not very
7004 useful as a biv, because it is used in two different loops.
7005 It is very unlikely that we would be able to optimize the giv
7006 using this biv anyways. */
7008 v->replaceable = 1;
7009 v->not_replaceable = 0;
7010 for (b = bl->biv; b; b = b->next_iv)
7012 if (INSN_UID (b->insn) >= max_uid_for_loop
7013 || ((INSN_LUID (b->insn)
7014 >= REGNO_FIRST_LUID (REGNO (dest_reg)))
7015 && (INSN_LUID (b->insn)
7016 <= REGNO_LAST_LUID (REGNO (dest_reg)))))
7018 v->replaceable = 0;
7019 v->not_replaceable = 1;
7020 break;
7024 /* If there are any backwards branches that go from after the
7025 biv update to before it, then this giv is not replaceable. */
7026 if (v->replaceable)
7027 for (b = bl->biv; b; b = b->next_iv)
7028 if (back_branch_in_range_p (loop, b->insn))
7030 v->replaceable = 0;
7031 v->not_replaceable = 1;
7032 break;
7035 else
7037 /* May still be replaceable, we don't have enough info here to
7038 decide. */
7039 v->replaceable = 0;
7040 v->not_replaceable = 0;
7044 /* Record whether the add_val contains a const_int, for later use by
7045 combine_givs. */
7047 rtx tem = add_val;
7049 v->no_const_addval = 1;
7050 if (tem == const0_rtx)
7052 else if (CONSTANT_P (add_val))
7053 v->no_const_addval = 0;
7054 if (GET_CODE (tem) == PLUS)
7056 while (1)
7058 if (GET_CODE (XEXP (tem, 0)) == PLUS)
7059 tem = XEXP (tem, 0);
7060 else if (GET_CODE (XEXP (tem, 1)) == PLUS)
7061 tem = XEXP (tem, 1);
7062 else
7063 break;
7065 if (CONSTANT_P (XEXP (tem, 1)))
7066 v->no_const_addval = 0;
7070 if (loop_dump_stream)
7071 loop_giv_dump (v, loop_dump_stream, 0);
7074 /* Try to calculate the final value of the giv, the value it will have at
7075 the end of the loop. If we can do it, return that value. */
7077 static rtx
7078 final_giv_value (const struct loop *loop, struct induction *v)
7080 struct loop_ivs *ivs = LOOP_IVS (loop);
7081 struct iv_class *bl;
7082 rtx insn;
7083 rtx increment, tem;
7084 rtx seq;
7085 rtx loop_end = loop->end;
7086 unsigned HOST_WIDE_INT n_iterations = LOOP_INFO (loop)->n_iterations;
7088 bl = REG_IV_CLASS (ivs, REGNO (v->src_reg));
7090 /* The final value for givs which depend on reversed bivs must be calculated
7091 differently than for ordinary givs. In this case, there is already an
7092 insn after the loop which sets this giv's final value (if necessary),
7093 and there are no other loop exits, so we can return any value. */
7094 if (bl->reversed)
7096 if (loop_dump_stream)
7097 fprintf (loop_dump_stream,
7098 "Final giv value for %d, depends on reversed biv\n",
7099 REGNO (v->dest_reg));
7100 return const0_rtx;
7103 /* Try to calculate the final value as a function of the biv it depends
7104 upon. The only exit from the loop must be the fall through at the bottom
7105 and the insn that sets the giv must be executed on every iteration
7106 (otherwise the giv may not have its final value when the loop exits). */
7108 /* ??? Can calculate the final giv value by subtracting off the
7109 extra biv increments times the giv's mult_val. The loop must have
7110 only one exit for this to work, but the loop iterations does not need
7111 to be known. */
7113 if (n_iterations != 0
7114 && ! loop->exit_count
7115 && v->always_executed)
7117 /* ?? It is tempting to use the biv's value here since these insns will
7118 be put after the loop, and hence the biv will have its final value
7119 then. However, this fails if the biv is subsequently eliminated.
7120 Perhaps determine whether biv's are eliminable before trying to
7121 determine whether giv's are replaceable so that we can use the
7122 biv value here if it is not eliminable. */
7124 /* We are emitting code after the end of the loop, so we must make
7125 sure that bl->initial_value is still valid then. It will still
7126 be valid if it is invariant. */
7128 increment = biv_total_increment (bl);
7130 if (increment && loop_invariant_p (loop, increment)
7131 && loop_invariant_p (loop, bl->initial_value))
7133 /* Can calculate the loop exit value of its biv as
7134 (n_iterations * increment) + initial_value */
7136 /* The loop exit value of the giv is then
7137 (final_biv_value - extra increments) * mult_val + add_val.
7138 The extra increments are any increments to the biv which
7139 occur in the loop after the giv's value is calculated.
7140 We must search from the insn that sets the giv to the end
7141 of the loop to calculate this value. */
7143 /* Put the final biv value in tem. */
7144 tem = gen_reg_rtx (v->mode);
7145 record_base_value (REGNO (tem), bl->biv->add_val, 0);
7146 loop_iv_add_mult_sink (loop, extend_value_for_giv (v, increment),
7147 GEN_INT (n_iterations),
7148 extend_value_for_giv (v, bl->initial_value),
7149 tem);
7151 /* Subtract off extra increments as we find them. */
7152 for (insn = NEXT_INSN (v->insn); insn != loop_end;
7153 insn = NEXT_INSN (insn))
7155 struct induction *biv;
7157 for (biv = bl->biv; biv; biv = biv->next_iv)
7158 if (biv->insn == insn)
7160 start_sequence ();
7161 tem = expand_simple_binop (GET_MODE (tem), MINUS, tem,
7162 biv->add_val, NULL_RTX, 0,
7163 OPTAB_LIB_WIDEN);
7164 seq = get_insns ();
7165 end_sequence ();
7166 loop_insn_sink (loop, seq);
7170 /* Now calculate the giv's final value. */
7171 loop_iv_add_mult_sink (loop, tem, v->mult_val, v->add_val, tem);
7173 if (loop_dump_stream)
7174 fprintf (loop_dump_stream,
7175 "Final giv value for %d, calc from biv's value.\n",
7176 REGNO (v->dest_reg));
7178 return tem;
7182 /* Replaceable giv's should never reach here. */
7183 if (v->replaceable)
7184 abort ();
7186 /* Check to see if the biv is dead at all loop exits. */
7187 if (reg_dead_after_loop (loop, v->dest_reg))
7189 if (loop_dump_stream)
7190 fprintf (loop_dump_stream,
7191 "Final giv value for %d, giv dead after loop exit.\n",
7192 REGNO (v->dest_reg));
7194 return const0_rtx;
7197 return 0;
7200 /* All this does is determine whether a giv can be made replaceable because
7201 its final value can be calculated. This code can not be part of record_giv
7202 above, because final_giv_value requires that the number of loop iterations
7203 be known, and that can not be accurately calculated until after all givs
7204 have been identified. */
7206 static void
7207 check_final_value (const struct loop *loop, struct induction *v)
7209 rtx final_value = 0;
7211 /* DEST_ADDR givs will never reach here, because they are always marked
7212 replaceable above in record_giv. */
7214 /* The giv can be replaced outright by the reduced register only if all
7215 of the following conditions are true:
7216 - the insn that sets the giv is always executed on any iteration
7217 on which the giv is used at all
7218 (there are two ways to deduce this:
7219 either the insn is executed on every iteration,
7220 or all uses follow that insn in the same basic block),
7221 - its final value can be calculated (this condition is different
7222 than the one above in record_giv)
7223 - it's not used before the it's set
7224 - no assignments to the biv occur during the giv's lifetime. */
7226 #if 0
7227 /* This is only called now when replaceable is known to be false. */
7228 /* Clear replaceable, so that it won't confuse final_giv_value. */
7229 v->replaceable = 0;
7230 #endif
7232 if ((final_value = final_giv_value (loop, v))
7233 && (v->always_executed
7234 || last_use_this_basic_block (v->dest_reg, v->insn)))
7236 int biv_increment_seen = 0, before_giv_insn = 0;
7237 rtx p = v->insn;
7238 rtx last_giv_use;
7240 v->replaceable = 1;
7241 v->not_replaceable = 0;
7243 /* When trying to determine whether or not a biv increment occurs
7244 during the lifetime of the giv, we can ignore uses of the variable
7245 outside the loop because final_value is true. Hence we can not
7246 use regno_last_uid and regno_first_uid as above in record_giv. */
7248 /* Search the loop to determine whether any assignments to the
7249 biv occur during the giv's lifetime. Start with the insn
7250 that sets the giv, and search around the loop until we come
7251 back to that insn again.
7253 Also fail if there is a jump within the giv's lifetime that jumps
7254 to somewhere outside the lifetime but still within the loop. This
7255 catches spaghetti code where the execution order is not linear, and
7256 hence the above test fails. Here we assume that the giv lifetime
7257 does not extend from one iteration of the loop to the next, so as
7258 to make the test easier. Since the lifetime isn't known yet,
7259 this requires two loops. See also record_giv above. */
7261 last_giv_use = v->insn;
7263 while (1)
7265 p = NEXT_INSN (p);
7266 if (p == loop->end)
7268 before_giv_insn = 1;
7269 p = NEXT_INSN (loop->start);
7271 if (p == v->insn)
7272 break;
7274 if (INSN_P (p))
7276 /* It is possible for the BIV increment to use the GIV if we
7277 have a cycle. Thus we must be sure to check each insn for
7278 both BIV and GIV uses, and we must check for BIV uses
7279 first. */
7281 if (! biv_increment_seen
7282 && reg_set_p (v->src_reg, PATTERN (p)))
7283 biv_increment_seen = 1;
7285 if (reg_mentioned_p (v->dest_reg, PATTERN (p)))
7287 if (biv_increment_seen || before_giv_insn)
7289 v->replaceable = 0;
7290 v->not_replaceable = 1;
7291 break;
7293 last_giv_use = p;
7298 /* Now that the lifetime of the giv is known, check for branches
7299 from within the lifetime to outside the lifetime if it is still
7300 replaceable. */
7302 if (v->replaceable)
7304 p = v->insn;
7305 while (1)
7307 p = NEXT_INSN (p);
7308 if (p == loop->end)
7309 p = NEXT_INSN (loop->start);
7310 if (p == last_giv_use)
7311 break;
7313 if (JUMP_P (p) && JUMP_LABEL (p)
7314 && LABEL_NAME (JUMP_LABEL (p))
7315 && ((loop_insn_first_p (JUMP_LABEL (p), v->insn)
7316 && loop_insn_first_p (loop->start, JUMP_LABEL (p)))
7317 || (loop_insn_first_p (last_giv_use, JUMP_LABEL (p))
7318 && loop_insn_first_p (JUMP_LABEL (p), loop->end))))
7320 v->replaceable = 0;
7321 v->not_replaceable = 1;
7323 if (loop_dump_stream)
7324 fprintf (loop_dump_stream,
7325 "Found branch outside giv lifetime.\n");
7327 break;
7332 /* If it is replaceable, then save the final value. */
7333 if (v->replaceable)
7334 v->final_value = final_value;
7337 if (loop_dump_stream && v->replaceable)
7338 fprintf (loop_dump_stream, "Insn %d: giv reg %d final_value replaceable\n",
7339 INSN_UID (v->insn), REGNO (v->dest_reg));
7342 /* Update the status of whether a giv can derive other givs.
7344 We need to do something special if there is or may be an update to the biv
7345 between the time the giv is defined and the time it is used to derive
7346 another giv.
7348 In addition, a giv that is only conditionally set is not allowed to
7349 derive another giv once a label has been passed.
7351 The cases we look at are when a label or an update to a biv is passed. */
7353 static void
7354 update_giv_derive (const struct loop *loop, rtx p)
7356 struct loop_ivs *ivs = LOOP_IVS (loop);
7357 struct iv_class *bl;
7358 struct induction *biv, *giv;
7359 rtx tem;
7360 int dummy;
7362 /* Search all IV classes, then all bivs, and finally all givs.
7364 There are three cases we are concerned with. First we have the situation
7365 of a giv that is only updated conditionally. In that case, it may not
7366 derive any givs after a label is passed.
7368 The second case is when a biv update occurs, or may occur, after the
7369 definition of a giv. For certain biv updates (see below) that are
7370 known to occur between the giv definition and use, we can adjust the
7371 giv definition. For others, or when the biv update is conditional,
7372 we must prevent the giv from deriving any other givs. There are two
7373 sub-cases within this case.
7375 If this is a label, we are concerned with any biv update that is done
7376 conditionally, since it may be done after the giv is defined followed by
7377 a branch here (actually, we need to pass both a jump and a label, but
7378 this extra tracking doesn't seem worth it).
7380 If this is a jump, we are concerned about any biv update that may be
7381 executed multiple times. We are actually only concerned about
7382 backward jumps, but it is probably not worth performing the test
7383 on the jump again here.
7385 If this is a biv update, we must adjust the giv status to show that a
7386 subsequent biv update was performed. If this adjustment cannot be done,
7387 the giv cannot derive further givs. */
7389 for (bl = ivs->list; bl; bl = bl->next)
7390 for (biv = bl->biv; biv; biv = biv->next_iv)
7391 if (LABEL_P (p) || JUMP_P (p)
7392 || biv->insn == p)
7394 /* Skip if location is the same as a previous one. */
7395 if (biv->same)
7396 continue;
7398 for (giv = bl->giv; giv; giv = giv->next_iv)
7400 /* If cant_derive is already true, there is no point in
7401 checking all of these conditions again. */
7402 if (giv->cant_derive)
7403 continue;
7405 /* If this giv is conditionally set and we have passed a label,
7406 it cannot derive anything. */
7407 if (LABEL_P (p) && ! giv->always_computable)
7408 giv->cant_derive = 1;
7410 /* Skip givs that have mult_val == 0, since
7411 they are really invariants. Also skip those that are
7412 replaceable, since we know their lifetime doesn't contain
7413 any biv update. */
7414 else if (giv->mult_val == const0_rtx || giv->replaceable)
7415 continue;
7417 /* The only way we can allow this giv to derive another
7418 is if this is a biv increment and we can form the product
7419 of biv->add_val and giv->mult_val. In this case, we will
7420 be able to compute a compensation. */
7421 else if (biv->insn == p)
7423 rtx ext_val_dummy;
7425 tem = 0;
7426 if (biv->mult_val == const1_rtx)
7427 tem = simplify_giv_expr (loop,
7428 gen_rtx_MULT (giv->mode,
7429 biv->add_val,
7430 giv->mult_val),
7431 &ext_val_dummy, &dummy);
7433 if (tem && giv->derive_adjustment)
7434 tem = simplify_giv_expr
7435 (loop,
7436 gen_rtx_PLUS (giv->mode, tem, giv->derive_adjustment),
7437 &ext_val_dummy, &dummy);
7439 if (tem)
7440 giv->derive_adjustment = tem;
7441 else
7442 giv->cant_derive = 1;
7444 else if ((LABEL_P (p) && ! biv->always_computable)
7445 || (JUMP_P (p) && biv->maybe_multiple))
7446 giv->cant_derive = 1;
7451 /* Check whether an insn is an increment legitimate for a basic induction var.
7452 X is the source of insn P, or a part of it.
7453 MODE is the mode in which X should be interpreted.
7455 DEST_REG is the putative biv, also the destination of the insn.
7456 We accept patterns of these forms:
7457 REG = REG + INVARIANT (includes REG = REG - CONSTANT)
7458 REG = INVARIANT + REG
7460 If X is suitable, we return 1, set *MULT_VAL to CONST1_RTX,
7461 store the additive term into *INC_VAL, and store the place where
7462 we found the additive term into *LOCATION.
7464 If X is an assignment of an invariant into DEST_REG, we set
7465 *MULT_VAL to CONST0_RTX, and store the invariant into *INC_VAL.
7467 We also want to detect a BIV when it corresponds to a variable
7468 whose mode was promoted. In that case, an increment
7469 of the variable may be a PLUS that adds a SUBREG of that variable to
7470 an invariant and then sign- or zero-extends the result of the PLUS
7471 into the variable.
7473 Most GIVs in such cases will be in the promoted mode, since that is the
7474 probably the natural computation mode (and almost certainly the mode
7475 used for addresses) on the machine. So we view the pseudo-reg containing
7476 the variable as the BIV, as if it were simply incremented.
7478 Note that treating the entire pseudo as a BIV will result in making
7479 simple increments to any GIVs based on it. However, if the variable
7480 overflows in its declared mode but not its promoted mode, the result will
7481 be incorrect. This is acceptable if the variable is signed, since
7482 overflows in such cases are undefined, but not if it is unsigned, since
7483 those overflows are defined. So we only check for SIGN_EXTEND and
7484 not ZERO_EXTEND.
7486 If we cannot find a biv, we return 0. */
7488 static int
7489 basic_induction_var (const struct loop *loop, rtx x, enum machine_mode mode,
7490 rtx dest_reg, rtx p, rtx *inc_val, rtx *mult_val,
7491 rtx **location)
7493 enum rtx_code code;
7494 rtx *argp, arg;
7495 rtx insn, set = 0, last, inc;
7497 code = GET_CODE (x);
7498 *location = NULL;
7499 switch (code)
7501 case PLUS:
7502 if (rtx_equal_p (XEXP (x, 0), dest_reg)
7503 || (GET_CODE (XEXP (x, 0)) == SUBREG
7504 && SUBREG_PROMOTED_VAR_P (XEXP (x, 0))
7505 && SUBREG_REG (XEXP (x, 0)) == dest_reg))
7507 argp = &XEXP (x, 1);
7509 else if (rtx_equal_p (XEXP (x, 1), dest_reg)
7510 || (GET_CODE (XEXP (x, 1)) == SUBREG
7511 && SUBREG_PROMOTED_VAR_P (XEXP (x, 1))
7512 && SUBREG_REG (XEXP (x, 1)) == dest_reg))
7514 argp = &XEXP (x, 0);
7516 else
7517 return 0;
7519 arg = *argp;
7520 if (loop_invariant_p (loop, arg) != 1)
7521 return 0;
7523 /* convert_modes can emit new instructions, e.g. when arg is a loop
7524 invariant MEM and dest_reg has a different mode.
7525 These instructions would be emitted after the end of the function
7526 and then *inc_val would be an uninitialized pseudo.
7527 Detect this and bail in this case.
7528 Other alternatives to solve this can be introducing a convert_modes
7529 variant which is allowed to fail but not allowed to emit new
7530 instructions, emit these instructions before loop start and let
7531 it be garbage collected if *inc_val is never used or saving the
7532 *inc_val initialization sequence generated here and when *inc_val
7533 is going to be actually used, emit it at some suitable place. */
7534 last = get_last_insn ();
7535 inc = convert_modes (GET_MODE (dest_reg), GET_MODE (x), arg, 0);
7536 if (get_last_insn () != last)
7538 delete_insns_since (last);
7539 return 0;
7542 *inc_val = inc;
7543 *mult_val = const1_rtx;
7544 *location = argp;
7545 return 1;
7547 case SUBREG:
7548 /* If what's inside the SUBREG is a BIV, then the SUBREG. This will
7549 handle addition of promoted variables.
7550 ??? The comment at the start of this function is wrong: promoted
7551 variable increments don't look like it says they do. */
7552 return basic_induction_var (loop, SUBREG_REG (x),
7553 GET_MODE (SUBREG_REG (x)),
7554 dest_reg, p, inc_val, mult_val, location);
7556 case REG:
7557 /* If this register is assigned in a previous insn, look at its
7558 source, but don't go outside the loop or past a label. */
7560 /* If this sets a register to itself, we would repeat any previous
7561 biv increment if we applied this strategy blindly. */
7562 if (rtx_equal_p (dest_reg, x))
7563 return 0;
7565 insn = p;
7566 while (1)
7568 rtx dest;
7571 insn = PREV_INSN (insn);
7573 while (insn && NOTE_P (insn)
7574 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
7576 if (!insn)
7577 break;
7578 set = single_set (insn);
7579 if (set == 0)
7580 break;
7581 dest = SET_DEST (set);
7582 if (dest == x
7583 || (GET_CODE (dest) == SUBREG
7584 && (GET_MODE_SIZE (GET_MODE (dest)) <= UNITS_PER_WORD)
7585 && (GET_MODE_CLASS (GET_MODE (dest)) == MODE_INT)
7586 && SUBREG_REG (dest) == x))
7587 return basic_induction_var (loop, SET_SRC (set),
7588 (GET_MODE (SET_SRC (set)) == VOIDmode
7589 ? GET_MODE (x)
7590 : GET_MODE (SET_SRC (set))),
7591 dest_reg, insn,
7592 inc_val, mult_val, location);
7594 while (GET_CODE (dest) == SIGN_EXTRACT
7595 || GET_CODE (dest) == ZERO_EXTRACT
7596 || GET_CODE (dest) == SUBREG
7597 || GET_CODE (dest) == STRICT_LOW_PART)
7598 dest = XEXP (dest, 0);
7599 if (dest == x)
7600 break;
7602 /* Fall through. */
7604 /* Can accept constant setting of biv only when inside inner most loop.
7605 Otherwise, a biv of an inner loop may be incorrectly recognized
7606 as a biv of the outer loop,
7607 causing code to be moved INTO the inner loop. */
7608 case MEM:
7609 if (loop_invariant_p (loop, x) != 1)
7610 return 0;
7611 case CONST_INT:
7612 case SYMBOL_REF:
7613 case CONST:
7614 /* convert_modes aborts if we try to convert to or from CCmode, so just
7615 exclude that case. It is very unlikely that a condition code value
7616 would be a useful iterator anyways. convert_modes aborts if we try to
7617 convert a float mode to non-float or vice versa too. */
7618 if (loop->level == 1
7619 && GET_MODE_CLASS (mode) == GET_MODE_CLASS (GET_MODE (dest_reg))
7620 && GET_MODE_CLASS (mode) != MODE_CC)
7622 /* Possible bug here? Perhaps we don't know the mode of X. */
7623 last = get_last_insn ();
7624 inc = convert_modes (GET_MODE (dest_reg), mode, x, 0);
7625 if (get_last_insn () != last)
7627 delete_insns_since (last);
7628 return 0;
7631 *inc_val = inc;
7632 *mult_val = const0_rtx;
7633 return 1;
7635 else
7636 return 0;
7638 case SIGN_EXTEND:
7639 /* Ignore this BIV if signed arithmetic overflow is defined. */
7640 if (flag_wrapv)
7641 return 0;
7642 return basic_induction_var (loop, XEXP (x, 0), GET_MODE (XEXP (x, 0)),
7643 dest_reg, p, inc_val, mult_val, location);
7645 case ASHIFTRT:
7646 /* Similar, since this can be a sign extension. */
7647 for (insn = PREV_INSN (p);
7648 (insn && NOTE_P (insn)
7649 && NOTE_LINE_NUMBER (insn) != NOTE_INSN_LOOP_BEG);
7650 insn = PREV_INSN (insn))
7653 if (insn)
7654 set = single_set (insn);
7656 if (! rtx_equal_p (dest_reg, XEXP (x, 0))
7657 && set && SET_DEST (set) == XEXP (x, 0)
7658 && GET_CODE (XEXP (x, 1)) == CONST_INT
7659 && INTVAL (XEXP (x, 1)) >= 0
7660 && GET_CODE (SET_SRC (set)) == ASHIFT
7661 && XEXP (x, 1) == XEXP (SET_SRC (set), 1))
7662 return basic_induction_var (loop, XEXP (SET_SRC (set), 0),
7663 GET_MODE (XEXP (x, 0)),
7664 dest_reg, insn, inc_val, mult_val,
7665 location);
7666 return 0;
7668 default:
7669 return 0;
7673 /* A general induction variable (giv) is any quantity that is a linear
7674 function of a basic induction variable,
7675 i.e. giv = biv * mult_val + add_val.
7676 The coefficients can be any loop invariant quantity.
7677 A giv need not be computed directly from the biv;
7678 it can be computed by way of other givs. */
7680 /* Determine whether X computes a giv.
7681 If it does, return a nonzero value
7682 which is the benefit from eliminating the computation of X;
7683 set *SRC_REG to the register of the biv that it is computed from;
7684 set *ADD_VAL and *MULT_VAL to the coefficients,
7685 such that the value of X is biv * mult + add; */
7687 static int
7688 general_induction_var (const struct loop *loop, rtx x, rtx *src_reg,
7689 rtx *add_val, rtx *mult_val, rtx *ext_val,
7690 int is_addr, int *pbenefit,
7691 enum machine_mode addr_mode)
7693 struct loop_ivs *ivs = LOOP_IVS (loop);
7694 rtx orig_x = x;
7696 /* If this is an invariant, forget it, it isn't a giv. */
7697 if (loop_invariant_p (loop, x) == 1)
7698 return 0;
7700 *pbenefit = 0;
7701 *ext_val = NULL_RTX;
7702 x = simplify_giv_expr (loop, x, ext_val, pbenefit);
7703 if (x == 0)
7704 return 0;
7706 switch (GET_CODE (x))
7708 case USE:
7709 case CONST_INT:
7710 /* Since this is now an invariant and wasn't before, it must be a giv
7711 with MULT_VAL == 0. It doesn't matter which BIV we associate this
7712 with. */
7713 *src_reg = ivs->list->biv->dest_reg;
7714 *mult_val = const0_rtx;
7715 *add_val = x;
7716 break;
7718 case REG:
7719 /* This is equivalent to a BIV. */
7720 *src_reg = x;
7721 *mult_val = const1_rtx;
7722 *add_val = const0_rtx;
7723 break;
7725 case PLUS:
7726 /* Either (plus (biv) (invar)) or
7727 (plus (mult (biv) (invar_1)) (invar_2)). */
7728 if (GET_CODE (XEXP (x, 0)) == MULT)
7730 *src_reg = XEXP (XEXP (x, 0), 0);
7731 *mult_val = XEXP (XEXP (x, 0), 1);
7733 else
7735 *src_reg = XEXP (x, 0);
7736 *mult_val = const1_rtx;
7738 *add_val = XEXP (x, 1);
7739 break;
7741 case MULT:
7742 /* ADD_VAL is zero. */
7743 *src_reg = XEXP (x, 0);
7744 *mult_val = XEXP (x, 1);
7745 *add_val = const0_rtx;
7746 break;
7748 default:
7749 abort ();
7752 /* Remove any enclosing USE from ADD_VAL and MULT_VAL (there will be
7753 unless they are CONST_INT). */
7754 if (GET_CODE (*add_val) == USE)
7755 *add_val = XEXP (*add_val, 0);
7756 if (GET_CODE (*mult_val) == USE)
7757 *mult_val = XEXP (*mult_val, 0);
7759 if (is_addr)
7760 *pbenefit += address_cost (orig_x, addr_mode) - reg_address_cost;
7761 else
7762 *pbenefit += rtx_cost (orig_x, SET);
7764 /* Always return true if this is a giv so it will be detected as such,
7765 even if the benefit is zero or negative. This allows elimination
7766 of bivs that might otherwise not be eliminated. */
7767 return 1;
7770 /* Given an expression, X, try to form it as a linear function of a biv.
7771 We will canonicalize it to be of the form
7772 (plus (mult (BIV) (invar_1))
7773 (invar_2))
7774 with possible degeneracies.
7776 The invariant expressions must each be of a form that can be used as a
7777 machine operand. We surround then with a USE rtx (a hack, but localized
7778 and certainly unambiguous!) if not a CONST_INT for simplicity in this
7779 routine; it is the caller's responsibility to strip them.
7781 If no such canonicalization is possible (i.e., two biv's are used or an
7782 expression that is neither invariant nor a biv or giv), this routine
7783 returns 0.
7785 For a nonzero return, the result will have a code of CONST_INT, USE,
7786 REG (for a BIV), PLUS, or MULT. No other codes will occur.
7788 *BENEFIT will be incremented by the benefit of any sub-giv encountered. */
7790 static rtx sge_plus (enum machine_mode, rtx, rtx);
7791 static rtx sge_plus_constant (rtx, rtx);
7793 static rtx
7794 simplify_giv_expr (const struct loop *loop, rtx x, rtx *ext_val, int *benefit)
7796 struct loop_ivs *ivs = LOOP_IVS (loop);
7797 struct loop_regs *regs = LOOP_REGS (loop);
7798 enum machine_mode mode = GET_MODE (x);
7799 rtx arg0, arg1;
7800 rtx tem;
7802 /* If this is not an integer mode, or if we cannot do arithmetic in this
7803 mode, this can't be a giv. */
7804 if (mode != VOIDmode
7805 && (GET_MODE_CLASS (mode) != MODE_INT
7806 || GET_MODE_BITSIZE (mode) > HOST_BITS_PER_WIDE_INT))
7807 return NULL_RTX;
7809 switch (GET_CODE (x))
7811 case PLUS:
7812 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
7813 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
7814 if (arg0 == 0 || arg1 == 0)
7815 return NULL_RTX;
7817 /* Put constant last, CONST_INT last if both constant. */
7818 if ((GET_CODE (arg0) == USE
7819 || GET_CODE (arg0) == CONST_INT)
7820 && ! ((GET_CODE (arg0) == USE
7821 && GET_CODE (arg1) == USE)
7822 || GET_CODE (arg1) == CONST_INT))
7823 tem = arg0, arg0 = arg1, arg1 = tem;
7825 /* Handle addition of zero, then addition of an invariant. */
7826 if (arg1 == const0_rtx)
7827 return arg0;
7828 else if (GET_CODE (arg1) == CONST_INT || GET_CODE (arg1) == USE)
7829 switch (GET_CODE (arg0))
7831 case CONST_INT:
7832 case USE:
7833 /* Adding two invariants must result in an invariant, so enclose
7834 addition operation inside a USE and return it. */
7835 if (GET_CODE (arg0) == USE)
7836 arg0 = XEXP (arg0, 0);
7837 if (GET_CODE (arg1) == USE)
7838 arg1 = XEXP (arg1, 0);
7840 if (GET_CODE (arg0) == CONST_INT)
7841 tem = arg0, arg0 = arg1, arg1 = tem;
7842 if (GET_CODE (arg1) == CONST_INT)
7843 tem = sge_plus_constant (arg0, arg1);
7844 else
7845 tem = sge_plus (mode, arg0, arg1);
7847 if (GET_CODE (tem) != CONST_INT)
7848 tem = gen_rtx_USE (mode, tem);
7849 return tem;
7851 case REG:
7852 case MULT:
7853 /* biv + invar or mult + invar. Return sum. */
7854 return gen_rtx_PLUS (mode, arg0, arg1);
7856 case PLUS:
7857 /* (a + invar_1) + invar_2. Associate. */
7858 return
7859 simplify_giv_expr (loop,
7860 gen_rtx_PLUS (mode,
7861 XEXP (arg0, 0),
7862 gen_rtx_PLUS (mode,
7863 XEXP (arg0, 1),
7864 arg1)),
7865 ext_val, benefit);
7867 default:
7868 abort ();
7871 /* Each argument must be either REG, PLUS, or MULT. Convert REG to
7872 MULT to reduce cases. */
7873 if (REG_P (arg0))
7874 arg0 = gen_rtx_MULT (mode, arg0, const1_rtx);
7875 if (REG_P (arg1))
7876 arg1 = gen_rtx_MULT (mode, arg1, const1_rtx);
7878 /* Now have PLUS + PLUS, PLUS + MULT, MULT + PLUS, or MULT + MULT.
7879 Put a MULT first, leaving PLUS + PLUS, MULT + PLUS, or MULT + MULT.
7880 Recurse to associate the second PLUS. */
7881 if (GET_CODE (arg1) == MULT)
7882 tem = arg0, arg0 = arg1, arg1 = tem;
7884 if (GET_CODE (arg1) == PLUS)
7885 return
7886 simplify_giv_expr (loop,
7887 gen_rtx_PLUS (mode,
7888 gen_rtx_PLUS (mode, arg0,
7889 XEXP (arg1, 0)),
7890 XEXP (arg1, 1)),
7891 ext_val, benefit);
7893 /* Now must have MULT + MULT. Distribute if same biv, else not giv. */
7894 if (GET_CODE (arg0) != MULT || GET_CODE (arg1) != MULT)
7895 return NULL_RTX;
7897 if (!rtx_equal_p (arg0, arg1))
7898 return NULL_RTX;
7900 return simplify_giv_expr (loop,
7901 gen_rtx_MULT (mode,
7902 XEXP (arg0, 0),
7903 gen_rtx_PLUS (mode,
7904 XEXP (arg0, 1),
7905 XEXP (arg1, 1))),
7906 ext_val, benefit);
7908 case MINUS:
7909 /* Handle "a - b" as "a + b * (-1)". */
7910 return simplify_giv_expr (loop,
7911 gen_rtx_PLUS (mode,
7912 XEXP (x, 0),
7913 gen_rtx_MULT (mode,
7914 XEXP (x, 1),
7915 constm1_rtx)),
7916 ext_val, benefit);
7918 case MULT:
7919 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
7920 arg1 = simplify_giv_expr (loop, XEXP (x, 1), ext_val, benefit);
7921 if (arg0 == 0 || arg1 == 0)
7922 return NULL_RTX;
7924 /* Put constant last, CONST_INT last if both constant. */
7925 if ((GET_CODE (arg0) == USE || GET_CODE (arg0) == CONST_INT)
7926 && GET_CODE (arg1) != CONST_INT)
7927 tem = arg0, arg0 = arg1, arg1 = tem;
7929 /* If second argument is not now constant, not giv. */
7930 if (GET_CODE (arg1) != USE && GET_CODE (arg1) != CONST_INT)
7931 return NULL_RTX;
7933 /* Handle multiply by 0 or 1. */
7934 if (arg1 == const0_rtx)
7935 return const0_rtx;
7937 else if (arg1 == const1_rtx)
7938 return arg0;
7940 switch (GET_CODE (arg0))
7942 case REG:
7943 /* biv * invar. Done. */
7944 return gen_rtx_MULT (mode, arg0, arg1);
7946 case CONST_INT:
7947 /* Product of two constants. */
7948 return GEN_INT (INTVAL (arg0) * INTVAL (arg1));
7950 case USE:
7951 /* invar * invar is a giv, but attempt to simplify it somehow. */
7952 if (GET_CODE (arg1) != CONST_INT)
7953 return NULL_RTX;
7955 arg0 = XEXP (arg0, 0);
7956 if (GET_CODE (arg0) == MULT)
7958 /* (invar_0 * invar_1) * invar_2. Associate. */
7959 return simplify_giv_expr (loop,
7960 gen_rtx_MULT (mode,
7961 XEXP (arg0, 0),
7962 gen_rtx_MULT (mode,
7963 XEXP (arg0,
7965 arg1)),
7966 ext_val, benefit);
7968 /* Propagate the MULT expressions to the innermost nodes. */
7969 else if (GET_CODE (arg0) == PLUS)
7971 /* (invar_0 + invar_1) * invar_2. Distribute. */
7972 return simplify_giv_expr (loop,
7973 gen_rtx_PLUS (mode,
7974 gen_rtx_MULT (mode,
7975 XEXP (arg0,
7977 arg1),
7978 gen_rtx_MULT (mode,
7979 XEXP (arg0,
7981 arg1)),
7982 ext_val, benefit);
7984 return gen_rtx_USE (mode, gen_rtx_MULT (mode, arg0, arg1));
7986 case MULT:
7987 /* (a * invar_1) * invar_2. Associate. */
7988 return simplify_giv_expr (loop,
7989 gen_rtx_MULT (mode,
7990 XEXP (arg0, 0),
7991 gen_rtx_MULT (mode,
7992 XEXP (arg0, 1),
7993 arg1)),
7994 ext_val, benefit);
7996 case PLUS:
7997 /* (a + invar_1) * invar_2. Distribute. */
7998 return simplify_giv_expr (loop,
7999 gen_rtx_PLUS (mode,
8000 gen_rtx_MULT (mode,
8001 XEXP (arg0, 0),
8002 arg1),
8003 gen_rtx_MULT (mode,
8004 XEXP (arg0, 1),
8005 arg1)),
8006 ext_val, benefit);
8008 default:
8009 abort ();
8012 case ASHIFT:
8013 /* Shift by constant is multiply by power of two. */
8014 if (GET_CODE (XEXP (x, 1)) != CONST_INT)
8015 return 0;
8017 return
8018 simplify_giv_expr (loop,
8019 gen_rtx_MULT (mode,
8020 XEXP (x, 0),
8021 GEN_INT ((HOST_WIDE_INT) 1
8022 << INTVAL (XEXP (x, 1)))),
8023 ext_val, benefit);
8025 case NEG:
8026 /* "-a" is "a * (-1)" */
8027 return simplify_giv_expr (loop,
8028 gen_rtx_MULT (mode, XEXP (x, 0), constm1_rtx),
8029 ext_val, benefit);
8031 case NOT:
8032 /* "~a" is "-a - 1". Silly, but easy. */
8033 return simplify_giv_expr (loop,
8034 gen_rtx_MINUS (mode,
8035 gen_rtx_NEG (mode, XEXP (x, 0)),
8036 const1_rtx),
8037 ext_val, benefit);
8039 case USE:
8040 /* Already in proper form for invariant. */
8041 return x;
8043 case SIGN_EXTEND:
8044 case ZERO_EXTEND:
8045 case TRUNCATE:
8046 /* Conditionally recognize extensions of simple IVs. After we've
8047 computed loop traversal counts and verified the range of the
8048 source IV, we'll reevaluate this as a GIV. */
8049 if (*ext_val == NULL_RTX)
8051 arg0 = simplify_giv_expr (loop, XEXP (x, 0), ext_val, benefit);
8052 if (arg0 && *ext_val == NULL_RTX && REG_P (arg0))
8054 *ext_val = gen_rtx_fmt_e (GET_CODE (x), mode, arg0);
8055 return arg0;
8058 goto do_default;
8060 case REG:
8061 /* If this is a new register, we can't deal with it. */
8062 if (REGNO (x) >= max_reg_before_loop)
8063 return 0;
8065 /* Check for biv or giv. */
8066 switch (REG_IV_TYPE (ivs, REGNO (x)))
8068 case BASIC_INDUCT:
8069 return x;
8070 case GENERAL_INDUCT:
8072 struct induction *v = REG_IV_INFO (ivs, REGNO (x));
8074 /* Form expression from giv and add benefit. Ensure this giv
8075 can derive another and subtract any needed adjustment if so. */
8077 /* Increasing the benefit here is risky. The only case in which it
8078 is arguably correct is if this is the only use of V. In other
8079 cases, this will artificially inflate the benefit of the current
8080 giv, and lead to suboptimal code. Thus, it is disabled, since
8081 potentially not reducing an only marginally beneficial giv is
8082 less harmful than reducing many givs that are not really
8083 beneficial. */
8085 rtx single_use = regs->array[REGNO (x)].single_usage;
8086 if (single_use && single_use != const0_rtx)
8087 *benefit += v->benefit;
8090 if (v->cant_derive)
8091 return 0;
8093 tem = gen_rtx_PLUS (mode, gen_rtx_MULT (mode,
8094 v->src_reg, v->mult_val),
8095 v->add_val);
8097 if (v->derive_adjustment)
8098 tem = gen_rtx_MINUS (mode, tem, v->derive_adjustment);
8099 arg0 = simplify_giv_expr (loop, tem, ext_val, benefit);
8100 if (*ext_val)
8102 if (!v->ext_dependent)
8103 return arg0;
8105 else
8107 *ext_val = v->ext_dependent;
8108 return arg0;
8110 return 0;
8113 default:
8114 do_default:
8115 /* If it isn't an induction variable, and it is invariant, we
8116 may be able to simplify things further by looking through
8117 the bits we just moved outside the loop. */
8118 if (loop_invariant_p (loop, x) == 1)
8120 struct movable *m;
8121 struct loop_movables *movables = LOOP_MOVABLES (loop);
8123 for (m = movables->head; m; m = m->next)
8124 if (rtx_equal_p (x, m->set_dest))
8126 /* Ok, we found a match. Substitute and simplify. */
8128 /* If we match another movable, we must use that, as
8129 this one is going away. */
8130 if (m->match)
8131 return simplify_giv_expr (loop, m->match->set_dest,
8132 ext_val, benefit);
8134 /* If consec is nonzero, this is a member of a group of
8135 instructions that were moved together. We handle this
8136 case only to the point of seeking to the last insn and
8137 looking for a REG_EQUAL. Fail if we don't find one. */
8138 if (m->consec != 0)
8140 int i = m->consec;
8141 tem = m->insn;
8144 tem = NEXT_INSN (tem);
8146 while (--i > 0);
8148 tem = find_reg_note (tem, REG_EQUAL, NULL_RTX);
8149 if (tem)
8150 tem = XEXP (tem, 0);
8152 else
8154 tem = single_set (m->insn);
8155 if (tem)
8156 tem = SET_SRC (tem);
8159 if (tem)
8161 /* What we are most interested in is pointer
8162 arithmetic on invariants -- only take
8163 patterns we may be able to do something with. */
8164 if (GET_CODE (tem) == PLUS
8165 || GET_CODE (tem) == MULT
8166 || GET_CODE (tem) == ASHIFT
8167 || GET_CODE (tem) == CONST_INT
8168 || GET_CODE (tem) == SYMBOL_REF)
8170 tem = simplify_giv_expr (loop, tem, ext_val,
8171 benefit);
8172 if (tem)
8173 return tem;
8175 else if (GET_CODE (tem) == CONST
8176 && GET_CODE (XEXP (tem, 0)) == PLUS
8177 && GET_CODE (XEXP (XEXP (tem, 0), 0)) == SYMBOL_REF
8178 && GET_CODE (XEXP (XEXP (tem, 0), 1)) == CONST_INT)
8180 tem = simplify_giv_expr (loop, XEXP (tem, 0),
8181 ext_val, benefit);
8182 if (tem)
8183 return tem;
8186 break;
8189 break;
8192 /* Fall through to general case. */
8193 default:
8194 /* If invariant, return as USE (unless CONST_INT).
8195 Otherwise, not giv. */
8196 if (GET_CODE (x) == USE)
8197 x = XEXP (x, 0);
8199 if (loop_invariant_p (loop, x) == 1)
8201 if (GET_CODE (x) == CONST_INT)
8202 return x;
8203 if (GET_CODE (x) == CONST
8204 && GET_CODE (XEXP (x, 0)) == PLUS
8205 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
8206 && GET_CODE (XEXP (XEXP (x, 0), 1)) == CONST_INT)
8207 x = XEXP (x, 0);
8208 return gen_rtx_USE (mode, x);
8210 else
8211 return 0;
8215 /* This routine folds invariants such that there is only ever one
8216 CONST_INT in the summation. It is only used by simplify_giv_expr. */
8218 static rtx
8219 sge_plus_constant (rtx x, rtx c)
8221 if (GET_CODE (x) == CONST_INT)
8222 return GEN_INT (INTVAL (x) + INTVAL (c));
8223 else if (GET_CODE (x) != PLUS)
8224 return gen_rtx_PLUS (GET_MODE (x), x, c);
8225 else if (GET_CODE (XEXP (x, 1)) == CONST_INT)
8227 return gen_rtx_PLUS (GET_MODE (x), XEXP (x, 0),
8228 GEN_INT (INTVAL (XEXP (x, 1)) + INTVAL (c)));
8230 else if (GET_CODE (XEXP (x, 0)) == PLUS
8231 || GET_CODE (XEXP (x, 1)) != PLUS)
8233 return gen_rtx_PLUS (GET_MODE (x),
8234 sge_plus_constant (XEXP (x, 0), c), XEXP (x, 1));
8236 else
8238 return gen_rtx_PLUS (GET_MODE (x),
8239 sge_plus_constant (XEXP (x, 1), c), XEXP (x, 0));
8243 static rtx
8244 sge_plus (enum machine_mode mode, rtx x, rtx y)
8246 while (GET_CODE (y) == PLUS)
8248 rtx a = XEXP (y, 0);
8249 if (GET_CODE (a) == CONST_INT)
8250 x = sge_plus_constant (x, a);
8251 else
8252 x = gen_rtx_PLUS (mode, x, a);
8253 y = XEXP (y, 1);
8255 if (GET_CODE (y) == CONST_INT)
8256 x = sge_plus_constant (x, y);
8257 else
8258 x = gen_rtx_PLUS (mode, x, y);
8259 return x;
8262 /* Help detect a giv that is calculated by several consecutive insns;
8263 for example,
8264 giv = biv * M
8265 giv = giv + A
8266 The caller has already identified the first insn P as having a giv as dest;
8267 we check that all other insns that set the same register follow
8268 immediately after P, that they alter nothing else,
8269 and that the result of the last is still a giv.
8271 The value is 0 if the reg set in P is not really a giv.
8272 Otherwise, the value is the amount gained by eliminating
8273 all the consecutive insns that compute the value.
8275 FIRST_BENEFIT is the amount gained by eliminating the first insn, P.
8276 SRC_REG is the reg of the biv; DEST_REG is the reg of the giv.
8278 The coefficients of the ultimate giv value are stored in
8279 *MULT_VAL and *ADD_VAL. */
8281 static int
8282 consec_sets_giv (const struct loop *loop, int first_benefit, rtx p,
8283 rtx src_reg, rtx dest_reg, rtx *add_val, rtx *mult_val,
8284 rtx *ext_val, rtx *last_consec_insn)
8286 struct loop_ivs *ivs = LOOP_IVS (loop);
8287 struct loop_regs *regs = LOOP_REGS (loop);
8288 int count;
8289 enum rtx_code code;
8290 int benefit;
8291 rtx temp;
8292 rtx set;
8294 /* Indicate that this is a giv so that we can update the value produced in
8295 each insn of the multi-insn sequence.
8297 This induction structure will be used only by the call to
8298 general_induction_var below, so we can allocate it on our stack.
8299 If this is a giv, our caller will replace the induct var entry with
8300 a new induction structure. */
8301 struct induction *v;
8303 if (REG_IV_TYPE (ivs, REGNO (dest_reg)) != UNKNOWN_INDUCT)
8304 return 0;
8306 v = alloca (sizeof (struct induction));
8307 v->src_reg = src_reg;
8308 v->mult_val = *mult_val;
8309 v->add_val = *add_val;
8310 v->benefit = first_benefit;
8311 v->cant_derive = 0;
8312 v->derive_adjustment = 0;
8313 v->ext_dependent = NULL_RTX;
8315 REG_IV_TYPE (ivs, REGNO (dest_reg)) = GENERAL_INDUCT;
8316 REG_IV_INFO (ivs, REGNO (dest_reg)) = v;
8318 count = regs->array[REGNO (dest_reg)].n_times_set - 1;
8320 while (count > 0)
8322 p = NEXT_INSN (p);
8323 code = GET_CODE (p);
8325 /* If libcall, skip to end of call sequence. */
8326 if (code == INSN && (temp = find_reg_note (p, REG_LIBCALL, NULL_RTX)))
8327 p = XEXP (temp, 0);
8329 if (code == INSN
8330 && (set = single_set (p))
8331 && REG_P (SET_DEST (set))
8332 && SET_DEST (set) == dest_reg
8333 && (general_induction_var (loop, SET_SRC (set), &src_reg,
8334 add_val, mult_val, ext_val, 0,
8335 &benefit, VOIDmode)
8336 /* Giv created by equivalent expression. */
8337 || ((temp = find_reg_note (p, REG_EQUAL, NULL_RTX))
8338 && general_induction_var (loop, XEXP (temp, 0), &src_reg,
8339 add_val, mult_val, ext_val, 0,
8340 &benefit, VOIDmode)))
8341 && src_reg == v->src_reg)
8343 if (find_reg_note (p, REG_RETVAL, NULL_RTX))
8344 benefit += libcall_benefit (p);
8346 count--;
8347 v->mult_val = *mult_val;
8348 v->add_val = *add_val;
8349 v->benefit += benefit;
8351 else if (code != NOTE)
8353 /* Allow insns that set something other than this giv to a
8354 constant. Such insns are needed on machines which cannot
8355 include long constants and should not disqualify a giv. */
8356 if (code == INSN
8357 && (set = single_set (p))
8358 && SET_DEST (set) != dest_reg
8359 && CONSTANT_P (SET_SRC (set)))
8360 continue;
8362 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
8363 return 0;
8367 REG_IV_TYPE (ivs, REGNO (dest_reg)) = UNKNOWN_INDUCT;
8368 *last_consec_insn = p;
8369 return v->benefit;
8372 /* Return an rtx, if any, that expresses giv G2 as a function of the register
8373 represented by G1. If no such expression can be found, or it is clear that
8374 it cannot possibly be a valid address, 0 is returned.
8376 To perform the computation, we note that
8377 G1 = x * v + a and
8378 G2 = y * v + b
8379 where `v' is the biv.
8381 So G2 = (y/b) * G1 + (b - a*y/x).
8383 Note that MULT = y/x.
8385 Update: A and B are now allowed to be additive expressions such that
8386 B contains all variables in A. That is, computing B-A will not require
8387 subtracting variables. */
8389 static rtx
8390 express_from_1 (rtx a, rtx b, rtx mult)
8392 /* If MULT is zero, then A*MULT is zero, and our expression is B. */
8394 if (mult == const0_rtx)
8395 return b;
8397 /* If MULT is not 1, we cannot handle A with non-constants, since we
8398 would then be required to subtract multiples of the registers in A.
8399 This is theoretically possible, and may even apply to some Fortran
8400 constructs, but it is a lot of work and we do not attempt it here. */
8402 if (mult != const1_rtx && GET_CODE (a) != CONST_INT)
8403 return NULL_RTX;
8405 /* In general these structures are sorted top to bottom (down the PLUS
8406 chain), but not left to right across the PLUS. If B is a higher
8407 order giv than A, we can strip one level and recurse. If A is higher
8408 order, we'll eventually bail out, but won't know that until the end.
8409 If they are the same, we'll strip one level around this loop. */
8411 while (GET_CODE (a) == PLUS && GET_CODE (b) == PLUS)
8413 rtx ra, rb, oa, ob, tmp;
8415 ra = XEXP (a, 0), oa = XEXP (a, 1);
8416 if (GET_CODE (ra) == PLUS)
8417 tmp = ra, ra = oa, oa = tmp;
8419 rb = XEXP (b, 0), ob = XEXP (b, 1);
8420 if (GET_CODE (rb) == PLUS)
8421 tmp = rb, rb = ob, ob = tmp;
8423 if (rtx_equal_p (ra, rb))
8424 /* We matched: remove one reg completely. */
8425 a = oa, b = ob;
8426 else if (GET_CODE (ob) != PLUS && rtx_equal_p (ra, ob))
8427 /* An alternate match. */
8428 a = oa, b = rb;
8429 else if (GET_CODE (oa) != PLUS && rtx_equal_p (oa, rb))
8430 /* An alternate match. */
8431 a = ra, b = ob;
8432 else
8434 /* Indicates an extra register in B. Strip one level from B and
8435 recurse, hoping B was the higher order expression. */
8436 ob = express_from_1 (a, ob, mult);
8437 if (ob == NULL_RTX)
8438 return NULL_RTX;
8439 return gen_rtx_PLUS (GET_MODE (b), rb, ob);
8443 /* Here we are at the last level of A, go through the cases hoping to
8444 get rid of everything but a constant. */
8446 if (GET_CODE (a) == PLUS)
8448 rtx ra, oa;
8450 ra = XEXP (a, 0), oa = XEXP (a, 1);
8451 if (rtx_equal_p (oa, b))
8452 oa = ra;
8453 else if (!rtx_equal_p (ra, b))
8454 return NULL_RTX;
8456 if (GET_CODE (oa) != CONST_INT)
8457 return NULL_RTX;
8459 return GEN_INT (-INTVAL (oa) * INTVAL (mult));
8461 else if (GET_CODE (a) == CONST_INT)
8463 return plus_constant (b, -INTVAL (a) * INTVAL (mult));
8465 else if (CONSTANT_P (a))
8467 enum machine_mode mode_a = GET_MODE (a);
8468 enum machine_mode mode_b = GET_MODE (b);
8469 enum machine_mode mode = mode_b == VOIDmode ? mode_a : mode_b;
8470 return simplify_gen_binary (MINUS, mode, b, a);
8472 else if (GET_CODE (b) == PLUS)
8474 if (rtx_equal_p (a, XEXP (b, 0)))
8475 return XEXP (b, 1);
8476 else if (rtx_equal_p (a, XEXP (b, 1)))
8477 return XEXP (b, 0);
8478 else
8479 return NULL_RTX;
8481 else if (rtx_equal_p (a, b))
8482 return const0_rtx;
8484 return NULL_RTX;
8487 static rtx
8488 express_from (struct induction *g1, struct induction *g2)
8490 rtx mult, add;
8492 /* The value that G1 will be multiplied by must be a constant integer. Also,
8493 the only chance we have of getting a valid address is if b*c/a (see above
8494 for notation) is also an integer. */
8495 if (GET_CODE (g1->mult_val) == CONST_INT
8496 && GET_CODE (g2->mult_val) == CONST_INT)
8498 if (g1->mult_val == const0_rtx
8499 || (g1->mult_val == constm1_rtx
8500 && INTVAL (g2->mult_val)
8501 == (HOST_WIDE_INT) 1 << (HOST_BITS_PER_WIDE_INT - 1))
8502 || INTVAL (g2->mult_val) % INTVAL (g1->mult_val) != 0)
8503 return NULL_RTX;
8504 mult = GEN_INT (INTVAL (g2->mult_val) / INTVAL (g1->mult_val));
8506 else if (rtx_equal_p (g1->mult_val, g2->mult_val))
8507 mult = const1_rtx;
8508 else
8510 /* ??? Find out if the one is a multiple of the other? */
8511 return NULL_RTX;
8514 add = express_from_1 (g1->add_val, g2->add_val, mult);
8515 if (add == NULL_RTX)
8517 /* Failed. If we've got a multiplication factor between G1 and G2,
8518 scale G1's addend and try again. */
8519 if (INTVAL (mult) > 1)
8521 rtx g1_add_val = g1->add_val;
8522 if (GET_CODE (g1_add_val) == MULT
8523 && GET_CODE (XEXP (g1_add_val, 1)) == CONST_INT)
8525 HOST_WIDE_INT m;
8526 m = INTVAL (mult) * INTVAL (XEXP (g1_add_val, 1));
8527 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val),
8528 XEXP (g1_add_val, 0), GEN_INT (m));
8530 else
8532 g1_add_val = gen_rtx_MULT (GET_MODE (g1_add_val), g1_add_val,
8533 mult);
8536 add = express_from_1 (g1_add_val, g2->add_val, const1_rtx);
8539 if (add == NULL_RTX)
8540 return NULL_RTX;
8542 /* Form simplified final result. */
8543 if (mult == const0_rtx)
8544 return add;
8545 else if (mult == const1_rtx)
8546 mult = g1->dest_reg;
8547 else
8548 mult = gen_rtx_MULT (g2->mode, g1->dest_reg, mult);
8550 if (add == const0_rtx)
8551 return mult;
8552 else
8554 if (GET_CODE (add) == PLUS
8555 && CONSTANT_P (XEXP (add, 1)))
8557 rtx tem = XEXP (add, 1);
8558 mult = gen_rtx_PLUS (g2->mode, mult, XEXP (add, 0));
8559 add = tem;
8562 return gen_rtx_PLUS (g2->mode, mult, add);
8566 /* Return an rtx, if any, that expresses giv G2 as a function of the register
8567 represented by G1. This indicates that G2 should be combined with G1 and
8568 that G2 can use (either directly or via an address expression) a register
8569 used to represent G1. */
8571 static rtx
8572 combine_givs_p (struct induction *g1, struct induction *g2)
8574 rtx comb, ret;
8576 /* With the introduction of ext dependent givs, we must care for modes.
8577 G2 must not use a wider mode than G1. */
8578 if (GET_MODE_SIZE (g1->mode) < GET_MODE_SIZE (g2->mode))
8579 return NULL_RTX;
8581 ret = comb = express_from (g1, g2);
8582 if (comb == NULL_RTX)
8583 return NULL_RTX;
8584 if (g1->mode != g2->mode)
8585 ret = gen_lowpart (g2->mode, comb);
8587 /* If these givs are identical, they can be combined. We use the results
8588 of express_from because the addends are not in a canonical form, so
8589 rtx_equal_p is a weaker test. */
8590 /* But don't combine a DEST_REG giv with a DEST_ADDR giv; we want the
8591 combination to be the other way round. */
8592 if (comb == g1->dest_reg
8593 && (g1->giv_type == DEST_REG || g2->giv_type == DEST_ADDR))
8595 return ret;
8598 /* If G2 can be expressed as a function of G1 and that function is valid
8599 as an address and no more expensive than using a register for G2,
8600 the expression of G2 in terms of G1 can be used. */
8601 if (ret != NULL_RTX
8602 && g2->giv_type == DEST_ADDR
8603 && memory_address_p (GET_MODE (g2->mem), ret))
8604 return ret;
8606 return NULL_RTX;
8609 /* Check each extension dependent giv in this class to see if its
8610 root biv is safe from wrapping in the interior mode, which would
8611 make the giv illegal. */
8613 static void
8614 check_ext_dependent_givs (const struct loop *loop, struct iv_class *bl)
8616 struct loop_info *loop_info = LOOP_INFO (loop);
8617 int ze_ok = 0, se_ok = 0, info_ok = 0;
8618 enum machine_mode biv_mode = GET_MODE (bl->biv->src_reg);
8619 HOST_WIDE_INT start_val;
8620 unsigned HOST_WIDE_INT u_end_val = 0;
8621 unsigned HOST_WIDE_INT u_start_val = 0;
8622 rtx incr = pc_rtx;
8623 struct induction *v;
8625 /* Make sure the iteration data is available. We must have
8626 constants in order to be certain of no overflow. */
8627 if (loop_info->n_iterations > 0
8628 && bl->initial_value
8629 && GET_CODE (bl->initial_value) == CONST_INT
8630 && (incr = biv_total_increment (bl))
8631 && GET_CODE (incr) == CONST_INT
8632 /* Make sure the host can represent the arithmetic. */
8633 && HOST_BITS_PER_WIDE_INT >= GET_MODE_BITSIZE (biv_mode))
8635 unsigned HOST_WIDE_INT abs_incr, total_incr;
8636 HOST_WIDE_INT s_end_val;
8637 int neg_incr;
8639 info_ok = 1;
8640 start_val = INTVAL (bl->initial_value);
8641 u_start_val = start_val;
8643 neg_incr = 0, abs_incr = INTVAL (incr);
8644 if (INTVAL (incr) < 0)
8645 neg_incr = 1, abs_incr = -abs_incr;
8646 total_incr = abs_incr * loop_info->n_iterations;
8648 /* Check for host arithmetic overflow. */
8649 if (total_incr / loop_info->n_iterations == abs_incr)
8651 unsigned HOST_WIDE_INT u_max;
8652 HOST_WIDE_INT s_max;
8654 u_end_val = start_val + (neg_incr ? -total_incr : total_incr);
8655 s_end_val = u_end_val;
8656 u_max = GET_MODE_MASK (biv_mode);
8657 s_max = u_max >> 1;
8659 /* Check zero extension of biv ok. */
8660 if (start_val >= 0
8661 /* Check for host arithmetic overflow. */
8662 && (neg_incr
8663 ? u_end_val < u_start_val
8664 : u_end_val > u_start_val)
8665 /* Check for target arithmetic overflow. */
8666 && (neg_incr
8667 ? 1 /* taken care of with host overflow */
8668 : u_end_val <= u_max))
8670 ze_ok = 1;
8673 /* Check sign extension of biv ok. */
8674 /* ??? While it is true that overflow with signed and pointer
8675 arithmetic is undefined, I fear too many programmers don't
8676 keep this fact in mind -- myself included on occasion.
8677 So leave alone with the signed overflow optimizations. */
8678 if (start_val >= -s_max - 1
8679 /* Check for host arithmetic overflow. */
8680 && (neg_incr
8681 ? s_end_val < start_val
8682 : s_end_val > start_val)
8683 /* Check for target arithmetic overflow. */
8684 && (neg_incr
8685 ? s_end_val >= -s_max - 1
8686 : s_end_val <= s_max))
8688 se_ok = 1;
8693 /* If we know the BIV is compared at run-time against an
8694 invariant value, and the increment is +/- 1, we may also
8695 be able to prove that the BIV cannot overflow. */
8696 else if (bl->biv->src_reg == loop_info->iteration_var
8697 && loop_info->comparison_value
8698 && loop_invariant_p (loop, loop_info->comparison_value)
8699 && (incr = biv_total_increment (bl))
8700 && GET_CODE (incr) == CONST_INT)
8702 /* If the increment is +1, and the exit test is a <,
8703 the BIV cannot overflow. (For <=, we have the
8704 problematic case that the comparison value might
8705 be the maximum value of the range.) */
8706 if (INTVAL (incr) == 1)
8708 if (loop_info->comparison_code == LT)
8709 se_ok = ze_ok = 1;
8710 else if (loop_info->comparison_code == LTU)
8711 ze_ok = 1;
8714 /* Likewise for increment -1 and exit test >. */
8715 if (INTVAL (incr) == -1)
8717 if (loop_info->comparison_code == GT)
8718 se_ok = ze_ok = 1;
8719 else if (loop_info->comparison_code == GTU)
8720 ze_ok = 1;
8724 /* Invalidate givs that fail the tests. */
8725 for (v = bl->giv; v; v = v->next_iv)
8726 if (v->ext_dependent)
8728 enum rtx_code code = GET_CODE (v->ext_dependent);
8729 int ok = 0;
8731 switch (code)
8733 case SIGN_EXTEND:
8734 ok = se_ok;
8735 break;
8736 case ZERO_EXTEND:
8737 ok = ze_ok;
8738 break;
8740 case TRUNCATE:
8741 /* We don't know whether this value is being used as either
8742 signed or unsigned, so to safely truncate we must satisfy
8743 both. The initial check here verifies the BIV itself;
8744 once that is successful we may check its range wrt the
8745 derived GIV. This works only if we were able to determine
8746 constant start and end values above. */
8747 if (se_ok && ze_ok && info_ok)
8749 enum machine_mode outer_mode = GET_MODE (v->ext_dependent);
8750 unsigned HOST_WIDE_INT max = GET_MODE_MASK (outer_mode) >> 1;
8752 /* We know from the above that both endpoints are nonnegative,
8753 and that there is no wrapping. Verify that both endpoints
8754 are within the (signed) range of the outer mode. */
8755 if (u_start_val <= max && u_end_val <= max)
8756 ok = 1;
8758 break;
8760 default:
8761 abort ();
8764 if (ok)
8766 if (loop_dump_stream)
8768 fprintf (loop_dump_stream,
8769 "Verified ext dependent giv at %d of reg %d\n",
8770 INSN_UID (v->insn), bl->regno);
8773 else
8775 if (loop_dump_stream)
8777 const char *why;
8779 if (info_ok)
8780 why = "biv iteration values overflowed";
8781 else
8783 if (incr == pc_rtx)
8784 incr = biv_total_increment (bl);
8785 if (incr == const1_rtx)
8786 why = "biv iteration info incomplete; incr by 1";
8787 else
8788 why = "biv iteration info incomplete";
8791 fprintf (loop_dump_stream,
8792 "Failed ext dependent giv at %d, %s\n",
8793 INSN_UID (v->insn), why);
8795 v->ignore = 1;
8796 bl->all_reduced = 0;
8801 /* Generate a version of VALUE in a mode appropriate for initializing V. */
8803 static rtx
8804 extend_value_for_giv (struct induction *v, rtx value)
8806 rtx ext_dep = v->ext_dependent;
8808 if (! ext_dep)
8809 return value;
8811 /* Recall that check_ext_dependent_givs verified that the known bounds
8812 of a biv did not overflow or wrap with respect to the extension for
8813 the giv. Therefore, constants need no additional adjustment. */
8814 if (CONSTANT_P (value) && GET_MODE (value) == VOIDmode)
8815 return value;
8817 /* Otherwise, we must adjust the value to compensate for the
8818 differing modes of the biv and the giv. */
8819 return gen_rtx_fmt_e (GET_CODE (ext_dep), GET_MODE (ext_dep), value);
8822 struct combine_givs_stats
8824 int giv_number;
8825 int total_benefit;
8828 static int
8829 cmp_combine_givs_stats (const void *xp, const void *yp)
8831 const struct combine_givs_stats * const x =
8832 (const struct combine_givs_stats *) xp;
8833 const struct combine_givs_stats * const y =
8834 (const struct combine_givs_stats *) yp;
8835 int d;
8836 d = y->total_benefit - x->total_benefit;
8837 /* Stabilize the sort. */
8838 if (!d)
8839 d = x->giv_number - y->giv_number;
8840 return d;
8843 /* Check all pairs of givs for iv_class BL and see if any can be combined with
8844 any other. If so, point SAME to the giv combined with and set NEW_REG to
8845 be an expression (in terms of the other giv's DEST_REG) equivalent to the
8846 giv. Also, update BENEFIT and related fields for cost/benefit analysis. */
8848 static void
8849 combine_givs (struct loop_regs *regs, struct iv_class *bl)
8851 /* Additional benefit to add for being combined multiple times. */
8852 const int extra_benefit = 3;
8854 struct induction *g1, *g2, **giv_array;
8855 int i, j, k, giv_count;
8856 struct combine_givs_stats *stats;
8857 rtx *can_combine;
8859 /* Count givs, because bl->giv_count is incorrect here. */
8860 giv_count = 0;
8861 for (g1 = bl->giv; g1; g1 = g1->next_iv)
8862 if (!g1->ignore)
8863 giv_count++;
8865 giv_array = alloca (giv_count * sizeof (struct induction *));
8866 i = 0;
8867 for (g1 = bl->giv; g1; g1 = g1->next_iv)
8868 if (!g1->ignore)
8869 giv_array[i++] = g1;
8871 stats = xcalloc (giv_count, sizeof (*stats));
8872 can_combine = xcalloc (giv_count, giv_count * sizeof (rtx));
8874 for (i = 0; i < giv_count; i++)
8876 int this_benefit;
8877 rtx single_use;
8879 g1 = giv_array[i];
8880 stats[i].giv_number = i;
8882 /* If a DEST_REG GIV is used only once, do not allow it to combine
8883 with anything, for in doing so we will gain nothing that cannot
8884 be had by simply letting the GIV with which we would have combined
8885 to be reduced on its own. The losage shows up in particular with
8886 DEST_ADDR targets on hosts with reg+reg addressing, though it can
8887 be seen elsewhere as well. */
8888 if (g1->giv_type == DEST_REG
8889 && (single_use = regs->array[REGNO (g1->dest_reg)].single_usage)
8890 && single_use != const0_rtx)
8891 continue;
8893 this_benefit = g1->benefit;
8894 /* Add an additional weight for zero addends. */
8895 if (g1->no_const_addval)
8896 this_benefit += 1;
8898 for (j = 0; j < giv_count; j++)
8900 rtx this_combine;
8902 g2 = giv_array[j];
8903 if (g1 != g2
8904 && (this_combine = combine_givs_p (g1, g2)) != NULL_RTX)
8906 can_combine[i * giv_count + j] = this_combine;
8907 this_benefit += g2->benefit + extra_benefit;
8910 stats[i].total_benefit = this_benefit;
8913 /* Iterate, combining until we can't. */
8914 restart:
8915 qsort (stats, giv_count, sizeof (*stats), cmp_combine_givs_stats);
8917 if (loop_dump_stream)
8919 fprintf (loop_dump_stream, "Sorted combine statistics:\n");
8920 for (k = 0; k < giv_count; k++)
8922 g1 = giv_array[stats[k].giv_number];
8923 if (!g1->combined_with && !g1->same)
8924 fprintf (loop_dump_stream, " {%d, %d}",
8925 INSN_UID (giv_array[stats[k].giv_number]->insn),
8926 stats[k].total_benefit);
8928 putc ('\n', loop_dump_stream);
8931 for (k = 0; k < giv_count; k++)
8933 int g1_add_benefit = 0;
8935 i = stats[k].giv_number;
8936 g1 = giv_array[i];
8938 /* If it has already been combined, skip. */
8939 if (g1->combined_with || g1->same)
8940 continue;
8942 for (j = 0; j < giv_count; j++)
8944 g2 = giv_array[j];
8945 if (g1 != g2 && can_combine[i * giv_count + j]
8946 /* If it has already been combined, skip. */
8947 && ! g2->same && ! g2->combined_with)
8949 int l;
8951 g2->new_reg = can_combine[i * giv_count + j];
8952 g2->same = g1;
8953 /* For destination, we now may replace by mem expression instead
8954 of register. This changes the costs considerably, so add the
8955 compensation. */
8956 if (g2->giv_type == DEST_ADDR)
8957 g2->benefit = (g2->benefit + reg_address_cost
8958 - address_cost (g2->new_reg,
8959 GET_MODE (g2->mem)));
8960 g1->combined_with++;
8961 g1->lifetime += g2->lifetime;
8963 g1_add_benefit += g2->benefit;
8965 /* ??? The new final_[bg]iv_value code does a much better job
8966 of finding replaceable giv's, and hence this code may no
8967 longer be necessary. */
8968 if (! g2->replaceable && REG_USERVAR_P (g2->dest_reg))
8969 g1_add_benefit -= copy_cost;
8971 /* To help optimize the next set of combinations, remove
8972 this giv from the benefits of other potential mates. */
8973 for (l = 0; l < giv_count; ++l)
8975 int m = stats[l].giv_number;
8976 if (can_combine[m * giv_count + j])
8977 stats[l].total_benefit -= g2->benefit + extra_benefit;
8980 if (loop_dump_stream)
8981 fprintf (loop_dump_stream,
8982 "giv at %d combined with giv at %d; new benefit %d + %d, lifetime %d\n",
8983 INSN_UID (g2->insn), INSN_UID (g1->insn),
8984 g1->benefit, g1_add_benefit, g1->lifetime);
8988 /* To help optimize the next set of combinations, remove
8989 this giv from the benefits of other potential mates. */
8990 if (g1->combined_with)
8992 for (j = 0; j < giv_count; ++j)
8994 int m = stats[j].giv_number;
8995 if (can_combine[m * giv_count + i])
8996 stats[j].total_benefit -= g1->benefit + extra_benefit;
8999 g1->benefit += g1_add_benefit;
9001 /* We've finished with this giv, and everything it touched.
9002 Restart the combination so that proper weights for the
9003 rest of the givs are properly taken into account. */
9004 /* ??? Ideally we would compact the arrays at this point, so
9005 as to not cover old ground. But sanely compacting
9006 can_combine is tricky. */
9007 goto restart;
9011 /* Clean up. */
9012 free (stats);
9013 free (can_combine);
9016 /* Generate sequence for REG = B * M + A. B is the initial value of
9017 the basic induction variable, M a multiplicative constant, A an
9018 additive constant and REG the destination register. */
9020 static rtx
9021 gen_add_mult (rtx b, rtx m, rtx a, rtx reg)
9023 rtx seq;
9024 rtx result;
9026 start_sequence ();
9027 /* Use unsigned arithmetic. */
9028 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
9029 if (reg != result)
9030 emit_move_insn (reg, result);
9031 seq = get_insns ();
9032 end_sequence ();
9034 return seq;
9038 /* Update registers created in insn sequence SEQ. */
9040 static void
9041 loop_regs_update (const struct loop *loop ATTRIBUTE_UNUSED, rtx seq)
9043 rtx insn;
9045 /* Update register info for alias analysis. */
9047 insn = seq;
9048 while (insn != NULL_RTX)
9050 rtx set = single_set (insn);
9052 if (set && REG_P (SET_DEST (set)))
9053 record_base_value (REGNO (SET_DEST (set)), SET_SRC (set), 0);
9055 insn = NEXT_INSN (insn);
9060 /* EMIT code before BEFORE_BB/BEFORE_INSN to set REG = B * M + A. B
9061 is the initial value of the basic induction variable, M a
9062 multiplicative constant, A an additive constant and REG the
9063 destination register. */
9065 static void
9066 loop_iv_add_mult_emit_before (const struct loop *loop, rtx b, rtx m, rtx a,
9067 rtx reg, basic_block before_bb, rtx before_insn)
9069 rtx seq;
9071 if (! before_insn)
9073 loop_iv_add_mult_hoist (loop, b, m, a, reg);
9074 return;
9077 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
9078 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
9080 /* Increase the lifetime of any invariants moved further in code. */
9081 update_reg_last_use (a, before_insn);
9082 update_reg_last_use (b, before_insn);
9083 update_reg_last_use (m, before_insn);
9085 /* It is possible that the expansion created lots of new registers.
9086 Iterate over the sequence we just created and record them all. We
9087 must do this before inserting the sequence. */
9088 loop_regs_update (loop, seq);
9090 loop_insn_emit_before (loop, before_bb, before_insn, seq);
9094 /* Emit insns in loop pre-header to set REG = B * M + A. B is the
9095 initial value of the basic induction variable, M a multiplicative
9096 constant, A an additive constant and REG the destination
9097 register. */
9099 static void
9100 loop_iv_add_mult_sink (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
9102 rtx seq;
9104 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
9105 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
9107 /* Increase the lifetime of any invariants moved further in code.
9108 ???? Is this really necessary? */
9109 update_reg_last_use (a, loop->sink);
9110 update_reg_last_use (b, loop->sink);
9111 update_reg_last_use (m, loop->sink);
9113 /* It is possible that the expansion created lots of new registers.
9114 Iterate over the sequence we just created and record them all. We
9115 must do this before inserting the sequence. */
9116 loop_regs_update (loop, seq);
9118 loop_insn_sink (loop, seq);
9122 /* Emit insns after loop to set REG = B * M + A. B is the initial
9123 value of the basic induction variable, M a multiplicative constant,
9124 A an additive constant and REG the destination register. */
9126 static void
9127 loop_iv_add_mult_hoist (const struct loop *loop, rtx b, rtx m, rtx a, rtx reg)
9129 rtx seq;
9131 /* Use copy_rtx to prevent unexpected sharing of these rtx. */
9132 seq = gen_add_mult (copy_rtx (b), copy_rtx (m), copy_rtx (a), reg);
9134 /* It is possible that the expansion created lots of new registers.
9135 Iterate over the sequence we just created and record them all. We
9136 must do this before inserting the sequence. */
9137 loop_regs_update (loop, seq);
9139 loop_insn_hoist (loop, seq);
9144 /* Similar to gen_add_mult, but compute cost rather than generating
9145 sequence. */
9147 static int
9148 iv_add_mult_cost (rtx b, rtx m, rtx a, rtx reg)
9150 int cost = 0;
9151 rtx last, result;
9153 start_sequence ();
9154 result = expand_mult_add (b, reg, m, a, GET_MODE (reg), 1);
9155 if (reg != result)
9156 emit_move_insn (reg, result);
9157 last = get_last_insn ();
9158 while (last)
9160 rtx t = single_set (last);
9161 if (t)
9162 cost += rtx_cost (SET_SRC (t), SET);
9163 last = PREV_INSN (last);
9165 end_sequence ();
9166 return cost;
9169 /* Test whether A * B can be computed without
9170 an actual multiply insn. Value is 1 if so.
9172 ??? This function stinks because it generates a ton of wasted RTL
9173 ??? and as a result fragments GC memory to no end. There are other
9174 ??? places in the compiler which are invoked a lot and do the same
9175 ??? thing, generate wasted RTL just to see if something is possible. */
9177 static int
9178 product_cheap_p (rtx a, rtx b)
9180 rtx tmp;
9181 int win, n_insns;
9183 /* If only one is constant, make it B. */
9184 if (GET_CODE (a) == CONST_INT)
9185 tmp = a, a = b, b = tmp;
9187 /* If first constant, both constant, so don't need multiply. */
9188 if (GET_CODE (a) == CONST_INT)
9189 return 1;
9191 /* If second not constant, neither is constant, so would need multiply. */
9192 if (GET_CODE (b) != CONST_INT)
9193 return 0;
9195 /* One operand is constant, so might not need multiply insn. Generate the
9196 code for the multiply and see if a call or multiply, or long sequence
9197 of insns is generated. */
9199 start_sequence ();
9200 expand_mult (GET_MODE (a), a, b, NULL_RTX, 1);
9201 tmp = get_insns ();
9202 end_sequence ();
9204 win = 1;
9205 if (INSN_P (tmp))
9207 n_insns = 0;
9208 while (tmp != NULL_RTX)
9210 rtx next = NEXT_INSN (tmp);
9212 if (++n_insns > 3
9213 || !NONJUMP_INSN_P (tmp)
9214 || (GET_CODE (PATTERN (tmp)) == SET
9215 && GET_CODE (SET_SRC (PATTERN (tmp))) == MULT)
9216 || (GET_CODE (PATTERN (tmp)) == PARALLEL
9217 && GET_CODE (XVECEXP (PATTERN (tmp), 0, 0)) == SET
9218 && GET_CODE (SET_SRC (XVECEXP (PATTERN (tmp), 0, 0))) == MULT))
9220 win = 0;
9221 break;
9224 tmp = next;
9227 else if (GET_CODE (tmp) == SET
9228 && GET_CODE (SET_SRC (tmp)) == MULT)
9229 win = 0;
9230 else if (GET_CODE (tmp) == PARALLEL
9231 && GET_CODE (XVECEXP (tmp, 0, 0)) == SET
9232 && GET_CODE (SET_SRC (XVECEXP (tmp, 0, 0))) == MULT)
9233 win = 0;
9235 return win;
9238 /* Check to see if loop can be terminated by a "decrement and branch until
9239 zero" instruction. If so, add a REG_NONNEG note to the branch insn if so.
9240 Also try reversing an increment loop to a decrement loop
9241 to see if the optimization can be performed.
9242 Value is nonzero if optimization was performed. */
9244 /* This is useful even if the architecture doesn't have such an insn,
9245 because it might change a loops which increments from 0 to n to a loop
9246 which decrements from n to 0. A loop that decrements to zero is usually
9247 faster than one that increments from zero. */
9249 /* ??? This could be rewritten to use some of the loop unrolling procedures,
9250 such as approx_final_value, biv_total_increment, loop_iterations, and
9251 final_[bg]iv_value. */
9253 static int
9254 check_dbra_loop (struct loop *loop, int insn_count)
9256 struct loop_info *loop_info = LOOP_INFO (loop);
9257 struct loop_regs *regs = LOOP_REGS (loop);
9258 struct loop_ivs *ivs = LOOP_IVS (loop);
9259 struct iv_class *bl;
9260 rtx reg;
9261 enum machine_mode mode;
9262 rtx jump_label;
9263 rtx final_value;
9264 rtx start_value;
9265 rtx new_add_val;
9266 rtx comparison;
9267 rtx before_comparison;
9268 rtx p;
9269 rtx jump;
9270 rtx first_compare;
9271 int compare_and_branch;
9272 rtx loop_start = loop->start;
9273 rtx loop_end = loop->end;
9275 /* If last insn is a conditional branch, and the insn before tests a
9276 register value, try to optimize it. Otherwise, we can't do anything. */
9278 jump = PREV_INSN (loop_end);
9279 comparison = get_condition_for_loop (loop, jump);
9280 if (comparison == 0)
9281 return 0;
9282 if (!onlyjump_p (jump))
9283 return 0;
9285 /* Try to compute whether the compare/branch at the loop end is one or
9286 two instructions. */
9287 get_condition (jump, &first_compare, false, true);
9288 if (first_compare == jump)
9289 compare_and_branch = 1;
9290 else if (first_compare == prev_nonnote_insn (jump))
9291 compare_and_branch = 2;
9292 else
9293 return 0;
9296 /* If more than one condition is present to control the loop, then
9297 do not proceed, as this function does not know how to rewrite
9298 loop tests with more than one condition.
9300 Look backwards from the first insn in the last comparison
9301 sequence and see if we've got another comparison sequence. */
9303 rtx jump1;
9304 if ((jump1 = prev_nonnote_insn (first_compare))
9305 && JUMP_P (jump1))
9306 return 0;
9309 /* Check all of the bivs to see if the compare uses one of them.
9310 Skip biv's set more than once because we can't guarantee that
9311 it will be zero on the last iteration. Also skip if the biv is
9312 used between its update and the test insn. */
9314 for (bl = ivs->list; bl; bl = bl->next)
9316 if (bl->biv_count == 1
9317 && ! bl->biv->maybe_multiple
9318 && bl->biv->dest_reg == XEXP (comparison, 0)
9319 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
9320 first_compare))
9321 break;
9324 /* Try swapping the comparison to identify a suitable biv. */
9325 if (!bl)
9326 for (bl = ivs->list; bl; bl = bl->next)
9327 if (bl->biv_count == 1
9328 && ! bl->biv->maybe_multiple
9329 && bl->biv->dest_reg == XEXP (comparison, 1)
9330 && ! reg_used_between_p (regno_reg_rtx[bl->regno], bl->biv->insn,
9331 first_compare))
9333 comparison = gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)),
9334 VOIDmode,
9335 XEXP (comparison, 1),
9336 XEXP (comparison, 0));
9337 break;
9340 if (! bl)
9341 return 0;
9343 /* Look for the case where the basic induction variable is always
9344 nonnegative, and equals zero on the last iteration.
9345 In this case, add a reg_note REG_NONNEG, which allows the
9346 m68k DBRA instruction to be used. */
9348 if (((GET_CODE (comparison) == GT && XEXP (comparison, 1) == constm1_rtx)
9349 || (GET_CODE (comparison) == NE && XEXP (comparison, 1) == const0_rtx))
9350 && GET_CODE (bl->biv->add_val) == CONST_INT
9351 && INTVAL (bl->biv->add_val) < 0)
9353 /* Initial value must be greater than 0,
9354 init_val % -dec_value == 0 to ensure that it equals zero on
9355 the last iteration */
9357 if (GET_CODE (bl->initial_value) == CONST_INT
9358 && INTVAL (bl->initial_value) > 0
9359 && (INTVAL (bl->initial_value)
9360 % (-INTVAL (bl->biv->add_val))) == 0)
9362 /* Register always nonnegative, add REG_NOTE to branch. */
9363 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
9364 REG_NOTES (jump)
9365 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
9366 REG_NOTES (jump));
9367 bl->nonneg = 1;
9369 return 1;
9372 /* If the decrement is 1 and the value was tested as >= 0 before
9373 the loop, then we can safely optimize. */
9374 for (p = loop_start; p; p = PREV_INSN (p))
9376 if (LABEL_P (p))
9377 break;
9378 if (!JUMP_P (p))
9379 continue;
9381 before_comparison = get_condition_for_loop (loop, p);
9382 if (before_comparison
9383 && XEXP (before_comparison, 0) == bl->biv->dest_reg
9384 && (GET_CODE (before_comparison) == LT
9385 || GET_CODE (before_comparison) == LTU)
9386 && XEXP (before_comparison, 1) == const0_rtx
9387 && ! reg_set_between_p (bl->biv->dest_reg, p, loop_start)
9388 && INTVAL (bl->biv->add_val) == -1)
9390 if (! find_reg_note (jump, REG_NONNEG, NULL_RTX))
9391 REG_NOTES (jump)
9392 = gen_rtx_EXPR_LIST (REG_NONNEG, bl->biv->dest_reg,
9393 REG_NOTES (jump));
9394 bl->nonneg = 1;
9396 return 1;
9400 else if (GET_CODE (bl->biv->add_val) == CONST_INT
9401 && INTVAL (bl->biv->add_val) > 0)
9403 /* Try to change inc to dec, so can apply above optimization. */
9404 /* Can do this if:
9405 all registers modified are induction variables or invariant,
9406 all memory references have non-overlapping addresses
9407 (obviously true if only one write)
9408 allow 2 insns for the compare/jump at the end of the loop. */
9409 /* Also, we must avoid any instructions which use both the reversed
9410 biv and another biv. Such instructions will fail if the loop is
9411 reversed. We meet this condition by requiring that either
9412 no_use_except_counting is true, or else that there is only
9413 one biv. */
9414 int num_nonfixed_reads = 0;
9415 /* 1 if the iteration var is used only to count iterations. */
9416 int no_use_except_counting = 0;
9417 /* 1 if the loop has no memory store, or it has a single memory store
9418 which is reversible. */
9419 int reversible_mem_store = 1;
9421 if (bl->giv_count == 0
9422 && !loop->exit_count
9423 && !loop_info->has_multiple_exit_targets)
9425 rtx bivreg = regno_reg_rtx[bl->regno];
9426 struct iv_class *blt;
9428 /* If there are no givs for this biv, and the only exit is the
9429 fall through at the end of the loop, then
9430 see if perhaps there are no uses except to count. */
9431 no_use_except_counting = 1;
9432 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
9433 if (INSN_P (p))
9435 rtx set = single_set (p);
9437 if (set && REG_P (SET_DEST (set))
9438 && REGNO (SET_DEST (set)) == bl->regno)
9439 /* An insn that sets the biv is okay. */
9441 else if (!reg_mentioned_p (bivreg, PATTERN (p)))
9442 /* An insn that doesn't mention the biv is okay. */
9444 else if (p == prev_nonnote_insn (prev_nonnote_insn (loop_end))
9445 || p == prev_nonnote_insn (loop_end))
9447 /* If either of these insns uses the biv and sets a pseudo
9448 that has more than one usage, then the biv has uses
9449 other than counting since it's used to derive a value
9450 that is used more than one time. */
9451 note_stores (PATTERN (p), note_set_pseudo_multiple_uses,
9452 regs);
9453 if (regs->multiple_uses)
9455 no_use_except_counting = 0;
9456 break;
9459 else
9461 no_use_except_counting = 0;
9462 break;
9466 /* A biv has uses besides counting if it is used to set
9467 another biv. */
9468 for (blt = ivs->list; blt; blt = blt->next)
9469 if (blt->init_set
9470 && reg_mentioned_p (bivreg, SET_SRC (blt->init_set)))
9472 no_use_except_counting = 0;
9473 break;
9477 if (no_use_except_counting)
9478 /* No need to worry about MEMs. */
9480 else if (loop_info->num_mem_sets <= 1)
9482 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
9483 if (INSN_P (p))
9484 num_nonfixed_reads += count_nonfixed_reads (loop, PATTERN (p));
9486 /* If the loop has a single store, and the destination address is
9487 invariant, then we can't reverse the loop, because this address
9488 might then have the wrong value at loop exit.
9489 This would work if the source was invariant also, however, in that
9490 case, the insn should have been moved out of the loop. */
9492 if (loop_info->num_mem_sets == 1)
9494 struct induction *v;
9496 /* If we could prove that each of the memory locations
9497 written to was different, then we could reverse the
9498 store -- but we don't presently have any way of
9499 knowing that. */
9500 reversible_mem_store = 0;
9502 /* If the store depends on a register that is set after the
9503 store, it depends on the initial value, and is thus not
9504 reversible. */
9505 for (v = bl->giv; reversible_mem_store && v; v = v->next_iv)
9507 if (v->giv_type == DEST_REG
9508 && reg_mentioned_p (v->dest_reg,
9509 PATTERN (loop_info->first_loop_store_insn))
9510 && loop_insn_first_p (loop_info->first_loop_store_insn,
9511 v->insn))
9512 reversible_mem_store = 0;
9516 else
9517 return 0;
9519 /* This code only acts for innermost loops. Also it simplifies
9520 the memory address check by only reversing loops with
9521 zero or one memory access.
9522 Two memory accesses could involve parts of the same array,
9523 and that can't be reversed.
9524 If the biv is used only for counting, than we don't need to worry
9525 about all these things. */
9527 if ((num_nonfixed_reads <= 1
9528 && ! loop_info->has_nonconst_call
9529 && ! loop_info->has_prefetch
9530 && ! loop_info->has_volatile
9531 && reversible_mem_store
9532 && (bl->giv_count + bl->biv_count + loop_info->num_mem_sets
9533 + num_unmoved_movables (loop) + compare_and_branch == insn_count)
9534 && (bl == ivs->list && bl->next == 0))
9535 || (no_use_except_counting && ! loop_info->has_prefetch))
9537 rtx tem;
9539 /* Loop can be reversed. */
9540 if (loop_dump_stream)
9541 fprintf (loop_dump_stream, "Can reverse loop\n");
9543 /* Now check other conditions:
9545 The increment must be a constant, as must the initial value,
9546 and the comparison code must be LT.
9548 This test can probably be improved since +/- 1 in the constant
9549 can be obtained by changing LT to LE and vice versa; this is
9550 confusing. */
9552 if (comparison
9553 /* for constants, LE gets turned into LT */
9554 && (GET_CODE (comparison) == LT
9555 || (GET_CODE (comparison) == LE
9556 && no_use_except_counting)
9557 || GET_CODE (comparison) == LTU))
9559 HOST_WIDE_INT add_val, add_adjust, comparison_val = 0;
9560 rtx initial_value, comparison_value;
9561 int nonneg = 0;
9562 enum rtx_code cmp_code;
9563 int comparison_const_width;
9564 unsigned HOST_WIDE_INT comparison_sign_mask;
9565 bool keep_first_compare;
9567 add_val = INTVAL (bl->biv->add_val);
9568 comparison_value = XEXP (comparison, 1);
9569 if (GET_MODE (comparison_value) == VOIDmode)
9570 comparison_const_width
9571 = GET_MODE_BITSIZE (GET_MODE (XEXP (comparison, 0)));
9572 else
9573 comparison_const_width
9574 = GET_MODE_BITSIZE (GET_MODE (comparison_value));
9575 if (comparison_const_width > HOST_BITS_PER_WIDE_INT)
9576 comparison_const_width = HOST_BITS_PER_WIDE_INT;
9577 comparison_sign_mask
9578 = (unsigned HOST_WIDE_INT) 1 << (comparison_const_width - 1);
9580 /* If the comparison value is not a loop invariant, then we
9581 can not reverse this loop.
9583 ??? If the insns which initialize the comparison value as
9584 a whole compute an invariant result, then we could move
9585 them out of the loop and proceed with loop reversal. */
9586 if (! loop_invariant_p (loop, comparison_value))
9587 return 0;
9589 if (GET_CODE (comparison_value) == CONST_INT)
9590 comparison_val = INTVAL (comparison_value);
9591 initial_value = bl->initial_value;
9593 /* Normalize the initial value if it is an integer and
9594 has no other use except as a counter. This will allow
9595 a few more loops to be reversed. */
9596 if (no_use_except_counting
9597 && GET_CODE (comparison_value) == CONST_INT
9598 && GET_CODE (initial_value) == CONST_INT)
9600 comparison_val = comparison_val - INTVAL (bl->initial_value);
9601 /* The code below requires comparison_val to be a multiple
9602 of add_val in order to do the loop reversal, so
9603 round up comparison_val to a multiple of add_val.
9604 Since comparison_value is constant, we know that the
9605 current comparison code is LT. */
9606 comparison_val = comparison_val + add_val - 1;
9607 comparison_val
9608 -= (unsigned HOST_WIDE_INT) comparison_val % add_val;
9609 /* We postpone overflow checks for COMPARISON_VAL here;
9610 even if there is an overflow, we might still be able to
9611 reverse the loop, if converting the loop exit test to
9612 NE is possible. */
9613 initial_value = const0_rtx;
9616 /* First check if we can do a vanilla loop reversal. */
9617 if (initial_value == const0_rtx
9618 && GET_CODE (comparison_value) == CONST_INT
9619 /* Now do postponed overflow checks on COMPARISON_VAL. */
9620 && ! (((comparison_val - add_val) ^ INTVAL (comparison_value))
9621 & comparison_sign_mask))
9623 /* Register will always be nonnegative, with value
9624 0 on last iteration */
9625 add_adjust = add_val;
9626 nonneg = 1;
9627 cmp_code = GE;
9629 else
9630 return 0;
9632 if (GET_CODE (comparison) == LE)
9633 add_adjust -= add_val;
9635 /* If the initial value is not zero, or if the comparison
9636 value is not an exact multiple of the increment, then we
9637 can not reverse this loop. */
9638 if (initial_value == const0_rtx
9639 && GET_CODE (comparison_value) == CONST_INT)
9641 if (((unsigned HOST_WIDE_INT) comparison_val % add_val) != 0)
9642 return 0;
9644 else
9646 if (! no_use_except_counting || add_val != 1)
9647 return 0;
9650 final_value = comparison_value;
9652 /* Reset these in case we normalized the initial value
9653 and comparison value above. */
9654 if (GET_CODE (comparison_value) == CONST_INT
9655 && GET_CODE (initial_value) == CONST_INT)
9657 comparison_value = GEN_INT (comparison_val);
9658 final_value
9659 = GEN_INT (comparison_val + INTVAL (bl->initial_value));
9661 bl->initial_value = initial_value;
9663 /* Save some info needed to produce the new insns. */
9664 reg = bl->biv->dest_reg;
9665 mode = GET_MODE (reg);
9666 jump_label = condjump_label (PREV_INSN (loop_end));
9667 new_add_val = GEN_INT (-INTVAL (bl->biv->add_val));
9669 /* Set start_value; if this is not a CONST_INT, we need
9670 to generate a SUB.
9671 Initialize biv to start_value before loop start.
9672 The old initializing insn will be deleted as a
9673 dead store by flow.c. */
9674 if (initial_value == const0_rtx
9675 && GET_CODE (comparison_value) == CONST_INT)
9677 start_value
9678 = gen_int_mode (comparison_val - add_adjust, mode);
9679 loop_insn_hoist (loop, gen_move_insn (reg, start_value));
9681 else if (GET_CODE (initial_value) == CONST_INT)
9683 rtx offset = GEN_INT (-INTVAL (initial_value) - add_adjust);
9684 rtx add_insn = gen_add3_insn (reg, comparison_value, offset);
9686 if (add_insn == 0)
9687 return 0;
9689 start_value
9690 = gen_rtx_PLUS (mode, comparison_value, offset);
9691 loop_insn_hoist (loop, add_insn);
9692 if (GET_CODE (comparison) == LE)
9693 final_value = gen_rtx_PLUS (mode, comparison_value,
9694 GEN_INT (add_val));
9696 else if (! add_adjust)
9698 rtx sub_insn = gen_sub3_insn (reg, comparison_value,
9699 initial_value);
9701 if (sub_insn == 0)
9702 return 0;
9703 start_value
9704 = gen_rtx_MINUS (mode, comparison_value, initial_value);
9705 loop_insn_hoist (loop, sub_insn);
9707 else
9708 /* We could handle the other cases too, but it'll be
9709 better to have a testcase first. */
9710 return 0;
9712 /* We may not have a single insn which can increment a reg, so
9713 create a sequence to hold all the insns from expand_inc. */
9714 start_sequence ();
9715 expand_inc (reg, new_add_val);
9716 tem = get_insns ();
9717 end_sequence ();
9719 p = loop_insn_emit_before (loop, 0, bl->biv->insn, tem);
9720 delete_insn (bl->biv->insn);
9722 /* Update biv info to reflect its new status. */
9723 bl->biv->insn = p;
9724 bl->initial_value = start_value;
9725 bl->biv->add_val = new_add_val;
9727 /* Update loop info. */
9728 loop_info->initial_value = reg;
9729 loop_info->initial_equiv_value = reg;
9730 loop_info->final_value = const0_rtx;
9731 loop_info->final_equiv_value = const0_rtx;
9732 loop_info->comparison_value = const0_rtx;
9733 loop_info->comparison_code = cmp_code;
9734 loop_info->increment = new_add_val;
9736 /* Inc LABEL_NUSES so that delete_insn will
9737 not delete the label. */
9738 LABEL_NUSES (XEXP (jump_label, 0))++;
9740 /* If we have a separate comparison insn that does more
9741 than just set cc0, the result of the comparison might
9742 be used outside the loop. */
9743 keep_first_compare = (compare_and_branch == 2
9744 #ifdef HAVE_CC0
9745 && sets_cc0_p (first_compare) <= 0
9746 #endif
9749 /* Emit an insn after the end of the loop to set the biv's
9750 proper exit value if it is used anywhere outside the loop. */
9751 if (keep_first_compare
9752 || (REGNO_LAST_UID (bl->regno) != INSN_UID (first_compare))
9753 || ! bl->init_insn
9754 || REGNO_FIRST_UID (bl->regno) != INSN_UID (bl->init_insn))
9755 loop_insn_sink (loop, gen_load_of_final_value (reg, final_value));
9757 if (keep_first_compare)
9758 loop_insn_sink (loop, PATTERN (first_compare));
9760 /* Delete compare/branch at end of loop. */
9761 delete_related_insns (PREV_INSN (loop_end));
9762 if (compare_and_branch == 2)
9763 delete_related_insns (first_compare);
9765 /* Add new compare/branch insn at end of loop. */
9766 start_sequence ();
9767 emit_cmp_and_jump_insns (reg, const0_rtx, cmp_code, NULL_RTX,
9768 mode, 0,
9769 XEXP (jump_label, 0));
9770 tem = get_insns ();
9771 end_sequence ();
9772 emit_jump_insn_before (tem, loop_end);
9774 for (tem = PREV_INSN (loop_end);
9775 tem && !JUMP_P (tem);
9776 tem = PREV_INSN (tem))
9779 if (tem)
9780 JUMP_LABEL (tem) = XEXP (jump_label, 0);
9782 if (nonneg)
9784 if (tem)
9786 /* Increment of LABEL_NUSES done above. */
9787 /* Register is now always nonnegative,
9788 so add REG_NONNEG note to the branch. */
9789 REG_NOTES (tem) = gen_rtx_EXPR_LIST (REG_NONNEG, reg,
9790 REG_NOTES (tem));
9792 bl->nonneg = 1;
9795 /* No insn may reference both the reversed and another biv or it
9796 will fail (see comment near the top of the loop reversal
9797 code).
9798 Earlier on, we have verified that the biv has no use except
9799 counting, or it is the only biv in this function.
9800 However, the code that computes no_use_except_counting does
9801 not verify reg notes. It's possible to have an insn that
9802 references another biv, and has a REG_EQUAL note with an
9803 expression based on the reversed biv. To avoid this case,
9804 remove all REG_EQUAL notes based on the reversed biv
9805 here. */
9806 for (p = loop_start; p != loop_end; p = NEXT_INSN (p))
9807 if (INSN_P (p))
9809 rtx *pnote;
9810 rtx set = single_set (p);
9811 /* If this is a set of a GIV based on the reversed biv, any
9812 REG_EQUAL notes should still be correct. */
9813 if (! set
9814 || !REG_P (SET_DEST (set))
9815 || (size_t) REGNO (SET_DEST (set)) >= ivs->n_regs
9816 || REG_IV_TYPE (ivs, REGNO (SET_DEST (set))) != GENERAL_INDUCT
9817 || REG_IV_INFO (ivs, REGNO (SET_DEST (set)))->src_reg != bl->biv->src_reg)
9818 for (pnote = &REG_NOTES (p); *pnote;)
9820 if (REG_NOTE_KIND (*pnote) == REG_EQUAL
9821 && reg_mentioned_p (regno_reg_rtx[bl->regno],
9822 XEXP (*pnote, 0)))
9823 *pnote = XEXP (*pnote, 1);
9824 else
9825 pnote = &XEXP (*pnote, 1);
9829 /* Mark that this biv has been reversed. Each giv which depends
9830 on this biv, and which is also live past the end of the loop
9831 will have to be fixed up. */
9833 bl->reversed = 1;
9835 if (loop_dump_stream)
9837 fprintf (loop_dump_stream, "Reversed loop");
9838 if (bl->nonneg)
9839 fprintf (loop_dump_stream, " and added reg_nonneg\n");
9840 else
9841 fprintf (loop_dump_stream, "\n");
9844 return 1;
9849 return 0;
9852 /* Verify whether the biv BL appears to be eliminable,
9853 based on the insns in the loop that refer to it.
9855 If ELIMINATE_P is nonzero, actually do the elimination.
9857 THRESHOLD and INSN_COUNT are from loop_optimize and are used to
9858 determine whether invariant insns should be placed inside or at the
9859 start of the loop. */
9861 static int
9862 maybe_eliminate_biv (const struct loop *loop, struct iv_class *bl,
9863 int eliminate_p, int threshold, int insn_count)
9865 struct loop_ivs *ivs = LOOP_IVS (loop);
9866 rtx reg = bl->biv->dest_reg;
9867 rtx p;
9869 /* Scan all insns in the loop, stopping if we find one that uses the
9870 biv in a way that we cannot eliminate. */
9872 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
9874 enum rtx_code code = GET_CODE (p);
9875 basic_block where_bb = 0;
9876 rtx where_insn = threshold >= insn_count ? 0 : p;
9877 rtx note;
9879 /* If this is a libcall that sets a giv, skip ahead to its end. */
9880 if (INSN_P (p))
9882 note = find_reg_note (p, REG_LIBCALL, NULL_RTX);
9884 if (note)
9886 rtx last = XEXP (note, 0);
9887 rtx set = single_set (last);
9889 if (set && REG_P (SET_DEST (set)))
9891 unsigned int regno = REGNO (SET_DEST (set));
9893 if (regno < ivs->n_regs
9894 && REG_IV_TYPE (ivs, regno) == GENERAL_INDUCT
9895 && REG_IV_INFO (ivs, regno)->src_reg == bl->biv->src_reg)
9896 p = last;
9901 /* Closely examine the insn if the biv is mentioned. */
9902 if ((code == INSN || code == JUMP_INSN || code == CALL_INSN)
9903 && reg_mentioned_p (reg, PATTERN (p))
9904 && ! maybe_eliminate_biv_1 (loop, PATTERN (p), p, bl,
9905 eliminate_p, where_bb, where_insn))
9907 if (loop_dump_stream)
9908 fprintf (loop_dump_stream,
9909 "Cannot eliminate biv %d: biv used in insn %d.\n",
9910 bl->regno, INSN_UID (p));
9911 break;
9914 /* If we are eliminating, kill REG_EQUAL notes mentioning the biv. */
9915 if (eliminate_p
9916 && (note = find_reg_note (p, REG_EQUAL, NULL_RTX)) != NULL_RTX
9917 && reg_mentioned_p (reg, XEXP (note, 0)))
9918 remove_note (p, note);
9921 if (p == loop->end)
9923 if (loop_dump_stream)
9924 fprintf (loop_dump_stream, "biv %d %s eliminated.\n",
9925 bl->regno, eliminate_p ? "was" : "can be");
9926 return 1;
9929 return 0;
9932 /* INSN and REFERENCE are instructions in the same insn chain.
9933 Return nonzero if INSN is first. */
9935 static int
9936 loop_insn_first_p (rtx insn, rtx reference)
9938 rtx p, q;
9940 for (p = insn, q = reference;;)
9942 /* Start with test for not first so that INSN == REFERENCE yields not
9943 first. */
9944 if (q == insn || ! p)
9945 return 0;
9946 if (p == reference || ! q)
9947 return 1;
9949 /* Either of P or Q might be a NOTE. Notes have the same LUID as the
9950 previous insn, hence the <= comparison below does not work if
9951 P is a note. */
9952 if (INSN_UID (p) < max_uid_for_loop
9953 && INSN_UID (q) < max_uid_for_loop
9954 && !NOTE_P (p))
9955 return INSN_LUID (p) <= INSN_LUID (q);
9957 if (INSN_UID (p) >= max_uid_for_loop
9958 || NOTE_P (p))
9959 p = NEXT_INSN (p);
9960 if (INSN_UID (q) >= max_uid_for_loop)
9961 q = NEXT_INSN (q);
9965 /* We are trying to eliminate BIV in INSN using GIV. Return nonzero if
9966 the offset that we have to take into account due to auto-increment /
9967 div derivation is zero. */
9968 static int
9969 biv_elimination_giv_has_0_offset (struct induction *biv,
9970 struct induction *giv, rtx insn)
9972 /* If the giv V had the auto-inc address optimization applied
9973 to it, and INSN occurs between the giv insn and the biv
9974 insn, then we'd have to adjust the value used here.
9975 This is rare, so we don't bother to make this possible. */
9976 if (giv->auto_inc_opt
9977 && ((loop_insn_first_p (giv->insn, insn)
9978 && loop_insn_first_p (insn, biv->insn))
9979 || (loop_insn_first_p (biv->insn, insn)
9980 && loop_insn_first_p (insn, giv->insn))))
9981 return 0;
9983 return 1;
9986 /* If BL appears in X (part of the pattern of INSN), see if we can
9987 eliminate its use. If so, return 1. If not, return 0.
9989 If BIV does not appear in X, return 1.
9991 If ELIMINATE_P is nonzero, actually do the elimination.
9992 WHERE_INSN/WHERE_BB indicate where extra insns should be added.
9993 Depending on how many items have been moved out of the loop, it
9994 will either be before INSN (when WHERE_INSN is nonzero) or at the
9995 start of the loop (when WHERE_INSN is zero). */
9997 static int
9998 maybe_eliminate_biv_1 (const struct loop *loop, rtx x, rtx insn,
9999 struct iv_class *bl, int eliminate_p,
10000 basic_block where_bb, rtx where_insn)
10002 enum rtx_code code = GET_CODE (x);
10003 rtx reg = bl->biv->dest_reg;
10004 enum machine_mode mode = GET_MODE (reg);
10005 struct induction *v;
10006 rtx arg, tem;
10007 #ifdef HAVE_cc0
10008 rtx new;
10009 #endif
10010 int arg_operand;
10011 const char *fmt;
10012 int i, j;
10014 switch (code)
10016 case REG:
10017 /* If we haven't already been able to do something with this BIV,
10018 we can't eliminate it. */
10019 if (x == reg)
10020 return 0;
10021 return 1;
10023 case SET:
10024 /* If this sets the BIV, it is not a problem. */
10025 if (SET_DEST (x) == reg)
10026 return 1;
10028 /* If this is an insn that defines a giv, it is also ok because
10029 it will go away when the giv is reduced. */
10030 for (v = bl->giv; v; v = v->next_iv)
10031 if (v->giv_type == DEST_REG && SET_DEST (x) == v->dest_reg)
10032 return 1;
10034 #ifdef HAVE_cc0
10035 if (SET_DEST (x) == cc0_rtx && SET_SRC (x) == reg)
10037 /* Can replace with any giv that was reduced and
10038 that has (MULT_VAL != 0) and (ADD_VAL == 0).
10039 Require a constant for MULT_VAL, so we know it's nonzero.
10040 ??? We disable this optimization to avoid potential
10041 overflows. */
10043 for (v = bl->giv; v; v = v->next_iv)
10044 if (GET_CODE (v->mult_val) == CONST_INT && v->mult_val != const0_rtx
10045 && v->add_val == const0_rtx
10046 && ! v->ignore && ! v->maybe_dead && v->always_computable
10047 && v->mode == mode
10048 && 0)
10050 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
10051 continue;
10053 if (! eliminate_p)
10054 return 1;
10056 /* If the giv has the opposite direction of change,
10057 then reverse the comparison. */
10058 if (INTVAL (v->mult_val) < 0)
10059 new = gen_rtx_COMPARE (GET_MODE (v->new_reg),
10060 const0_rtx, v->new_reg);
10061 else
10062 new = v->new_reg;
10064 /* We can probably test that giv's reduced reg. */
10065 if (validate_change (insn, &SET_SRC (x), new, 0))
10066 return 1;
10069 /* Look for a giv with (MULT_VAL != 0) and (ADD_VAL != 0);
10070 replace test insn with a compare insn (cmp REDUCED_GIV ADD_VAL).
10071 Require a constant for MULT_VAL, so we know it's nonzero.
10072 ??? Do this only if ADD_VAL is a pointer to avoid a potential
10073 overflow problem. */
10075 for (v = bl->giv; v; v = v->next_iv)
10076 if (GET_CODE (v->mult_val) == CONST_INT
10077 && v->mult_val != const0_rtx
10078 && ! v->ignore && ! v->maybe_dead && v->always_computable
10079 && v->mode == mode
10080 && (GET_CODE (v->add_val) == SYMBOL_REF
10081 || GET_CODE (v->add_val) == LABEL_REF
10082 || GET_CODE (v->add_val) == CONST
10083 || (REG_P (v->add_val)
10084 && REG_POINTER (v->add_val))))
10086 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
10087 continue;
10089 if (! eliminate_p)
10090 return 1;
10092 /* If the giv has the opposite direction of change,
10093 then reverse the comparison. */
10094 if (INTVAL (v->mult_val) < 0)
10095 new = gen_rtx_COMPARE (VOIDmode, copy_rtx (v->add_val),
10096 v->new_reg);
10097 else
10098 new = gen_rtx_COMPARE (VOIDmode, v->new_reg,
10099 copy_rtx (v->add_val));
10101 /* Replace biv with the giv's reduced register. */
10102 update_reg_last_use (v->add_val, insn);
10103 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
10104 return 1;
10106 /* Insn doesn't support that constant or invariant. Copy it
10107 into a register (it will be a loop invariant.) */
10108 tem = gen_reg_rtx (GET_MODE (v->new_reg));
10110 loop_insn_emit_before (loop, 0, where_insn,
10111 gen_move_insn (tem,
10112 copy_rtx (v->add_val)));
10114 /* Substitute the new register for its invariant value in
10115 the compare expression. */
10116 XEXP (new, (INTVAL (v->mult_val) < 0) ? 0 : 1) = tem;
10117 if (validate_change (insn, &SET_SRC (PATTERN (insn)), new, 0))
10118 return 1;
10121 #endif
10122 break;
10124 case COMPARE:
10125 case EQ: case NE:
10126 case GT: case GE: case GTU: case GEU:
10127 case LT: case LE: case LTU: case LEU:
10128 /* See if either argument is the biv. */
10129 if (XEXP (x, 0) == reg)
10130 arg = XEXP (x, 1), arg_operand = 1;
10131 else if (XEXP (x, 1) == reg)
10132 arg = XEXP (x, 0), arg_operand = 0;
10133 else
10134 break;
10136 if (CONSTANT_P (arg))
10138 /* First try to replace with any giv that has constant positive
10139 mult_val and constant add_val. We might be able to support
10140 negative mult_val, but it seems complex to do it in general. */
10142 for (v = bl->giv; v; v = v->next_iv)
10143 if (GET_CODE (v->mult_val) == CONST_INT
10144 && INTVAL (v->mult_val) > 0
10145 && (GET_CODE (v->add_val) == SYMBOL_REF
10146 || GET_CODE (v->add_val) == LABEL_REF
10147 || GET_CODE (v->add_val) == CONST
10148 || (REG_P (v->add_val)
10149 && REG_POINTER (v->add_val)))
10150 && ! v->ignore && ! v->maybe_dead && v->always_computable
10151 && v->mode == mode)
10153 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
10154 continue;
10156 /* Don't eliminate if the linear combination that makes up
10157 the giv overflows when it is applied to ARG. */
10158 if (GET_CODE (arg) == CONST_INT)
10160 rtx add_val;
10162 if (GET_CODE (v->add_val) == CONST_INT)
10163 add_val = v->add_val;
10164 else
10165 add_val = const0_rtx;
10167 if (const_mult_add_overflow_p (arg, v->mult_val,
10168 add_val, mode, 1))
10169 continue;
10172 if (! eliminate_p)
10173 return 1;
10175 /* Replace biv with the giv's reduced reg. */
10176 validate_change (insn, &XEXP (x, 1 - arg_operand), v->new_reg, 1);
10178 /* If all constants are actually constant integers and
10179 the derived constant can be directly placed in the COMPARE,
10180 do so. */
10181 if (GET_CODE (arg) == CONST_INT
10182 && GET_CODE (v->add_val) == CONST_INT)
10184 tem = expand_mult_add (arg, NULL_RTX, v->mult_val,
10185 v->add_val, mode, 1);
10187 else
10189 /* Otherwise, load it into a register. */
10190 tem = gen_reg_rtx (mode);
10191 loop_iv_add_mult_emit_before (loop, arg,
10192 v->mult_val, v->add_val,
10193 tem, where_bb, where_insn);
10196 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
10198 if (apply_change_group ())
10199 return 1;
10202 /* Look for giv with positive constant mult_val and nonconst add_val.
10203 Insert insns to calculate new compare value.
10204 ??? Turn this off due to possible overflow. */
10206 for (v = bl->giv; v; v = v->next_iv)
10207 if (GET_CODE (v->mult_val) == CONST_INT
10208 && INTVAL (v->mult_val) > 0
10209 && ! v->ignore && ! v->maybe_dead && v->always_computable
10210 && v->mode == mode
10211 && 0)
10213 rtx tem;
10215 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
10216 continue;
10218 if (! eliminate_p)
10219 return 1;
10221 tem = gen_reg_rtx (mode);
10223 /* Replace biv with giv's reduced register. */
10224 validate_change (insn, &XEXP (x, 1 - arg_operand),
10225 v->new_reg, 1);
10227 /* Compute value to compare against. */
10228 loop_iv_add_mult_emit_before (loop, arg,
10229 v->mult_val, v->add_val,
10230 tem, where_bb, where_insn);
10231 /* Use it in this insn. */
10232 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
10233 if (apply_change_group ())
10234 return 1;
10237 else if (REG_P (arg) || MEM_P (arg))
10239 if (loop_invariant_p (loop, arg) == 1)
10241 /* Look for giv with constant positive mult_val and nonconst
10242 add_val. Insert insns to compute new compare value.
10243 ??? Turn this off due to possible overflow. */
10245 for (v = bl->giv; v; v = v->next_iv)
10246 if (GET_CODE (v->mult_val) == CONST_INT && INTVAL (v->mult_val) > 0
10247 && ! v->ignore && ! v->maybe_dead && v->always_computable
10248 && v->mode == mode
10249 && 0)
10251 rtx tem;
10253 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
10254 continue;
10256 if (! eliminate_p)
10257 return 1;
10259 tem = gen_reg_rtx (mode);
10261 /* Replace biv with giv's reduced register. */
10262 validate_change (insn, &XEXP (x, 1 - arg_operand),
10263 v->new_reg, 1);
10265 /* Compute value to compare against. */
10266 loop_iv_add_mult_emit_before (loop, arg,
10267 v->mult_val, v->add_val,
10268 tem, where_bb, where_insn);
10269 validate_change (insn, &XEXP (x, arg_operand), tem, 1);
10270 if (apply_change_group ())
10271 return 1;
10275 /* This code has problems. Basically, you can't know when
10276 seeing if we will eliminate BL, whether a particular giv
10277 of ARG will be reduced. If it isn't going to be reduced,
10278 we can't eliminate BL. We can try forcing it to be reduced,
10279 but that can generate poor code.
10281 The problem is that the benefit of reducing TV, below should
10282 be increased if BL can actually be eliminated, but this means
10283 we might have to do a topological sort of the order in which
10284 we try to process biv. It doesn't seem worthwhile to do
10285 this sort of thing now. */
10287 #if 0
10288 /* Otherwise the reg compared with had better be a biv. */
10289 if (!REG_P (arg)
10290 || REG_IV_TYPE (ivs, REGNO (arg)) != BASIC_INDUCT)
10291 return 0;
10293 /* Look for a pair of givs, one for each biv,
10294 with identical coefficients. */
10295 for (v = bl->giv; v; v = v->next_iv)
10297 struct induction *tv;
10299 if (v->ignore || v->maybe_dead || v->mode != mode)
10300 continue;
10302 for (tv = REG_IV_CLASS (ivs, REGNO (arg))->giv; tv;
10303 tv = tv->next_iv)
10304 if (! tv->ignore && ! tv->maybe_dead
10305 && rtx_equal_p (tv->mult_val, v->mult_val)
10306 && rtx_equal_p (tv->add_val, v->add_val)
10307 && tv->mode == mode)
10309 if (! biv_elimination_giv_has_0_offset (bl->biv, v, insn))
10310 continue;
10312 if (! eliminate_p)
10313 return 1;
10315 /* Replace biv with its giv's reduced reg. */
10316 XEXP (x, 1 - arg_operand) = v->new_reg;
10317 /* Replace other operand with the other giv's
10318 reduced reg. */
10319 XEXP (x, arg_operand) = tv->new_reg;
10320 return 1;
10323 #endif
10326 /* If we get here, the biv can't be eliminated. */
10327 return 0;
10329 case MEM:
10330 /* If this address is a DEST_ADDR giv, it doesn't matter if the
10331 biv is used in it, since it will be replaced. */
10332 for (v = bl->giv; v; v = v->next_iv)
10333 if (v->giv_type == DEST_ADDR && v->location == &XEXP (x, 0))
10334 return 1;
10335 break;
10337 default:
10338 break;
10341 /* See if any subexpression fails elimination. */
10342 fmt = GET_RTX_FORMAT (code);
10343 for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
10345 switch (fmt[i])
10347 case 'e':
10348 if (! maybe_eliminate_biv_1 (loop, XEXP (x, i), insn, bl,
10349 eliminate_p, where_bb, where_insn))
10350 return 0;
10351 break;
10353 case 'E':
10354 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
10355 if (! maybe_eliminate_biv_1 (loop, XVECEXP (x, i, j), insn, bl,
10356 eliminate_p, where_bb, where_insn))
10357 return 0;
10358 break;
10362 return 1;
10365 /* Return nonzero if the last use of REG
10366 is in an insn following INSN in the same basic block. */
10368 static int
10369 last_use_this_basic_block (rtx reg, rtx insn)
10371 rtx n;
10372 for (n = insn;
10373 n && !LABEL_P (n) && !JUMP_P (n);
10374 n = NEXT_INSN (n))
10376 if (REGNO_LAST_UID (REGNO (reg)) == INSN_UID (n))
10377 return 1;
10379 return 0;
10382 /* Called via `note_stores' to record the initial value of a biv. Here we
10383 just record the location of the set and process it later. */
10385 static void
10386 record_initial (rtx dest, rtx set, void *data ATTRIBUTE_UNUSED)
10388 struct loop_ivs *ivs = (struct loop_ivs *) data;
10389 struct iv_class *bl;
10391 if (!REG_P (dest)
10392 || REGNO (dest) >= ivs->n_regs
10393 || REG_IV_TYPE (ivs, REGNO (dest)) != BASIC_INDUCT)
10394 return;
10396 bl = REG_IV_CLASS (ivs, REGNO (dest));
10398 /* If this is the first set found, record it. */
10399 if (bl->init_insn == 0)
10401 bl->init_insn = note_insn;
10402 bl->init_set = set;
10406 /* If any of the registers in X are "old" and currently have a last use earlier
10407 than INSN, update them to have a last use of INSN. Their actual last use
10408 will be the previous insn but it will not have a valid uid_luid so we can't
10409 use it. X must be a source expression only. */
10411 static void
10412 update_reg_last_use (rtx x, rtx insn)
10414 /* Check for the case where INSN does not have a valid luid. In this case,
10415 there is no need to modify the regno_last_uid, as this can only happen
10416 when code is inserted after the loop_end to set a pseudo's final value,
10417 and hence this insn will never be the last use of x.
10418 ???? This comment is not correct. See for example loop_givs_reduce.
10419 This may insert an insn before another new insn. */
10420 if (REG_P (x) && REGNO (x) < max_reg_before_loop
10421 && INSN_UID (insn) < max_uid_for_loop
10422 && REGNO_LAST_LUID (REGNO (x)) < INSN_LUID (insn))
10424 REGNO_LAST_UID (REGNO (x)) = INSN_UID (insn);
10426 else
10428 int i, j;
10429 const char *fmt = GET_RTX_FORMAT (GET_CODE (x));
10430 for (i = GET_RTX_LENGTH (GET_CODE (x)) - 1; i >= 0; i--)
10432 if (fmt[i] == 'e')
10433 update_reg_last_use (XEXP (x, i), insn);
10434 else if (fmt[i] == 'E')
10435 for (j = XVECLEN (x, i) - 1; j >= 0; j--)
10436 update_reg_last_use (XVECEXP (x, i, j), insn);
10441 /* Similar to rtlanal.c:get_condition, except that we also put an
10442 invariant last unless both operands are invariants. */
10444 static rtx
10445 get_condition_for_loop (const struct loop *loop, rtx x)
10447 rtx comparison = get_condition (x, (rtx*) 0, false, true);
10449 if (comparison == 0
10450 || ! loop_invariant_p (loop, XEXP (comparison, 0))
10451 || loop_invariant_p (loop, XEXP (comparison, 1)))
10452 return comparison;
10454 return gen_rtx_fmt_ee (swap_condition (GET_CODE (comparison)), VOIDmode,
10455 XEXP (comparison, 1), XEXP (comparison, 0));
10458 /* Scan the function and determine whether it has indirect (computed) jumps.
10460 This is taken mostly from flow.c; similar code exists elsewhere
10461 in the compiler. It may be useful to put this into rtlanal.c. */
10462 static int
10463 indirect_jump_in_function_p (rtx start)
10465 rtx insn;
10467 for (insn = start; insn; insn = NEXT_INSN (insn))
10468 if (computed_jump_p (insn))
10469 return 1;
10471 return 0;
10474 /* Add MEM to the LOOP_MEMS array, if appropriate. See the
10475 documentation for LOOP_MEMS for the definition of `appropriate'.
10476 This function is called from prescan_loop via for_each_rtx. */
10478 static int
10479 insert_loop_mem (rtx *mem, void *data ATTRIBUTE_UNUSED)
10481 struct loop_info *loop_info = data;
10482 int i;
10483 rtx m = *mem;
10485 if (m == NULL_RTX)
10486 return 0;
10488 switch (GET_CODE (m))
10490 case MEM:
10491 break;
10493 case CLOBBER:
10494 /* We're not interested in MEMs that are only clobbered. */
10495 return -1;
10497 case CONST_DOUBLE:
10498 /* We're not interested in the MEM associated with a
10499 CONST_DOUBLE, so there's no need to traverse into this. */
10500 return -1;
10502 case EXPR_LIST:
10503 /* We're not interested in any MEMs that only appear in notes. */
10504 return -1;
10506 default:
10507 /* This is not a MEM. */
10508 return 0;
10511 /* See if we've already seen this MEM. */
10512 for (i = 0; i < loop_info->mems_idx; ++i)
10513 if (rtx_equal_p (m, loop_info->mems[i].mem))
10515 if (MEM_VOLATILE_P (m) && !MEM_VOLATILE_P (loop_info->mems[i].mem))
10516 loop_info->mems[i].mem = m;
10517 if (GET_MODE (m) != GET_MODE (loop_info->mems[i].mem))
10518 /* The modes of the two memory accesses are different. If
10519 this happens, something tricky is going on, and we just
10520 don't optimize accesses to this MEM. */
10521 loop_info->mems[i].optimize = 0;
10523 return 0;
10526 /* Resize the array, if necessary. */
10527 if (loop_info->mems_idx == loop_info->mems_allocated)
10529 if (loop_info->mems_allocated != 0)
10530 loop_info->mems_allocated *= 2;
10531 else
10532 loop_info->mems_allocated = 32;
10534 loop_info->mems = xrealloc (loop_info->mems,
10535 loop_info->mems_allocated * sizeof (loop_mem_info));
10538 /* Actually insert the MEM. */
10539 loop_info->mems[loop_info->mems_idx].mem = m;
10540 /* We can't hoist this MEM out of the loop if it's a BLKmode MEM
10541 because we can't put it in a register. We still store it in the
10542 table, though, so that if we see the same address later, but in a
10543 non-BLK mode, we'll not think we can optimize it at that point. */
10544 loop_info->mems[loop_info->mems_idx].optimize = (GET_MODE (m) != BLKmode);
10545 loop_info->mems[loop_info->mems_idx].reg = NULL_RTX;
10546 ++loop_info->mems_idx;
10548 return 0;
10552 /* Allocate REGS->ARRAY or reallocate it if it is too small.
10554 Increment REGS->ARRAY[I].SET_IN_LOOP at the index I of each
10555 register that is modified by an insn between FROM and TO. If the
10556 value of an element of REGS->array[I].SET_IN_LOOP becomes 127 or
10557 more, stop incrementing it, to avoid overflow.
10559 Store in REGS->ARRAY[I].SINGLE_USAGE the single insn in which
10560 register I is used, if it is only used once. Otherwise, it is set
10561 to 0 (for no uses) or const0_rtx for more than one use. This
10562 parameter may be zero, in which case this processing is not done.
10564 Set REGS->ARRAY[I].MAY_NOT_OPTIMIZE nonzero if we should not
10565 optimize register I. */
10567 static void
10568 loop_regs_scan (const struct loop *loop, int extra_size)
10570 struct loop_regs *regs = LOOP_REGS (loop);
10571 int old_nregs;
10572 /* last_set[n] is nonzero iff reg n has been set in the current
10573 basic block. In that case, it is the insn that last set reg n. */
10574 rtx *last_set;
10575 rtx insn;
10576 int i;
10578 old_nregs = regs->num;
10579 regs->num = max_reg_num ();
10581 /* Grow the regs array if not allocated or too small. */
10582 if (regs->num >= regs->size)
10584 regs->size = regs->num + extra_size;
10586 regs->array = xrealloc (regs->array, regs->size * sizeof (*regs->array));
10588 /* Zero the new elements. */
10589 memset (regs->array + old_nregs, 0,
10590 (regs->size - old_nregs) * sizeof (*regs->array));
10593 /* Clear previously scanned fields but do not clear n_times_set. */
10594 for (i = 0; i < old_nregs; i++)
10596 regs->array[i].set_in_loop = 0;
10597 regs->array[i].may_not_optimize = 0;
10598 regs->array[i].single_usage = NULL_RTX;
10601 last_set = xcalloc (regs->num, sizeof (rtx));
10603 /* Scan the loop, recording register usage. */
10604 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
10605 insn = NEXT_INSN (insn))
10607 if (INSN_P (insn))
10609 /* Record registers that have exactly one use. */
10610 find_single_use_in_loop (regs, insn, PATTERN (insn));
10612 /* Include uses in REG_EQUAL notes. */
10613 if (REG_NOTES (insn))
10614 find_single_use_in_loop (regs, insn, REG_NOTES (insn));
10616 if (GET_CODE (PATTERN (insn)) == SET
10617 || GET_CODE (PATTERN (insn)) == CLOBBER)
10618 count_one_set (regs, insn, PATTERN (insn), last_set);
10619 else if (GET_CODE (PATTERN (insn)) == PARALLEL)
10621 int i;
10622 for (i = XVECLEN (PATTERN (insn), 0) - 1; i >= 0; i--)
10623 count_one_set (regs, insn, XVECEXP (PATTERN (insn), 0, i),
10624 last_set);
10628 if (LABEL_P (insn) || JUMP_P (insn))
10629 memset (last_set, 0, regs->num * sizeof (rtx));
10631 /* Invalidate all registers used for function argument passing.
10632 We check rtx_varies_p for the same reason as below, to allow
10633 optimizing PIC calculations. */
10634 if (CALL_P (insn))
10636 rtx link;
10637 for (link = CALL_INSN_FUNCTION_USAGE (insn);
10638 link;
10639 link = XEXP (link, 1))
10641 rtx op, reg;
10643 if (GET_CODE (op = XEXP (link, 0)) == USE
10644 && REG_P (reg = XEXP (op, 0))
10645 && rtx_varies_p (reg, 1))
10646 regs->array[REGNO (reg)].may_not_optimize = 1;
10651 /* Invalidate all hard registers clobbered by calls. With one exception:
10652 a call-clobbered PIC register is still function-invariant for our
10653 purposes, since we can hoist any PIC calculations out of the loop.
10654 Thus the call to rtx_varies_p. */
10655 if (LOOP_INFO (loop)->has_call)
10656 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
10657 if (TEST_HARD_REG_BIT (regs_invalidated_by_call, i)
10658 && rtx_varies_p (regno_reg_rtx[i], 1))
10660 regs->array[i].may_not_optimize = 1;
10661 regs->array[i].set_in_loop = 1;
10664 #ifdef AVOID_CCMODE_COPIES
10665 /* Don't try to move insns which set CC registers if we should not
10666 create CCmode register copies. */
10667 for (i = regs->num - 1; i >= FIRST_PSEUDO_REGISTER; i--)
10668 if (GET_MODE_CLASS (GET_MODE (regno_reg_rtx[i])) == MODE_CC)
10669 regs->array[i].may_not_optimize = 1;
10670 #endif
10672 /* Set regs->array[I].n_times_set for the new registers. */
10673 for (i = old_nregs; i < regs->num; i++)
10674 regs->array[i].n_times_set = regs->array[i].set_in_loop;
10676 free (last_set);
10679 /* Returns the number of real INSNs in the LOOP. */
10681 static int
10682 count_insns_in_loop (const struct loop *loop)
10684 int count = 0;
10685 rtx insn;
10687 for (insn = loop->top ? loop->top : loop->start; insn != loop->end;
10688 insn = NEXT_INSN (insn))
10689 if (INSN_P (insn))
10690 ++count;
10692 return count;
10695 /* Move MEMs into registers for the duration of the loop. */
10697 static void
10698 load_mems (const struct loop *loop)
10700 struct loop_info *loop_info = LOOP_INFO (loop);
10701 struct loop_regs *regs = LOOP_REGS (loop);
10702 int maybe_never = 0;
10703 int i;
10704 rtx p, prev_ebb_head;
10705 rtx label = NULL_RTX;
10706 rtx end_label;
10707 /* Nonzero if the next instruction may never be executed. */
10708 int next_maybe_never = 0;
10709 unsigned int last_max_reg = max_reg_num ();
10711 if (loop_info->mems_idx == 0)
10712 return;
10714 /* We cannot use next_label here because it skips over normal insns. */
10715 end_label = next_nonnote_insn (loop->end);
10716 if (end_label && !LABEL_P (end_label))
10717 end_label = NULL_RTX;
10719 /* Check to see if it's possible that some instructions in the loop are
10720 never executed. Also check if there is a goto out of the loop other
10721 than right after the end of the loop. */
10722 for (p = next_insn_in_loop (loop, loop->scan_start);
10723 p != NULL_RTX;
10724 p = next_insn_in_loop (loop, p))
10726 if (LABEL_P (p))
10727 maybe_never = 1;
10728 else if (JUMP_P (p)
10729 /* If we enter the loop in the middle, and scan
10730 around to the beginning, don't set maybe_never
10731 for that. This must be an unconditional jump,
10732 otherwise the code at the top of the loop might
10733 never be executed. Unconditional jumps are
10734 followed a by barrier then loop end. */
10735 && ! (JUMP_P (p)
10736 && JUMP_LABEL (p) == loop->top
10737 && NEXT_INSN (NEXT_INSN (p)) == loop->end
10738 && any_uncondjump_p (p)))
10740 /* If this is a jump outside of the loop but not right
10741 after the end of the loop, we would have to emit new fixup
10742 sequences for each such label. */
10743 if (/* If we can't tell where control might go when this
10744 JUMP_INSN is executed, we must be conservative. */
10745 !JUMP_LABEL (p)
10746 || (JUMP_LABEL (p) != end_label
10747 && (INSN_UID (JUMP_LABEL (p)) >= max_uid_for_loop
10748 || INSN_LUID (JUMP_LABEL (p)) < INSN_LUID (loop->start)
10749 || INSN_LUID (JUMP_LABEL (p)) > INSN_LUID (loop->end))))
10750 return;
10752 if (!any_condjump_p (p))
10753 /* Something complicated. */
10754 maybe_never = 1;
10755 else
10756 /* If there are any more instructions in the loop, they
10757 might not be reached. */
10758 next_maybe_never = 1;
10760 else if (next_maybe_never)
10761 maybe_never = 1;
10764 /* Find start of the extended basic block that enters the loop. */
10765 for (p = loop->start;
10766 PREV_INSN (p) && !LABEL_P (p);
10767 p = PREV_INSN (p))
10769 prev_ebb_head = p;
10771 cselib_init (true);
10773 /* Build table of mems that get set to constant values before the
10774 loop. */
10775 for (; p != loop->start; p = NEXT_INSN (p))
10776 cselib_process_insn (p);
10778 /* Actually move the MEMs. */
10779 for (i = 0; i < loop_info->mems_idx; ++i)
10781 regset_head load_copies;
10782 regset_head store_copies;
10783 int written = 0;
10784 rtx reg;
10785 rtx mem = loop_info->mems[i].mem;
10786 rtx mem_list_entry;
10788 if (MEM_VOLATILE_P (mem)
10789 || loop_invariant_p (loop, XEXP (mem, 0)) != 1)
10790 /* There's no telling whether or not MEM is modified. */
10791 loop_info->mems[i].optimize = 0;
10793 /* Go through the MEMs written to in the loop to see if this
10794 one is aliased by one of them. */
10795 mem_list_entry = loop_info->store_mems;
10796 while (mem_list_entry)
10798 if (rtx_equal_p (mem, XEXP (mem_list_entry, 0)))
10799 written = 1;
10800 else if (true_dependence (XEXP (mem_list_entry, 0), VOIDmode,
10801 mem, rtx_varies_p))
10803 /* MEM is indeed aliased by this store. */
10804 loop_info->mems[i].optimize = 0;
10805 break;
10807 mem_list_entry = XEXP (mem_list_entry, 1);
10810 if (flag_float_store && written
10811 && GET_MODE_CLASS (GET_MODE (mem)) == MODE_FLOAT)
10812 loop_info->mems[i].optimize = 0;
10814 /* If this MEM is written to, we must be sure that there
10815 are no reads from another MEM that aliases this one. */
10816 if (loop_info->mems[i].optimize && written)
10818 int j;
10820 for (j = 0; j < loop_info->mems_idx; ++j)
10822 if (j == i)
10823 continue;
10824 else if (true_dependence (mem,
10825 VOIDmode,
10826 loop_info->mems[j].mem,
10827 rtx_varies_p))
10829 /* It's not safe to hoist loop_info->mems[i] out of
10830 the loop because writes to it might not be
10831 seen by reads from loop_info->mems[j]. */
10832 loop_info->mems[i].optimize = 0;
10833 break;
10838 if (maybe_never && may_trap_p (mem))
10839 /* We can't access the MEM outside the loop; it might
10840 cause a trap that wouldn't have happened otherwise. */
10841 loop_info->mems[i].optimize = 0;
10843 if (!loop_info->mems[i].optimize)
10844 /* We thought we were going to lift this MEM out of the
10845 loop, but later discovered that we could not. */
10846 continue;
10848 INIT_REG_SET (&load_copies);
10849 INIT_REG_SET (&store_copies);
10851 /* Allocate a pseudo for this MEM. We set REG_USERVAR_P in
10852 order to keep scan_loop from moving stores to this MEM
10853 out of the loop just because this REG is neither a
10854 user-variable nor used in the loop test. */
10855 reg = gen_reg_rtx (GET_MODE (mem));
10856 REG_USERVAR_P (reg) = 1;
10857 loop_info->mems[i].reg = reg;
10859 /* Now, replace all references to the MEM with the
10860 corresponding pseudos. */
10861 maybe_never = 0;
10862 for (p = next_insn_in_loop (loop, loop->scan_start);
10863 p != NULL_RTX;
10864 p = next_insn_in_loop (loop, p))
10866 if (INSN_P (p))
10868 rtx set;
10870 set = single_set (p);
10872 /* See if this copies the mem into a register that isn't
10873 modified afterwards. We'll try to do copy propagation
10874 a little further on. */
10875 if (set
10876 /* @@@ This test is _way_ too conservative. */
10877 && ! maybe_never
10878 && REG_P (SET_DEST (set))
10879 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER
10880 && REGNO (SET_DEST (set)) < last_max_reg
10881 && regs->array[REGNO (SET_DEST (set))].n_times_set == 1
10882 && rtx_equal_p (SET_SRC (set), mem))
10883 SET_REGNO_REG_SET (&load_copies, REGNO (SET_DEST (set)));
10885 /* See if this copies the mem from a register that isn't
10886 modified afterwards. We'll try to remove the
10887 redundant copy later on by doing a little register
10888 renaming and copy propagation. This will help
10889 to untangle things for the BIV detection code. */
10890 if (set
10891 && ! maybe_never
10892 && REG_P (SET_SRC (set))
10893 && REGNO (SET_SRC (set)) >= FIRST_PSEUDO_REGISTER
10894 && REGNO (SET_SRC (set)) < last_max_reg
10895 && regs->array[REGNO (SET_SRC (set))].n_times_set == 1
10896 && rtx_equal_p (SET_DEST (set), mem))
10897 SET_REGNO_REG_SET (&store_copies, REGNO (SET_SRC (set)));
10899 /* If this is a call which uses / clobbers this memory
10900 location, we must not change the interface here. */
10901 if (CALL_P (p)
10902 && reg_mentioned_p (loop_info->mems[i].mem,
10903 CALL_INSN_FUNCTION_USAGE (p)))
10905 cancel_changes (0);
10906 loop_info->mems[i].optimize = 0;
10907 break;
10909 else
10910 /* Replace the memory reference with the shadow register. */
10911 replace_loop_mems (p, loop_info->mems[i].mem,
10912 loop_info->mems[i].reg, written);
10915 if (LABEL_P (p)
10916 || JUMP_P (p))
10917 maybe_never = 1;
10920 if (! loop_info->mems[i].optimize)
10921 ; /* We found we couldn't do the replacement, so do nothing. */
10922 else if (! apply_change_group ())
10923 /* We couldn't replace all occurrences of the MEM. */
10924 loop_info->mems[i].optimize = 0;
10925 else
10927 /* Load the memory immediately before LOOP->START, which is
10928 the NOTE_LOOP_BEG. */
10929 cselib_val *e = cselib_lookup (mem, VOIDmode, 0);
10930 rtx set;
10931 rtx best = mem;
10932 unsigned j;
10933 struct elt_loc_list *const_equiv = 0;
10934 reg_set_iterator rsi;
10936 if (e)
10938 struct elt_loc_list *equiv;
10939 struct elt_loc_list *best_equiv = 0;
10940 for (equiv = e->locs; equiv; equiv = equiv->next)
10942 if (CONSTANT_P (equiv->loc))
10943 const_equiv = equiv;
10944 else if (REG_P (equiv->loc)
10945 /* Extending hard register lifetimes causes crash
10946 on SRC targets. Doing so on non-SRC is
10947 probably also not good idea, since we most
10948 probably have pseudoregister equivalence as
10949 well. */
10950 && REGNO (equiv->loc) >= FIRST_PSEUDO_REGISTER)
10951 best_equiv = equiv;
10953 /* Use the constant equivalence if that is cheap enough. */
10954 if (! best_equiv)
10955 best_equiv = const_equiv;
10956 else if (const_equiv
10957 && (rtx_cost (const_equiv->loc, SET)
10958 <= rtx_cost (best_equiv->loc, SET)))
10960 best_equiv = const_equiv;
10961 const_equiv = 0;
10964 /* If best_equiv is nonzero, we know that MEM is set to a
10965 constant or register before the loop. We will use this
10966 knowledge to initialize the shadow register with that
10967 constant or reg rather than by loading from MEM. */
10968 if (best_equiv)
10969 best = copy_rtx (best_equiv->loc);
10972 set = gen_move_insn (reg, best);
10973 set = loop_insn_hoist (loop, set);
10974 if (REG_P (best))
10976 for (p = prev_ebb_head; p != loop->start; p = NEXT_INSN (p))
10977 if (REGNO_LAST_UID (REGNO (best)) == INSN_UID (p))
10979 REGNO_LAST_UID (REGNO (best)) = INSN_UID (set);
10980 break;
10984 if (const_equiv)
10985 set_unique_reg_note (set, REG_EQUAL, copy_rtx (const_equiv->loc));
10987 if (written)
10989 if (label == NULL_RTX)
10991 label = gen_label_rtx ();
10992 emit_label_after (label, loop->end);
10995 /* Store the memory immediately after END, which is
10996 the NOTE_LOOP_END. */
10997 set = gen_move_insn (copy_rtx (mem), reg);
10998 loop_insn_emit_after (loop, 0, label, set);
11001 if (loop_dump_stream)
11003 fprintf (loop_dump_stream, "Hoisted regno %d %s from ",
11004 REGNO (reg), (written ? "r/w" : "r/o"));
11005 print_rtl (loop_dump_stream, mem);
11006 fputc ('\n', loop_dump_stream);
11009 /* Attempt a bit of copy propagation. This helps untangle the
11010 data flow, and enables {basic,general}_induction_var to find
11011 more bivs/givs. */
11012 EXECUTE_IF_SET_IN_REG_SET
11013 (&load_copies, FIRST_PSEUDO_REGISTER, j, rsi)
11015 try_copy_prop (loop, reg, j);
11017 CLEAR_REG_SET (&load_copies);
11019 EXECUTE_IF_SET_IN_REG_SET
11020 (&store_copies, FIRST_PSEUDO_REGISTER, j, rsi)
11022 try_swap_copy_prop (loop, reg, j);
11024 CLEAR_REG_SET (&store_copies);
11028 /* Now, we need to replace all references to the previous exit
11029 label with the new one. */
11030 if (label != NULL_RTX && end_label != NULL_RTX)
11031 for (p = loop->start; p != loop->end; p = NEXT_INSN (p))
11032 if (JUMP_P (p) && JUMP_LABEL (p) == end_label)
11033 redirect_jump (p, label, false);
11035 cselib_finish ();
11038 /* For communication between note_reg_stored and its caller. */
11039 struct note_reg_stored_arg
11041 int set_seen;
11042 rtx reg;
11045 /* Called via note_stores, record in SET_SEEN whether X, which is written,
11046 is equal to ARG. */
11047 static void
11048 note_reg_stored (rtx x, rtx setter ATTRIBUTE_UNUSED, void *arg)
11050 struct note_reg_stored_arg *t = (struct note_reg_stored_arg *) arg;
11051 if (t->reg == x)
11052 t->set_seen = 1;
11055 /* Try to replace every occurrence of pseudo REGNO with REPLACEMENT.
11056 There must be exactly one insn that sets this pseudo; it will be
11057 deleted if all replacements succeed and we can prove that the register
11058 is not used after the loop. */
11060 static void
11061 try_copy_prop (const struct loop *loop, rtx replacement, unsigned int regno)
11063 /* This is the reg that we are copying from. */
11064 rtx reg_rtx = regno_reg_rtx[regno];
11065 rtx init_insn = 0;
11066 rtx insn;
11067 /* These help keep track of whether we replaced all uses of the reg. */
11068 int replaced_last = 0;
11069 int store_is_first = 0;
11071 for (insn = next_insn_in_loop (loop, loop->scan_start);
11072 insn != NULL_RTX;
11073 insn = next_insn_in_loop (loop, insn))
11075 rtx set;
11077 /* Only substitute within one extended basic block from the initializing
11078 insn. */
11079 if (LABEL_P (insn) && init_insn)
11080 break;
11082 if (! INSN_P (insn))
11083 continue;
11085 /* Is this the initializing insn? */
11086 set = single_set (insn);
11087 if (set
11088 && REG_P (SET_DEST (set))
11089 && REGNO (SET_DEST (set)) == regno)
11091 if (init_insn)
11092 abort ();
11094 init_insn = insn;
11095 if (REGNO_FIRST_UID (regno) == INSN_UID (insn))
11096 store_is_first = 1;
11099 /* Only substitute after seeing the initializing insn. */
11100 if (init_insn && insn != init_insn)
11102 struct note_reg_stored_arg arg;
11104 replace_loop_regs (insn, reg_rtx, replacement);
11105 if (REGNO_LAST_UID (regno) == INSN_UID (insn))
11106 replaced_last = 1;
11108 /* Stop replacing when REPLACEMENT is modified. */
11109 arg.reg = replacement;
11110 arg.set_seen = 0;
11111 note_stores (PATTERN (insn), note_reg_stored, &arg);
11112 if (arg.set_seen)
11114 rtx note = find_reg_note (insn, REG_EQUAL, NULL);
11116 /* It is possible that we've turned previously valid REG_EQUAL to
11117 invalid, as we change the REGNO to REPLACEMENT and unlike REGNO,
11118 REPLACEMENT is modified, we get different meaning. */
11119 if (note && reg_mentioned_p (replacement, XEXP (note, 0)))
11120 remove_note (insn, note);
11121 break;
11125 if (! init_insn)
11126 abort ();
11127 if (apply_change_group ())
11129 if (loop_dump_stream)
11130 fprintf (loop_dump_stream, " Replaced reg %d", regno);
11131 if (store_is_first && replaced_last)
11133 rtx first;
11134 rtx retval_note;
11136 /* Assume we're just deleting INIT_INSN. */
11137 first = init_insn;
11138 /* Look for REG_RETVAL note. If we're deleting the end of
11139 the libcall sequence, the whole sequence can go. */
11140 retval_note = find_reg_note (init_insn, REG_RETVAL, NULL_RTX);
11141 /* If we found a REG_RETVAL note, find the first instruction
11142 in the sequence. */
11143 if (retval_note)
11144 first = XEXP (retval_note, 0);
11146 /* Delete the instructions. */
11147 loop_delete_insns (first, init_insn);
11149 if (loop_dump_stream)
11150 fprintf (loop_dump_stream, ".\n");
11154 /* Replace all the instructions from FIRST up to and including LAST
11155 with NOTE_INSN_DELETED notes. */
11157 static void
11158 loop_delete_insns (rtx first, rtx last)
11160 while (1)
11162 if (loop_dump_stream)
11163 fprintf (loop_dump_stream, ", deleting init_insn (%d)",
11164 INSN_UID (first));
11165 delete_insn (first);
11167 /* If this was the LAST instructions we're supposed to delete,
11168 we're done. */
11169 if (first == last)
11170 break;
11172 first = NEXT_INSN (first);
11176 /* Try to replace occurrences of pseudo REGNO with REPLACEMENT within
11177 loop LOOP if the order of the sets of these registers can be
11178 swapped. There must be exactly one insn within the loop that sets
11179 this pseudo followed immediately by a move insn that sets
11180 REPLACEMENT with REGNO. */
11181 static void
11182 try_swap_copy_prop (const struct loop *loop, rtx replacement,
11183 unsigned int regno)
11185 rtx insn;
11186 rtx set = NULL_RTX;
11187 unsigned int new_regno;
11189 new_regno = REGNO (replacement);
11191 for (insn = next_insn_in_loop (loop, loop->scan_start);
11192 insn != NULL_RTX;
11193 insn = next_insn_in_loop (loop, insn))
11195 /* Search for the insn that copies REGNO to NEW_REGNO? */
11196 if (INSN_P (insn)
11197 && (set = single_set (insn))
11198 && REG_P (SET_DEST (set))
11199 && REGNO (SET_DEST (set)) == new_regno
11200 && REG_P (SET_SRC (set))
11201 && REGNO (SET_SRC (set)) == regno)
11202 break;
11205 if (insn != NULL_RTX)
11207 rtx prev_insn;
11208 rtx prev_set;
11210 /* Some DEF-USE info would come in handy here to make this
11211 function more general. For now, just check the previous insn
11212 which is the most likely candidate for setting REGNO. */
11214 prev_insn = PREV_INSN (insn);
11216 if (INSN_P (insn)
11217 && (prev_set = single_set (prev_insn))
11218 && REG_P (SET_DEST (prev_set))
11219 && REGNO (SET_DEST (prev_set)) == regno)
11221 /* We have:
11222 (set (reg regno) (expr))
11223 (set (reg new_regno) (reg regno))
11225 so try converting this to:
11226 (set (reg new_regno) (expr))
11227 (set (reg regno) (reg new_regno))
11229 The former construct is often generated when a global
11230 variable used for an induction variable is shadowed by a
11231 register (NEW_REGNO). The latter construct improves the
11232 chances of GIV replacement and BIV elimination. */
11234 validate_change (prev_insn, &SET_DEST (prev_set),
11235 replacement, 1);
11236 validate_change (insn, &SET_DEST (set),
11237 SET_SRC (set), 1);
11238 validate_change (insn, &SET_SRC (set),
11239 replacement, 1);
11241 if (apply_change_group ())
11243 if (loop_dump_stream)
11244 fprintf (loop_dump_stream,
11245 " Swapped set of reg %d at %d with reg %d at %d.\n",
11246 regno, INSN_UID (insn),
11247 new_regno, INSN_UID (prev_insn));
11249 /* Update first use of REGNO. */
11250 if (REGNO_FIRST_UID (regno) == INSN_UID (prev_insn))
11251 REGNO_FIRST_UID (regno) = INSN_UID (insn);
11253 /* Now perform copy propagation to hopefully
11254 remove all uses of REGNO within the loop. */
11255 try_copy_prop (loop, replacement, regno);
11261 /* Worker function for find_mem_in_note, called via for_each_rtx. */
11263 static int
11264 find_mem_in_note_1 (rtx *x, void *data)
11266 if (*x != NULL_RTX && MEM_P (*x))
11268 rtx *res = (rtx *) data;
11269 *res = *x;
11270 return 1;
11272 return 0;
11275 /* Returns the first MEM found in NOTE by depth-first search. */
11277 static rtx
11278 find_mem_in_note (rtx note)
11280 if (note && for_each_rtx (&note, find_mem_in_note_1, &note))
11281 return note;
11282 return NULL_RTX;
11285 /* Replace MEM with its associated pseudo register. This function is
11286 called from load_mems via for_each_rtx. DATA is actually a pointer
11287 to a structure describing the instruction currently being scanned
11288 and the MEM we are currently replacing. */
11290 static int
11291 replace_loop_mem (rtx *mem, void *data)
11293 loop_replace_args *args = (loop_replace_args *) data;
11294 rtx m = *mem;
11296 if (m == NULL_RTX)
11297 return 0;
11299 switch (GET_CODE (m))
11301 case MEM:
11302 break;
11304 case CONST_DOUBLE:
11305 /* We're not interested in the MEM associated with a
11306 CONST_DOUBLE, so there's no need to traverse into one. */
11307 return -1;
11309 default:
11310 /* This is not a MEM. */
11311 return 0;
11314 if (!rtx_equal_p (args->match, m))
11315 /* This is not the MEM we are currently replacing. */
11316 return 0;
11318 /* Actually replace the MEM. */
11319 validate_change (args->insn, mem, args->replacement, 1);
11321 return 0;
11324 static void
11325 replace_loop_mems (rtx insn, rtx mem, rtx reg, int written)
11327 loop_replace_args args;
11329 args.insn = insn;
11330 args.match = mem;
11331 args.replacement = reg;
11333 for_each_rtx (&insn, replace_loop_mem, &args);
11335 /* If we hoist a mem write out of the loop, then REG_EQUAL
11336 notes referring to the mem are no longer valid. */
11337 if (written)
11339 rtx note, sub;
11340 rtx *link;
11342 for (link = &REG_NOTES (insn); (note = *link); link = &XEXP (note, 1))
11344 if (REG_NOTE_KIND (note) == REG_EQUAL
11345 && (sub = find_mem_in_note (note))
11346 && true_dependence (mem, VOIDmode, sub, rtx_varies_p))
11348 /* Remove the note. */
11349 validate_change (NULL_RTX, link, XEXP (note, 1), 1);
11350 break;
11356 /* Replace one register with another. Called through for_each_rtx; PX points
11357 to the rtx being scanned. DATA is actually a pointer to
11358 a structure of arguments. */
11360 static int
11361 replace_loop_reg (rtx *px, void *data)
11363 rtx x = *px;
11364 loop_replace_args *args = (loop_replace_args *) data;
11366 if (x == NULL_RTX)
11367 return 0;
11369 if (x == args->match)
11370 validate_change (args->insn, px, args->replacement, 1);
11372 return 0;
11375 static void
11376 replace_loop_regs (rtx insn, rtx reg, rtx replacement)
11378 loop_replace_args args;
11380 args.insn = insn;
11381 args.match = reg;
11382 args.replacement = replacement;
11384 for_each_rtx (&insn, replace_loop_reg, &args);
11387 /* Emit insn for PATTERN after WHERE_INSN in basic block WHERE_BB
11388 (ignored in the interim). */
11390 static rtx
11391 loop_insn_emit_after (const struct loop *loop ATTRIBUTE_UNUSED,
11392 basic_block where_bb ATTRIBUTE_UNUSED, rtx where_insn,
11393 rtx pattern)
11395 return emit_insn_after (pattern, where_insn);
11399 /* If WHERE_INSN is nonzero emit insn for PATTERN before WHERE_INSN
11400 in basic block WHERE_BB (ignored in the interim) within the loop
11401 otherwise hoist PATTERN into the loop pre-header. */
11403 static rtx
11404 loop_insn_emit_before (const struct loop *loop,
11405 basic_block where_bb ATTRIBUTE_UNUSED,
11406 rtx where_insn, rtx pattern)
11408 if (! where_insn)
11409 return loop_insn_hoist (loop, pattern);
11410 return emit_insn_before (pattern, where_insn);
11414 /* Emit call insn for PATTERN before WHERE_INSN in basic block
11415 WHERE_BB (ignored in the interim) within the loop. */
11417 static rtx
11418 loop_call_insn_emit_before (const struct loop *loop ATTRIBUTE_UNUSED,
11419 basic_block where_bb ATTRIBUTE_UNUSED,
11420 rtx where_insn, rtx pattern)
11422 return emit_call_insn_before (pattern, where_insn);
11426 /* Hoist insn for PATTERN into the loop pre-header. */
11428 static rtx
11429 loop_insn_hoist (const struct loop *loop, rtx pattern)
11431 return loop_insn_emit_before (loop, 0, loop->start, pattern);
11435 /* Hoist call insn for PATTERN into the loop pre-header. */
11437 static rtx
11438 loop_call_insn_hoist (const struct loop *loop, rtx pattern)
11440 return loop_call_insn_emit_before (loop, 0, loop->start, pattern);
11444 /* Sink insn for PATTERN after the loop end. */
11446 static rtx
11447 loop_insn_sink (const struct loop *loop, rtx pattern)
11449 return loop_insn_emit_before (loop, 0, loop->sink, pattern);
11452 /* bl->final_value can be either general_operand or PLUS of general_operand
11453 and constant. Emit sequence of instructions to load it into REG. */
11454 static rtx
11455 gen_load_of_final_value (rtx reg, rtx final_value)
11457 rtx seq;
11458 start_sequence ();
11459 final_value = force_operand (final_value, reg);
11460 if (final_value != reg)
11461 emit_move_insn (reg, final_value);
11462 seq = get_insns ();
11463 end_sequence ();
11464 return seq;
11467 /* If the loop has multiple exits, emit insn for PATTERN before the
11468 loop to ensure that it will always be executed no matter how the
11469 loop exits. Otherwise, emit the insn for PATTERN after the loop,
11470 since this is slightly more efficient. */
11472 static rtx
11473 loop_insn_sink_or_swim (const struct loop *loop, rtx pattern)
11475 if (loop->exit_count)
11476 return loop_insn_hoist (loop, pattern);
11477 else
11478 return loop_insn_sink (loop, pattern);
11481 static void
11482 loop_ivs_dump (const struct loop *loop, FILE *file, int verbose)
11484 struct iv_class *bl;
11485 int iv_num = 0;
11487 if (! loop || ! file)
11488 return;
11490 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
11491 iv_num++;
11493 fprintf (file, "Loop %d: %d IV classes\n", loop->num, iv_num);
11495 for (bl = LOOP_IVS (loop)->list; bl; bl = bl->next)
11497 loop_iv_class_dump (bl, file, verbose);
11498 fputc ('\n', file);
11503 static void
11504 loop_iv_class_dump (const struct iv_class *bl, FILE *file,
11505 int verbose ATTRIBUTE_UNUSED)
11507 struct induction *v;
11508 rtx incr;
11509 int i;
11511 if (! bl || ! file)
11512 return;
11514 fprintf (file, "IV class for reg %d, benefit %d\n",
11515 bl->regno, bl->total_benefit);
11517 fprintf (file, " Init insn %d", INSN_UID (bl->init_insn));
11518 if (bl->initial_value)
11520 fprintf (file, ", init val: ");
11521 print_simple_rtl (file, bl->initial_value);
11523 if (bl->initial_test)
11525 fprintf (file, ", init test: ");
11526 print_simple_rtl (file, bl->initial_test);
11528 fputc ('\n', file);
11530 if (bl->final_value)
11532 fprintf (file, " Final val: ");
11533 print_simple_rtl (file, bl->final_value);
11534 fputc ('\n', file);
11537 if ((incr = biv_total_increment (bl)))
11539 fprintf (file, " Total increment: ");
11540 print_simple_rtl (file, incr);
11541 fputc ('\n', file);
11544 /* List the increments. */
11545 for (i = 0, v = bl->biv; v; v = v->next_iv, i++)
11547 fprintf (file, " Inc%d: insn %d, incr: ", i, INSN_UID (v->insn));
11548 print_simple_rtl (file, v->add_val);
11549 fputc ('\n', file);
11552 /* List the givs. */
11553 for (i = 0, v = bl->giv; v; v = v->next_iv, i++)
11555 fprintf (file, " Giv%d: insn %d, benefit %d, ",
11556 i, INSN_UID (v->insn), v->benefit);
11557 if (v->giv_type == DEST_ADDR)
11558 print_simple_rtl (file, v->mem);
11559 else
11560 print_simple_rtl (file, single_set (v->insn));
11561 fputc ('\n', file);
11566 static void
11567 loop_biv_dump (const struct induction *v, FILE *file, int verbose)
11569 if (! v || ! file)
11570 return;
11572 fprintf (file,
11573 "Biv %d: insn %d",
11574 REGNO (v->dest_reg), INSN_UID (v->insn));
11575 fprintf (file, " const ");
11576 print_simple_rtl (file, v->add_val);
11578 if (verbose && v->final_value)
11580 fputc ('\n', file);
11581 fprintf (file, " final ");
11582 print_simple_rtl (file, v->final_value);
11585 fputc ('\n', file);
11589 static void
11590 loop_giv_dump (const struct induction *v, FILE *file, int verbose)
11592 if (! v || ! file)
11593 return;
11595 if (v->giv_type == DEST_REG)
11596 fprintf (file, "Giv %d: insn %d",
11597 REGNO (v->dest_reg), INSN_UID (v->insn));
11598 else
11599 fprintf (file, "Dest address: insn %d",
11600 INSN_UID (v->insn));
11602 fprintf (file, " src reg %d benefit %d",
11603 REGNO (v->src_reg), v->benefit);
11604 fprintf (file, " lifetime %d",
11605 v->lifetime);
11607 if (v->replaceable)
11608 fprintf (file, " replaceable");
11610 if (v->no_const_addval)
11611 fprintf (file, " ncav");
11613 if (v->ext_dependent)
11615 switch (GET_CODE (v->ext_dependent))
11617 case SIGN_EXTEND:
11618 fprintf (file, " ext se");
11619 break;
11620 case ZERO_EXTEND:
11621 fprintf (file, " ext ze");
11622 break;
11623 case TRUNCATE:
11624 fprintf (file, " ext tr");
11625 break;
11626 default:
11627 abort ();
11631 fputc ('\n', file);
11632 fprintf (file, " mult ");
11633 print_simple_rtl (file, v->mult_val);
11635 fputc ('\n', file);
11636 fprintf (file, " add ");
11637 print_simple_rtl (file, v->add_val);
11639 if (verbose && v->final_value)
11641 fputc ('\n', file);
11642 fprintf (file, " final ");
11643 print_simple_rtl (file, v->final_value);
11646 fputc ('\n', file);
11650 void
11651 debug_ivs (const struct loop *loop)
11653 loop_ivs_dump (loop, stderr, 1);
11657 void
11658 debug_iv_class (const struct iv_class *bl)
11660 loop_iv_class_dump (bl, stderr, 1);
11664 void
11665 debug_biv (const struct induction *v)
11667 loop_biv_dump (v, stderr, 1);
11671 void
11672 debug_giv (const struct induction *v)
11674 loop_giv_dump (v, stderr, 1);
11678 #define LOOP_BLOCK_NUM_1(INSN) \
11679 ((INSN) ? (BLOCK_FOR_INSN (INSN) ? BLOCK_NUM (INSN) : - 1) : -1)
11681 /* The notes do not have an assigned block, so look at the next insn. */
11682 #define LOOP_BLOCK_NUM(INSN) \
11683 ((INSN) ? (NOTE_P (INSN) \
11684 ? LOOP_BLOCK_NUM_1 (next_nonnote_insn (INSN)) \
11685 : LOOP_BLOCK_NUM_1 (INSN)) \
11686 : -1)
11688 #define LOOP_INSN_UID(INSN) ((INSN) ? INSN_UID (INSN) : -1)
11690 static void
11691 loop_dump_aux (const struct loop *loop, FILE *file,
11692 int verbose ATTRIBUTE_UNUSED)
11694 rtx label;
11696 if (! loop || ! file || !BB_HEAD (loop->first))
11697 return;
11699 /* Print diagnostics to compare our concept of a loop with
11700 what the loop notes say. */
11701 if (! PREV_INSN (BB_HEAD (loop->first))
11702 || !NOTE_P (PREV_INSN (BB_HEAD (loop->first)))
11703 || NOTE_LINE_NUMBER (PREV_INSN (BB_HEAD (loop->first)))
11704 != NOTE_INSN_LOOP_BEG)
11705 fprintf (file, ";; No NOTE_INSN_LOOP_BEG at %d\n",
11706 INSN_UID (PREV_INSN (BB_HEAD (loop->first))));
11707 if (! NEXT_INSN (BB_END (loop->last))
11708 || !NOTE_P (NEXT_INSN (BB_END (loop->last)))
11709 || NOTE_LINE_NUMBER (NEXT_INSN (BB_END (loop->last)))
11710 != NOTE_INSN_LOOP_END)
11711 fprintf (file, ";; No NOTE_INSN_LOOP_END at %d\n",
11712 INSN_UID (NEXT_INSN (BB_END (loop->last))));
11714 if (loop->start)
11716 fprintf (file,
11717 ";; start %d (%d), end %d (%d)\n",
11718 LOOP_BLOCK_NUM (loop->start),
11719 LOOP_INSN_UID (loop->start),
11720 LOOP_BLOCK_NUM (loop->end),
11721 LOOP_INSN_UID (loop->end));
11722 fprintf (file, ";; top %d (%d), scan start %d (%d)\n",
11723 LOOP_BLOCK_NUM (loop->top),
11724 LOOP_INSN_UID (loop->top),
11725 LOOP_BLOCK_NUM (loop->scan_start),
11726 LOOP_INSN_UID (loop->scan_start));
11727 fprintf (file, ";; exit_count %d", loop->exit_count);
11728 if (loop->exit_count)
11730 fputs (", labels:", file);
11731 for (label = loop->exit_labels; label; label = LABEL_NEXTREF (label))
11733 fprintf (file, " %d ",
11734 LOOP_INSN_UID (XEXP (label, 0)));
11737 fputs ("\n", file);
11741 /* Call this function from the debugger to dump LOOP. */
11743 void
11744 debug_loop (const struct loop *loop)
11746 flow_loop_dump (loop, stderr, loop_dump_aux, 1);
11749 /* Call this function from the debugger to dump LOOPS. */
11751 void
11752 debug_loops (const struct loops *loops)
11754 flow_loops_dump (loops, stderr, loop_dump_aux, 1);