2 Copyright (C) 2004-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
26 #include "tree-pass.h"
28 #include "fold-const.h"
32 #include "tree-ssa-loop-niter.h"
33 #include "tree-ssa-loop.h"
34 #include "tree-into-ssa.h"
37 #include "tree-inline.h"
38 #include "gimple-iterator.h"
40 #include "tree-ssa-loop-manip.h"
42 /* This file implements the loop unswitching, i.e. transformation of loops like
55 where inv is the loop invariant, into
74 Inv is considered invariant iff the values it compares are both invariant;
75 tree-ssa-loop-im.c ensures that all the suitable conditions are in this
78 static struct loop
*tree_unswitch_loop (struct loop
*, basic_block
, tree
);
79 static bool tree_unswitch_single_loop (struct loop
*, int);
80 static tree
tree_may_unswitch_on (basic_block
, struct loop
*);
81 static bool tree_unswitch_outer_loop (struct loop
*);
82 static edge
find_loop_guard (struct loop
*);
83 static bool empty_bb_without_guard_p (struct loop
*, basic_block
);
84 static bool used_outside_loop_p (struct loop
*, tree
);
85 static void hoist_guard (struct loop
*, edge
);
86 static bool check_exit_phi (struct loop
*);
87 static tree
get_vop_from_header (struct loop
*);
89 /* Main entry point. Perform loop unswitching on all suitable loops. */
92 tree_ssa_unswitch_loops (void)
97 /* Go through all loops starting from innermost. */
98 FOR_EACH_LOOP (loop
, LI_FROM_INNERMOST
)
101 /* Unswitch innermost loop. */
102 changed
|= tree_unswitch_single_loop (loop
, 0);
104 changed
|= tree_unswitch_outer_loop (loop
);
108 return TODO_cleanup_cfg
;
112 /* Checks whether we can unswitch LOOP on condition at end of BB -- one of its
113 basic blocks (for what it means see comments below). */
116 tree_may_unswitch_on (basic_block bb
, struct loop
*loop
)
124 /* BB must end in a simple conditional jump. */
125 last
= last_stmt (bb
);
126 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
128 stmt
= as_a
<gcond
*> (last
);
130 /* To keep the things simple, we do not directly remove the conditions,
131 but just replace tests with 0 != 0 resp. 1 != 0. Prevent the infinite
132 loop where we would unswitch again on such a condition. */
133 if (gimple_cond_true_p (stmt
) || gimple_cond_false_p (stmt
))
136 /* Condition must be invariant. */
137 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
139 /* Unswitching on undefined values would introduce undefined
140 behavior that the original program might never exercise. */
141 if (ssa_undefined_value_p (use
, true))
143 def
= SSA_NAME_DEF_STMT (use
);
144 def_bb
= gimple_bb (def
);
146 && flow_bb_inside_loop_p (loop
, def_bb
))
150 cond
= build2 (gimple_cond_code (stmt
), boolean_type_node
,
151 gimple_cond_lhs (stmt
), gimple_cond_rhs (stmt
));
156 /* Simplifies COND using checks in front of the entry of the LOOP. Just very
157 simplish (sufficient to prevent us from duplicating loop in unswitching
161 simplify_using_entry_checks (struct loop
*loop
, tree cond
)
163 edge e
= loop_preheader_edge (loop
);
168 stmt
= last_stmt (e
->src
);
170 && gimple_code (stmt
) == GIMPLE_COND
171 && gimple_cond_code (stmt
) == TREE_CODE (cond
)
172 && operand_equal_p (gimple_cond_lhs (stmt
),
173 TREE_OPERAND (cond
, 0), 0)
174 && operand_equal_p (gimple_cond_rhs (stmt
),
175 TREE_OPERAND (cond
, 1), 0))
176 return (e
->flags
& EDGE_TRUE_VALUE
178 : boolean_false_node
);
180 if (!single_pred_p (e
->src
))
183 e
= single_pred_edge (e
->src
);
184 if (e
->src
== ENTRY_BLOCK_PTR_FOR_FN (cfun
))
189 /* Unswitch single LOOP. NUM is number of unswitchings done; we do not allow
190 it to grow too much, it is too easy to create example on that the code would
191 grow exponentially. */
194 tree_unswitch_single_loop (struct loop
*loop
, int num
)
199 tree cond
= NULL_TREE
;
201 bool changed
= false;
202 HOST_WIDE_INT iterations
;
204 /* Perform initial tests if unswitch is eligible. */
207 /* Do not unswitch in cold regions. */
208 if (optimize_loop_for_size_p (loop
))
210 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
211 fprintf (dump_file
, ";; Not unswitching cold loops\n");
215 /* The loop should not be too large, to limit code growth. */
216 if (tree_num_loop_insns (loop
, &eni_size_weights
)
217 > (unsigned) PARAM_VALUE (PARAM_MAX_UNSWITCH_INSNS
))
219 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
220 fprintf (dump_file
, ";; Not unswitching, loop too big\n");
224 /* If the loop is not expected to iterate, there is no need
226 iterations
= estimated_loop_iterations_int (loop
);
228 iterations
= likely_max_loop_iterations_int (loop
);
229 if (iterations
>= 0 && iterations
<= 1)
231 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
232 fprintf (dump_file
, ";; Not unswitching, loop is not expected"
239 bbs
= get_loop_body (loop
);
240 found
= loop
->num_nodes
;
244 /* Find a bb to unswitch on. */
245 for (; i
< loop
->num_nodes
; i
++)
246 if ((cond
= tree_may_unswitch_on (bbs
[i
], loop
)))
249 if (i
== loop
->num_nodes
)
252 && num
> PARAM_VALUE (PARAM_MAX_UNSWITCH_LEVEL
)
253 && (dump_flags
& TDF_DETAILS
))
254 fprintf (dump_file
, ";; Not unswitching anymore, hit max level\n");
256 if (found
== loop
->num_nodes
)
264 cond
= simplify_using_entry_checks (loop
, cond
);
265 stmt
= last_stmt (bbs
[i
]);
266 if (integer_nonzerop (cond
))
268 /* Remove false path. */
269 gimple_cond_set_condition_from_tree (as_a
<gcond
*> (stmt
),
273 else if (integer_zerop (cond
))
275 /* Remove true path. */
276 gimple_cond_set_condition_from_tree (as_a
<gcond
*> (stmt
),
280 /* Do not unswitch too much. */
281 else if (num
> PARAM_VALUE (PARAM_MAX_UNSWITCH_LEVEL
))
286 /* In nested tree_unswitch_single_loop first optimize all conditions
287 using entry checks, then discover still reachable blocks in the
288 loop and find the condition only among those still reachable bbs. */
291 if (found
== loop
->num_nodes
)
308 basic_block
*tos
, *worklist
;
310 /* When called recursively, first do a quick discovery
311 of reachable bbs after the above changes and only
312 consider conditions in still reachable bbs. */
313 tos
= worklist
= XNEWVEC (basic_block
, loop
->num_nodes
);
315 for (i
= 0; i
< loop
->num_nodes
; i
++)
316 bbs
[i
]->flags
&= ~BB_REACHABLE
;
318 /* Start with marking header. */
320 bbs
[0]->flags
|= BB_REACHABLE
;
322 /* Iterate: find everything reachable from what we've already seen
323 within the same innermost loop. Don't look through false edges
324 if condition is always true or true edges if condition is
326 while (tos
!= worklist
)
328 basic_block b
= *--tos
;
333 if (EDGE_COUNT (b
->succs
) == 2)
335 gimple
*stmt
= last_stmt (b
);
337 && gimple_code (stmt
) == GIMPLE_COND
)
339 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
340 if (gimple_cond_true_p (cond_stmt
))
341 flags
= EDGE_FALSE_VALUE
;
342 else if (gimple_cond_false_p (cond_stmt
))
343 flags
= EDGE_TRUE_VALUE
;
347 FOR_EACH_EDGE (e
, ei
, b
->succs
)
349 basic_block dest
= e
->dest
;
351 if (dest
->loop_father
== loop
352 && !(dest
->flags
& BB_REACHABLE
)
353 && !(e
->flags
& flags
))
356 dest
->flags
|= BB_REACHABLE
;
363 /* Find a bb to unswitch on. */
364 for (; found
< loop
->num_nodes
; found
++)
365 if ((bbs
[found
]->flags
& BB_REACHABLE
)
366 && (cond
= tree_may_unswitch_on (bbs
[found
], loop
)))
369 if (found
== loop
->num_nodes
)
376 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
377 fprintf (dump_file
, ";; Unswitching loop\n");
379 initialize_original_copy_tables ();
380 /* Unswitch the loop on this condition. */
381 nloop
= tree_unswitch_loop (loop
, bbs
[found
], cond
);
384 free_original_copy_tables ();
389 /* Update the SSA form after unswitching. */
390 update_ssa (TODO_update_ssa
);
391 free_original_copy_tables ();
393 /* Invoke itself on modified loops. */
394 tree_unswitch_single_loop (nloop
, num
+ 1);
395 tree_unswitch_single_loop (loop
, num
+ 1);
400 /* Unswitch a LOOP w.r. to given basic block UNSWITCH_ON. We only support
401 unswitching of innermost loops. COND is the condition determining which
402 loop is entered -- the new loop is entered if COND is true. Returns NULL
403 if impossible, new loop otherwise. */
406 tree_unswitch_loop (struct loop
*loop
,
407 basic_block unswitch_on
, tree cond
)
410 edge edge_true
, edge_false
;
412 /* Some sanity checking. */
413 gcc_assert (flow_bb_inside_loop_p (loop
, unswitch_on
));
414 gcc_assert (EDGE_COUNT (unswitch_on
->succs
) == 2);
415 gcc_assert (loop
->inner
== NULL
);
417 extract_true_false_edges_from_block (unswitch_on
, &edge_true
, &edge_false
);
418 prob_true
= edge_true
->probability
;
419 return loop_version (loop
, unshare_expr (cond
),
420 NULL
, prob_true
, prob_true
,
421 REG_BR_PROB_BASE
- prob_true
, false);
424 /* Unswitch outer loops by hoisting invariant guard on
425 inner loop without code duplication. */
427 tree_unswitch_outer_loop (struct loop
*loop
)
430 HOST_WIDE_INT iterations
;
432 gcc_assert (loop
->inner
);
433 if (loop
->inner
->next
)
435 /* Accept loops with single exit only which is not from inner loop. */
436 exit
= single_exit (loop
);
437 if (!exit
|| exit
->src
->loop_father
!= loop
)
439 /* Check that phi argument of exit edge is not defined inside loop. */
440 if (!check_exit_phi (loop
))
442 /* If the loop is not expected to iterate, there is no need
444 iterations
= estimated_loop_iterations_int (loop
);
446 iterations
= likely_max_loop_iterations_int (loop
);
447 if (iterations
>= 0 && iterations
<= 1)
449 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
450 fprintf (dump_file
, ";; Not unswitching, loop is not expected"
455 bool changed
= false;
456 while ((guard
= find_loop_guard (loop
)))
459 rewrite_virtuals_into_loop_closed_ssa (loop
);
460 hoist_guard (loop
, guard
);
466 /* Checks if the body of the LOOP is within an invariant guard. If this
467 is the case, returns the edge that jumps over the real body of the loop,
468 otherwise returns NULL. */
471 find_loop_guard (struct loop
*loop
)
473 basic_block header
= loop
->header
;
474 edge guard_edge
, te
, fe
;
475 basic_block
*body
= NULL
;
480 /* We check for the following situation:
489 nvar = phi(orig, bvar) ... for all variables changed in body;
499 1) cond1 is loop invariant
500 2) If cond1 is false, then the loop is essentially empty; i.e.,
501 a) nothing in something1, something2 and something3 has side
503 b) anything defined in something1, something2 and something3
504 is not used outside of the loop. */
509 if (single_succ_p (header
))
510 header
= single_succ (header
);
513 cond
= dyn_cast
<gcond
*> (last_stmt (header
));
516 extract_true_false_edges_from_block (header
, &te
, &fe
);
517 /* Make sure to skip earlier hoisted guards that are left
518 in place as if (true). */
519 if (gimple_cond_true_p (cond
))
521 else if (gimple_cond_false_p (cond
))
528 if (!flow_bb_inside_loop_p (loop
, te
->dest
)
529 || !flow_bb_inside_loop_p (loop
, fe
->dest
))
532 if (just_once_each_iteration_p (loop
, te
->dest
)
533 || (single_succ_p (te
->dest
)
534 && just_once_each_iteration_p (loop
, single_succ (te
->dest
))))
536 if (just_once_each_iteration_p (loop
, fe
->dest
))
540 else if (just_once_each_iteration_p (loop
, fe
->dest
)
541 || (single_succ_p (fe
->dest
)
542 && just_once_each_iteration_p (loop
, single_succ (fe
->dest
))))
547 /* Guard edge must skip inner loop. */
548 if (!dominated_by_p (CDI_DOMINATORS
, loop
->inner
->header
,
549 guard_edge
== fe
? te
->dest
: fe
->dest
))
551 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
552 fprintf (dump_file
, "Guard edge %d --> %d is not around the loop!\n",
553 guard_edge
->src
->index
, guard_edge
->dest
->index
);
556 if (guard_edge
->dest
== loop
->latch
)
558 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
559 fprintf (dump_file
, "Guard edge destination is loop latch.\n");
563 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
565 "Considering guard %d -> %d in loop %d\n",
566 guard_edge
->src
->index
, guard_edge
->dest
->index
, loop
->num
);
567 /* Check if condition operands do not have definitions inside loop since
568 any bb copying is not performed. */
569 FOR_EACH_SSA_TREE_OPERAND (use
, cond
, iter
, SSA_OP_USE
)
571 gimple
*def
= SSA_NAME_DEF_STMT (use
);
572 basic_block def_bb
= gimple_bb (def
);
574 && flow_bb_inside_loop_p (loop
, def_bb
))
576 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
577 fprintf (dump_file
, " guard operands have definitions"
583 body
= get_loop_body (loop
);
584 for (i
= 0; i
< loop
->num_nodes
; i
++)
586 basic_block bb
= body
[i
];
587 if (bb
->loop_father
!= loop
)
589 if (bb
->flags
& BB_IRREDUCIBLE_LOOP
)
591 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
592 fprintf (dump_file
, "Block %d is marked as irreducible in loop\n",
597 if (!empty_bb_without_guard_p (loop
, bb
))
599 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
600 fprintf (dump_file
, " block %d has side effects\n", bb
->index
);
606 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
607 fprintf (dump_file
, " suitable to hoist\n");
615 1) no statement in BB has side effects
616 2) assuming that edge GUARD is always taken, all definitions in BB
617 are noy used outside of the loop.
618 KNOWN_INVARIANTS is a set of ssa names we know to be invariant, and
619 PROCESSED is a set of ssa names for that we already tested whether they
620 are invariant or not. */
623 empty_bb_without_guard_p (struct loop
*loop
, basic_block bb
)
625 basic_block exit_bb
= single_exit (loop
)->src
;
626 bool may_be_used_outside
= (bb
== exit_bb
627 || !dominated_by_p (CDI_DOMINATORS
, bb
, exit_bb
));
631 /* Phi nodes do not have side effects, but their results might be used
632 outside of the loop. */
633 if (may_be_used_outside
)
635 for (gphi_iterator gsi
= gsi_start_phis (bb
);
636 !gsi_end_p (gsi
); gsi_next (&gsi
))
638 gphi
*phi
= gsi
.phi ();
639 name
= PHI_RESULT (phi
);
640 if (virtual_operand_p (name
))
643 if (used_outside_loop_p (loop
, name
))
648 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
);
649 !gsi_end_p (gsi
); gsi_next (&gsi
))
651 gimple
*stmt
= gsi_stmt (gsi
);
652 if (gimple_has_side_effects (stmt
))
655 if (gimple_vdef(stmt
))
658 FOR_EACH_SSA_TREE_OPERAND (name
, stmt
, op_iter
, SSA_OP_DEF
)
660 if (may_be_used_outside
661 && used_outside_loop_p (loop
, name
))
668 /* Return true if NAME is used outside of LOOP. */
671 used_outside_loop_p (struct loop
*loop
, tree name
)
676 FOR_EACH_IMM_USE_FAST (use
, it
, name
)
678 gimple
*stmt
= USE_STMT (use
);
679 if (!flow_bb_inside_loop_p (loop
, gimple_bb (stmt
)))
686 /* Return argument for loop preheader edge in header virtual phi if any. */
689 get_vop_from_header (struct loop
*loop
)
691 for (gphi_iterator gsi
= gsi_start_phis (loop
->header
);
692 !gsi_end_p (gsi
); gsi_next (&gsi
))
694 gphi
*phi
= gsi
.phi ();
695 if (!virtual_operand_p (gimple_phi_result (phi
)))
697 return PHI_ARG_DEF_FROM_EDGE (phi
, loop_preheader_edge (loop
));
702 /* Move the check of GUARD outside of LOOP. */
705 hoist_guard (struct loop
*loop
, edge guard
)
707 edge exit
= single_exit (loop
);
708 edge preh
= loop_preheader_edge (loop
);
709 basic_block pre_header
= preh
->src
;
711 edge te
, fe
, e
, new_edge
;
713 basic_block guard_bb
= guard
->src
;
714 gimple_stmt_iterator gsi
;
716 bool fix_dom_of_exit
;
717 gcond
*cond_stmt
, *new_cond_stmt
;
719 bb
= get_immediate_dominator (CDI_DOMINATORS
, exit
->dest
);
720 fix_dom_of_exit
= flow_bb_inside_loop_p (loop
, bb
);
721 gsi
= gsi_last_bb (guard_bb
);
722 stmt
= gsi_stmt (gsi
);
723 gcc_assert (gimple_code (stmt
) == GIMPLE_COND
);
724 cond_stmt
= as_a
<gcond
*> (stmt
);
725 extract_true_false_edges_from_block (guard_bb
, &te
, &fe
);
726 /* Insert guard to PRE_HEADER. */
727 if (!empty_block_p (pre_header
))
728 gsi
= gsi_last_bb (pre_header
);
730 gsi
= gsi_start_bb (pre_header
);
731 /* Create copy of COND_STMT. */
732 new_cond_stmt
= gimple_build_cond (gimple_cond_code (cond_stmt
),
733 gimple_cond_lhs (cond_stmt
),
734 gimple_cond_rhs (cond_stmt
),
735 NULL_TREE
, NULL_TREE
);
736 gsi_insert_after (&gsi
, new_cond_stmt
, GSI_NEW_STMT
);
737 /* Convert COND_STMT to true/false conditional. */
739 gimple_cond_make_false (cond_stmt
);
741 gimple_cond_make_true (cond_stmt
);
742 update_stmt (cond_stmt
);
743 /* Create new loop pre-header. */
744 e
= split_block (pre_header
, last_stmt (pre_header
));
745 gcc_assert (loop_preheader_edge (loop
)->src
== e
->dest
);
748 e
->flags
= EDGE_TRUE_VALUE
;
749 flags
|= EDGE_FALSE_VALUE
;
753 e
->flags
= EDGE_FALSE_VALUE
;
754 flags
|= EDGE_TRUE_VALUE
;
756 new_edge
= make_edge (pre_header
, exit
->dest
, flags
);
758 set_immediate_dominator (CDI_DOMINATORS
, exit
->dest
, pre_header
);
759 /* Add NEW_ADGE argument for all phi in post-header block. */
761 for (gphi_iterator gsi
= gsi_start_phis (bb
);
762 !gsi_end_p (gsi
); gsi_next (&gsi
))
764 gphi
*phi
= gsi
.phi ();
766 if (virtual_operand_p (gimple_phi_result (phi
)))
768 arg
= get_vop_from_header (loop
);
769 if (arg
== NULL_TREE
)
770 /* Use exit edge argument. */
771 arg
= PHI_ARG_DEF_FROM_EDGE (phi
, exit
);
772 add_phi_arg (phi
, arg
, new_edge
, UNKNOWN_LOCATION
);
776 /* Use exit edge argument. */
777 arg
= PHI_ARG_DEF_FROM_EDGE (phi
, exit
);
778 add_phi_arg (phi
, arg
, new_edge
, UNKNOWN_LOCATION
);
782 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
783 fprintf (dump_file
, " guard hoisted.\n");
786 /* Return true if phi argument for exit edge can be used
787 for edge around loop. */
790 check_exit_phi (struct loop
*loop
)
792 edge exit
= single_exit (loop
);
793 basic_block pre_header
= loop_preheader_edge (loop
)->src
;
795 for (gphi_iterator gsi
= gsi_start_phis (exit
->dest
);
796 !gsi_end_p (gsi
); gsi_next (&gsi
))
798 gphi
*phi
= gsi
.phi ();
802 if (virtual_operand_p (gimple_phi_result (phi
)))
804 arg
= PHI_ARG_DEF_FROM_EDGE (phi
, exit
);
805 if (TREE_CODE (arg
) != SSA_NAME
)
807 def
= SSA_NAME_DEF_STMT (arg
);
810 def_bb
= gimple_bb (def
);
813 if (!dominated_by_p (CDI_DOMINATORS
, pre_header
, def_bb
))
814 /* Definition inside loop! */
816 /* Check loop closed phi invariant. */
817 if (!flow_bb_inside_loop_p (def_bb
->loop_father
, pre_header
))
823 /* Loop unswitching pass. */
827 const pass_data pass_data_tree_unswitch
=
829 GIMPLE_PASS
, /* type */
830 "unswitch", /* name */
831 OPTGROUP_LOOP
, /* optinfo_flags */
832 TV_TREE_LOOP_UNSWITCH
, /* tv_id */
833 PROP_cfg
, /* properties_required */
834 0, /* properties_provided */
835 0, /* properties_destroyed */
836 0, /* todo_flags_start */
837 0, /* todo_flags_finish */
840 class pass_tree_unswitch
: public gimple_opt_pass
843 pass_tree_unswitch (gcc::context
*ctxt
)
844 : gimple_opt_pass (pass_data_tree_unswitch
, ctxt
)
847 /* opt_pass methods: */
848 virtual bool gate (function
*) { return flag_unswitch_loops
!= 0; }
849 virtual unsigned int execute (function
*);
851 }; // class pass_tree_unswitch
854 pass_tree_unswitch::execute (function
*fun
)
856 if (number_of_loops (fun
) <= 1)
859 return tree_ssa_unswitch_loops ();
865 make_pass_tree_unswitch (gcc::context
*ctxt
)
867 return new pass_tree_unswitch (ctxt
);