1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "basic-block.h"
33 #include "tree-ssa-alias.h"
34 #include "internal-fn.h"
35 #include "gimple-fold.h"
36 #include "gimple-expr.h"
40 #include "gimple-iterator.h"
41 #include "gimplify-me.h"
42 #include "gimple-walk.h"
43 #include "tree-iterator.h"
44 #include "tree-inline.h"
45 #include "langhooks.h"
46 #include "diagnostic-core.h"
47 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
52 #include "tree-ssanames.h"
53 #include "tree-into-ssa.h"
60 #include "tree-pass.h"
62 #include "splay-tree.h"
67 #include "gimple-low.h"
68 #include "tree-cfgcleanup.h"
69 #include "pretty-print.h"
71 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
207 static vec
<omp_context
*> taskreg_contexts
;
209 static void scan_omp (gimple_seq
*, omp_context
*);
210 static tree
scan_omp_1_op (tree
*, int *, void *);
212 #define WALK_SUBSTMTS \
216 case GIMPLE_EH_FILTER: \
217 case GIMPLE_TRANSACTION: \
218 /* The sub-statements for these should be walked. */ \
219 *handled_ops_p = false; \
222 /* Convenience function for calling scan_omp_1_op on tree operands. */
225 scan_omp_op (tree
*tp
, omp_context
*ctx
)
227 struct walk_stmt_info wi
;
229 memset (&wi
, 0, sizeof (wi
));
231 wi
.want_locations
= true;
233 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
236 static void lower_omp (gimple_seq
*, omp_context
*);
237 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
238 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
240 /* Find an OpenMP clause of type KIND within CLAUSES. */
243 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
245 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
246 if (OMP_CLAUSE_CODE (clauses
) == kind
)
252 /* Return true if CTX is for an omp parallel. */
255 is_parallel_ctx (omp_context
*ctx
)
257 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
261 /* Return true if CTX is for an omp task. */
264 is_task_ctx (omp_context
*ctx
)
266 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
270 /* Return true if CTX is for an omp parallel or omp task. */
273 is_taskreg_ctx (omp_context
*ctx
)
275 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
276 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
280 /* Return true if REGION is a combined parallel+workshare region. */
283 is_combined_parallel (struct omp_region
*region
)
285 return region
->is_combined_parallel
;
289 /* Extract the header elements of parallel loop FOR_STMT and store
293 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
294 struct omp_for_data_loop
*loops
)
296 tree t
, var
, *collapse_iter
, *collapse_count
;
297 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
298 struct omp_for_data_loop
*loop
;
300 struct omp_for_data_loop dummy_loop
;
301 location_t loc
= gimple_location (for_stmt
);
302 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
303 bool distribute
= gimple_omp_for_kind (for_stmt
)
304 == GF_OMP_FOR_KIND_DISTRIBUTE
;
306 fd
->for_stmt
= for_stmt
;
308 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
309 if (fd
->collapse
> 1)
312 fd
->loops
= &fd
->loop
;
314 fd
->have_nowait
= distribute
|| simd
;
315 fd
->have_ordered
= false;
316 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
317 fd
->chunk_size
= NULL_TREE
;
318 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
319 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
320 collapse_iter
= NULL
;
321 collapse_count
= NULL
;
323 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
324 switch (OMP_CLAUSE_CODE (t
))
326 case OMP_CLAUSE_NOWAIT
:
327 fd
->have_nowait
= true;
329 case OMP_CLAUSE_ORDERED
:
330 fd
->have_ordered
= true;
332 case OMP_CLAUSE_SCHEDULE
:
333 gcc_assert (!distribute
);
334 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
335 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
337 case OMP_CLAUSE_DIST_SCHEDULE
:
338 gcc_assert (distribute
);
339 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
341 case OMP_CLAUSE_COLLAPSE
:
342 if (fd
->collapse
> 1)
344 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
345 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
352 /* FIXME: for now map schedule(auto) to schedule(static).
353 There should be analysis to determine whether all iterations
354 are approximately the same amount of work (then schedule(static)
355 is best) or if it varies (then schedule(dynamic,N) is better). */
356 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
358 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
359 gcc_assert (fd
->chunk_size
== NULL
);
361 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
362 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
363 gcc_assert (fd
->chunk_size
== NULL
);
364 else if (fd
->chunk_size
== NULL
)
366 /* We only need to compute a default chunk size for ordered
367 static loops and dynamic loops. */
368 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
370 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
371 ? integer_zero_node
: integer_one_node
;
374 for (i
= 0; i
< fd
->collapse
; i
++)
376 if (fd
->collapse
== 1)
378 else if (loops
!= NULL
)
383 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
384 gcc_assert (SSA_VAR_P (loop
->v
));
385 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
386 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
387 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
388 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
390 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
391 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
392 switch (loop
->cond_code
)
398 gcc_assert (gimple_omp_for_kind (for_stmt
)
399 == GF_OMP_FOR_KIND_CILKSIMD
400 || (gimple_omp_for_kind (for_stmt
)
401 == GF_OMP_FOR_KIND_CILKFOR
));
404 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
405 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
407 loop
->n2
= fold_build2_loc (loc
,
408 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
409 build_int_cst (TREE_TYPE (loop
->n2
), 1));
410 loop
->cond_code
= LT_EXPR
;
413 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
414 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
416 loop
->n2
= fold_build2_loc (loc
,
417 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
418 build_int_cst (TREE_TYPE (loop
->n2
), 1));
419 loop
->cond_code
= GT_EXPR
;
425 t
= gimple_omp_for_incr (for_stmt
, i
);
426 gcc_assert (TREE_OPERAND (t
, 0) == var
);
427 switch (TREE_CODE (t
))
430 loop
->step
= TREE_OPERAND (t
, 1);
432 case POINTER_PLUS_EXPR
:
433 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
436 loop
->step
= TREE_OPERAND (t
, 1);
437 loop
->step
= fold_build1_loc (loc
,
438 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
446 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
447 && !fd
->have_ordered
))
449 if (fd
->collapse
== 1)
450 iter_type
= TREE_TYPE (loop
->v
);
452 || TYPE_PRECISION (iter_type
)
453 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
455 = build_nonstandard_integer_type
456 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
458 else if (iter_type
!= long_long_unsigned_type_node
)
460 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
461 iter_type
= long_long_unsigned_type_node
;
462 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
463 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
464 >= TYPE_PRECISION (iter_type
))
468 if (loop
->cond_code
== LT_EXPR
)
469 n
= fold_build2_loc (loc
,
470 PLUS_EXPR
, TREE_TYPE (loop
->v
),
471 loop
->n2
, loop
->step
);
474 if (TREE_CODE (n
) != INTEGER_CST
475 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
476 iter_type
= long_long_unsigned_type_node
;
478 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
479 > TYPE_PRECISION (iter_type
))
483 if (loop
->cond_code
== LT_EXPR
)
486 n2
= fold_build2_loc (loc
,
487 PLUS_EXPR
, TREE_TYPE (loop
->v
),
488 loop
->n2
, loop
->step
);
492 n1
= fold_build2_loc (loc
,
493 MINUS_EXPR
, TREE_TYPE (loop
->v
),
494 loop
->n2
, loop
->step
);
497 if (TREE_CODE (n1
) != INTEGER_CST
498 || TREE_CODE (n2
) != INTEGER_CST
499 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
500 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
501 iter_type
= long_long_unsigned_type_node
;
505 if (collapse_count
&& *collapse_count
== NULL
)
507 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
508 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
509 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
510 if (t
&& integer_zerop (t
))
511 count
= build_zero_cst (long_long_unsigned_type_node
);
512 else if ((i
== 0 || count
!= NULL_TREE
)
513 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
514 && TREE_CONSTANT (loop
->n1
)
515 && TREE_CONSTANT (loop
->n2
)
516 && TREE_CODE (loop
->step
) == INTEGER_CST
)
518 tree itype
= TREE_TYPE (loop
->v
);
520 if (POINTER_TYPE_P (itype
))
521 itype
= signed_type_for (itype
);
522 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
523 t
= fold_build2_loc (loc
,
525 fold_convert_loc (loc
, itype
, loop
->step
), t
);
526 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
527 fold_convert_loc (loc
, itype
, loop
->n2
));
528 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
529 fold_convert_loc (loc
, itype
, loop
->n1
));
530 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
531 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
532 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
533 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
534 fold_convert_loc (loc
, itype
,
537 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
538 fold_convert_loc (loc
, itype
, loop
->step
));
539 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
540 if (count
!= NULL_TREE
)
541 count
= fold_build2_loc (loc
,
542 MULT_EXPR
, long_long_unsigned_type_node
,
546 if (TREE_CODE (count
) != INTEGER_CST
)
549 else if (count
&& !integer_zerop (count
))
556 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
557 || fd
->have_ordered
))
559 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
560 iter_type
= long_long_unsigned_type_node
;
562 iter_type
= long_integer_type_node
;
564 else if (collapse_iter
&& *collapse_iter
!= NULL
)
565 iter_type
= TREE_TYPE (*collapse_iter
);
566 fd
->iter_type
= iter_type
;
567 if (collapse_iter
&& *collapse_iter
== NULL
)
568 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
569 if (collapse_count
&& *collapse_count
== NULL
)
572 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
574 *collapse_count
= create_tmp_var (iter_type
, ".count");
577 if (fd
->collapse
> 1)
579 fd
->loop
.v
= *collapse_iter
;
580 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
581 fd
->loop
.n2
= *collapse_count
;
582 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
583 fd
->loop
.cond_code
= LT_EXPR
;
588 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
589 is the immediate dominator of PAR_ENTRY_BB, return true if there
590 are no data dependencies that would prevent expanding the parallel
591 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
593 When expanding a combined parallel+workshare region, the call to
594 the child function may need additional arguments in the case of
595 GIMPLE_OMP_FOR regions. In some cases, these arguments are
596 computed out of variables passed in from the parent to the child
597 via 'struct .omp_data_s'. For instance:
599 #pragma omp parallel for schedule (guided, i * 4)
604 # BLOCK 2 (PAR_ENTRY_BB)
606 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
608 # BLOCK 3 (WS_ENTRY_BB)
609 .omp_data_i = &.omp_data_o;
610 D.1667 = .omp_data_i->i;
612 #pragma omp for schedule (guided, D.1598)
614 When we outline the parallel region, the call to the child function
615 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
616 that value is computed *after* the call site. So, in principle we
617 cannot do the transformation.
619 To see whether the code in WS_ENTRY_BB blocks the combined
620 parallel+workshare call, we collect all the variables used in the
621 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
622 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
625 FIXME. If we had the SSA form built at this point, we could merely
626 hoist the code in block 3 into block 2 and be done with it. But at
627 this point we don't have dataflow information and though we could
628 hack something up here, it is really not worth the aggravation. */
631 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
633 struct omp_for_data fd
;
634 gimple ws_stmt
= last_stmt (ws_entry_bb
);
636 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
639 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
641 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
643 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
645 if (fd
.iter_type
!= long_integer_type_node
)
648 /* FIXME. We give up too easily here. If any of these arguments
649 are not constants, they will likely involve variables that have
650 been mapped into fields of .omp_data_s for sharing with the child
651 function. With appropriate data flow, it would be possible to
653 if (!is_gimple_min_invariant (fd
.loop
.n1
)
654 || !is_gimple_min_invariant (fd
.loop
.n2
)
655 || !is_gimple_min_invariant (fd
.loop
.step
)
656 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
663 /* Collect additional arguments needed to emit a combined
664 parallel+workshare call. WS_STMT is the workshare directive being
667 static vec
<tree
, va_gc
> *
668 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
671 location_t loc
= gimple_location (ws_stmt
);
672 vec
<tree
, va_gc
> *ws_args
;
674 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
676 struct omp_for_data fd
;
679 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
683 if (gimple_omp_for_combined_into_p (ws_stmt
))
686 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
687 OMP_CLAUSE__LOOPTEMP_
);
689 n1
= OMP_CLAUSE_DECL (innerc
);
690 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
691 OMP_CLAUSE__LOOPTEMP_
);
693 n2
= OMP_CLAUSE_DECL (innerc
);
696 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
698 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
699 ws_args
->quick_push (t
);
701 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
702 ws_args
->quick_push (t
);
704 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
705 ws_args
->quick_push (t
);
709 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
710 ws_args
->quick_push (t
);
715 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
717 /* Number of sections is equal to the number of edges from the
718 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
719 the exit of the sections region. */
720 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
721 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
722 vec_alloc (ws_args
, 1);
723 ws_args
->quick_push (t
);
731 /* Discover whether REGION is a combined parallel+workshare region. */
734 determine_parallel_type (struct omp_region
*region
)
736 basic_block par_entry_bb
, par_exit_bb
;
737 basic_block ws_entry_bb
, ws_exit_bb
;
739 if (region
== NULL
|| region
->inner
== NULL
740 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
741 || region
->inner
->cont
== NULL
)
744 /* We only support parallel+for and parallel+sections. */
745 if (region
->type
!= GIMPLE_OMP_PARALLEL
746 || (region
->inner
->type
!= GIMPLE_OMP_FOR
747 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
750 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
751 WS_EXIT_BB -> PAR_EXIT_BB. */
752 par_entry_bb
= region
->entry
;
753 par_exit_bb
= region
->exit
;
754 ws_entry_bb
= region
->inner
->entry
;
755 ws_exit_bb
= region
->inner
->exit
;
757 if (single_succ (par_entry_bb
) == ws_entry_bb
758 && single_succ (ws_exit_bb
) == par_exit_bb
759 && workshare_safe_to_combine_p (ws_entry_bb
)
760 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
761 || (last_and_only_stmt (ws_entry_bb
)
762 && last_and_only_stmt (par_exit_bb
))))
764 gimple par_stmt
= last_stmt (par_entry_bb
);
765 gimple ws_stmt
= last_stmt (ws_entry_bb
);
767 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
769 /* If this is a combined parallel loop, we need to determine
770 whether or not to use the combined library calls. There
771 are two cases where we do not apply the transformation:
772 static loops and any kind of ordered loop. In the first
773 case, we already open code the loop so there is no need
774 to do anything else. In the latter case, the combined
775 parallel loop call would still need extra synchronization
776 to implement ordered semantics, so there would not be any
777 gain in using the combined call. */
778 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
779 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
781 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
782 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
784 region
->is_combined_parallel
= false;
785 region
->inner
->is_combined_parallel
= false;
790 region
->is_combined_parallel
= true;
791 region
->inner
->is_combined_parallel
= true;
792 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
797 /* Return true if EXPR is variable sized. */
800 is_variable_sized (const_tree expr
)
802 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
805 /* Return true if DECL is a reference type. */
808 is_reference (tree decl
)
810 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
813 /* Lookup variables in the decl or field splay trees. The "maybe" form
814 allows for the variable form to not have been entered, otherwise we
815 assert that the variable must have been entered. */
818 lookup_decl (tree var
, omp_context
*ctx
)
820 tree
*n
= ctx
->cb
.decl_map
->get (var
);
825 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
827 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
828 return n
? *n
: NULL_TREE
;
832 lookup_field (tree var
, omp_context
*ctx
)
835 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
836 return (tree
) n
->value
;
840 lookup_sfield (tree var
, omp_context
*ctx
)
843 n
= splay_tree_lookup (ctx
->sfield_map
844 ? ctx
->sfield_map
: ctx
->field_map
,
845 (splay_tree_key
) var
);
846 return (tree
) n
->value
;
850 maybe_lookup_field (tree var
, omp_context
*ctx
)
853 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
854 return n
? (tree
) n
->value
: NULL_TREE
;
857 /* Return true if DECL should be copied by pointer. SHARED_CTX is
858 the parallel context if DECL is to be shared. */
861 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
863 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
866 /* We can only use copy-in/copy-out semantics for shared variables
867 when we know the value is not accessible from an outer scope. */
870 /* ??? Trivially accessible from anywhere. But why would we even
871 be passing an address in this case? Should we simply assert
872 this to be false, or should we have a cleanup pass that removes
873 these from the list of mappings? */
874 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
877 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
878 without analyzing the expression whether or not its location
879 is accessible to anyone else. In the case of nested parallel
880 regions it certainly may be. */
881 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
884 /* Do not use copy-in/copy-out for variables that have their
886 if (TREE_ADDRESSABLE (decl
))
889 /* lower_send_shared_vars only uses copy-in, but not copy-out
891 if (TREE_READONLY (decl
)
892 || ((TREE_CODE (decl
) == RESULT_DECL
893 || TREE_CODE (decl
) == PARM_DECL
)
894 && DECL_BY_REFERENCE (decl
)))
897 /* Disallow copy-in/out in nested parallel if
898 decl is shared in outer parallel, otherwise
899 each thread could store the shared variable
900 in its own copy-in location, making the
901 variable no longer really shared. */
902 if (shared_ctx
->is_nested
)
906 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
907 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
914 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
915 c
; c
= OMP_CLAUSE_CHAIN (c
))
916 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
917 && OMP_CLAUSE_DECL (c
) == decl
)
921 goto maybe_mark_addressable_and_ret
;
925 /* For tasks avoid using copy-in/out. As tasks can be
926 deferred or executed in different thread, when GOMP_task
927 returns, the task hasn't necessarily terminated. */
928 if (is_task_ctx (shared_ctx
))
931 maybe_mark_addressable_and_ret
:
932 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
933 if (is_gimple_reg (outer
))
935 /* Taking address of OUTER in lower_send_shared_vars
936 might need regimplification of everything that uses the
938 if (!task_shared_vars
)
939 task_shared_vars
= BITMAP_ALLOC (NULL
);
940 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
941 TREE_ADDRESSABLE (outer
) = 1;
950 /* Construct a new automatic decl similar to VAR. */
953 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
955 tree copy
= copy_var_decl (var
, name
, type
);
957 DECL_CONTEXT (copy
) = current_function_decl
;
958 DECL_CHAIN (copy
) = ctx
->block_vars
;
959 ctx
->block_vars
= copy
;
965 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
967 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
970 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
973 omp_build_component_ref (tree obj
, tree field
)
975 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
976 if (TREE_THIS_VOLATILE (field
))
977 TREE_THIS_VOLATILE (ret
) |= 1;
978 if (TREE_READONLY (field
))
979 TREE_READONLY (ret
) |= 1;
983 /* Build tree nodes to access the field for VAR on the receiver side. */
986 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
988 tree x
, field
= lookup_field (var
, ctx
);
990 /* If the receiver record type was remapped in the child function,
991 remap the field into the new record type. */
992 x
= maybe_lookup_field (field
, ctx
);
996 x
= build_simple_mem_ref (ctx
->receiver_decl
);
997 x
= omp_build_component_ref (x
, field
);
999 x
= build_simple_mem_ref (x
);
1004 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1005 of a parallel, this is a component reference; for workshare constructs
1006 this is some variable. */
1009 build_outer_var_ref (tree var
, omp_context
*ctx
)
1013 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1015 else if (is_variable_sized (var
))
1017 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1018 x
= build_outer_var_ref (x
, ctx
);
1019 x
= build_simple_mem_ref (x
);
1021 else if (is_taskreg_ctx (ctx
))
1023 bool by_ref
= use_pointer_for_field (var
, NULL
);
1024 x
= build_receiver_ref (var
, by_ref
, ctx
);
1026 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1027 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1029 /* #pragma omp simd isn't a worksharing construct, and can reference even
1030 private vars in its linear etc. clauses. */
1032 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1033 x
= lookup_decl (var
, ctx
->outer
);
1034 else if (ctx
->outer
)
1035 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1039 else if (ctx
->outer
)
1040 x
= lookup_decl (var
, ctx
->outer
);
1041 else if (is_reference (var
))
1042 /* This can happen with orphaned constructs. If var is reference, it is
1043 possible it is shared and as such valid. */
1048 if (is_reference (var
))
1049 x
= build_simple_mem_ref (x
);
1054 /* Build tree nodes to access the field for VAR on the sender side. */
1057 build_sender_ref (tree var
, omp_context
*ctx
)
1059 tree field
= lookup_sfield (var
, ctx
);
1060 return omp_build_component_ref (ctx
->sender_decl
, field
);
1063 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1066 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1068 tree field
, type
, sfield
= NULL_TREE
;
1070 gcc_assert ((mask
& 1) == 0
1071 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1072 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1073 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1075 type
= TREE_TYPE (var
);
1078 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1079 type
= build_pointer_type (build_pointer_type (type
));
1082 type
= build_pointer_type (type
);
1083 else if ((mask
& 3) == 1 && is_reference (var
))
1084 type
= TREE_TYPE (type
);
1086 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1087 FIELD_DECL
, DECL_NAME (var
), type
);
1089 /* Remember what variable this field was created for. This does have a
1090 side effect of making dwarf2out ignore this member, so for helpful
1091 debugging we clear it later in delete_omp_context. */
1092 DECL_ABSTRACT_ORIGIN (field
) = var
;
1093 if (type
== TREE_TYPE (var
))
1095 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1096 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1097 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1100 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1102 if ((mask
& 3) == 3)
1104 insert_field_into_struct (ctx
->record_type
, field
);
1105 if (ctx
->srecord_type
)
1107 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1108 FIELD_DECL
, DECL_NAME (var
), type
);
1109 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1110 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1111 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1112 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1113 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1118 if (ctx
->srecord_type
== NULL_TREE
)
1122 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1123 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1124 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1126 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1127 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1128 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1129 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1130 splay_tree_insert (ctx
->sfield_map
,
1131 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1132 (splay_tree_value
) sfield
);
1136 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1137 : ctx
->srecord_type
, field
);
1141 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1142 (splay_tree_value
) field
);
1143 if ((mask
& 2) && ctx
->sfield_map
)
1144 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1145 (splay_tree_value
) sfield
);
1149 install_var_local (tree var
, omp_context
*ctx
)
1151 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1152 insert_decl_map (&ctx
->cb
, var
, new_var
);
1156 /* Adjust the replacement for DECL in CTX for the new context. This means
1157 copying the DECL_VALUE_EXPR, and fixing up the type. */
1160 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1162 tree new_decl
, size
;
1164 new_decl
= lookup_decl (decl
, ctx
);
1166 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1168 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1169 && DECL_HAS_VALUE_EXPR_P (decl
))
1171 tree ve
= DECL_VALUE_EXPR (decl
);
1172 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1173 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1174 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1177 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1179 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1180 if (size
== error_mark_node
)
1181 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1182 DECL_SIZE (new_decl
) = size
;
1184 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1185 if (size
== error_mark_node
)
1186 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1187 DECL_SIZE_UNIT (new_decl
) = size
;
1191 /* The callback for remap_decl. Search all containing contexts for a
1192 mapping of the variable; this avoids having to duplicate the splay
1193 tree ahead of time. We know a mapping doesn't already exist in the
1194 given context. Create new mappings to implement default semantics. */
1197 omp_copy_decl (tree var
, copy_body_data
*cb
)
1199 omp_context
*ctx
= (omp_context
*) cb
;
1202 if (TREE_CODE (var
) == LABEL_DECL
)
1204 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1205 DECL_CONTEXT (new_var
) = current_function_decl
;
1206 insert_decl_map (&ctx
->cb
, var
, new_var
);
1210 while (!is_taskreg_ctx (ctx
))
1215 new_var
= maybe_lookup_decl (var
, ctx
);
1220 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1223 return error_mark_node
;
1227 /* Debugging dumps for parallel regions. */
1228 void dump_omp_region (FILE *, struct omp_region
*, int);
1229 void debug_omp_region (struct omp_region
*);
1230 void debug_all_omp_regions (void);
1232 /* Dump the parallel region tree rooted at REGION. */
1235 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1237 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1238 gimple_code_name
[region
->type
]);
1241 dump_omp_region (file
, region
->inner
, indent
+ 4);
1245 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1246 region
->cont
->index
);
1250 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1251 region
->exit
->index
);
1253 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1256 dump_omp_region (file
, region
->next
, indent
);
1260 debug_omp_region (struct omp_region
*region
)
1262 dump_omp_region (stderr
, region
, 0);
1266 debug_all_omp_regions (void)
1268 dump_omp_region (stderr
, root_omp_region
, 0);
1272 /* Create a new parallel region starting at STMT inside region PARENT. */
1274 static struct omp_region
*
1275 new_omp_region (basic_block bb
, enum gimple_code type
,
1276 struct omp_region
*parent
)
1278 struct omp_region
*region
= XCNEW (struct omp_region
);
1280 region
->outer
= parent
;
1282 region
->type
= type
;
1286 /* This is a nested region. Add it to the list of inner
1287 regions in PARENT. */
1288 region
->next
= parent
->inner
;
1289 parent
->inner
= region
;
1293 /* This is a toplevel region. Add it to the list of toplevel
1294 regions in ROOT_OMP_REGION. */
1295 region
->next
= root_omp_region
;
1296 root_omp_region
= region
;
1302 /* Release the memory associated with the region tree rooted at REGION. */
1305 free_omp_region_1 (struct omp_region
*region
)
1307 struct omp_region
*i
, *n
;
1309 for (i
= region
->inner
; i
; i
= n
)
1312 free_omp_region_1 (i
);
1318 /* Release the memory for the entire omp region tree. */
1321 free_omp_regions (void)
1323 struct omp_region
*r
, *n
;
1324 for (r
= root_omp_region
; r
; r
= n
)
1327 free_omp_region_1 (r
);
1329 root_omp_region
= NULL
;
1333 /* Create a new context, with OUTER_CTX being the surrounding context. */
1335 static omp_context
*
1336 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1338 omp_context
*ctx
= XCNEW (omp_context
);
1340 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1341 (splay_tree_value
) ctx
);
1346 ctx
->outer
= outer_ctx
;
1347 ctx
->cb
= outer_ctx
->cb
;
1348 ctx
->cb
.block
= NULL
;
1349 ctx
->depth
= outer_ctx
->depth
+ 1;
1353 ctx
->cb
.src_fn
= current_function_decl
;
1354 ctx
->cb
.dst_fn
= current_function_decl
;
1355 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1356 gcc_checking_assert (ctx
->cb
.src_node
);
1357 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1358 ctx
->cb
.src_cfun
= cfun
;
1359 ctx
->cb
.copy_decl
= omp_copy_decl
;
1360 ctx
->cb
.eh_lp_nr
= 0;
1361 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1365 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1370 static gimple_seq
maybe_catch_exception (gimple_seq
);
1372 /* Finalize task copyfn. */
1375 finalize_task_copyfn (gimple task_stmt
)
1377 struct function
*child_cfun
;
1379 gimple_seq seq
= NULL
, new_seq
;
1382 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1383 if (child_fn
== NULL_TREE
)
1386 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1387 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1389 push_cfun (child_cfun
);
1390 bind
= gimplify_body (child_fn
, false);
1391 gimple_seq_add_stmt (&seq
, bind
);
1392 new_seq
= maybe_catch_exception (seq
);
1395 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1397 gimple_seq_add_stmt (&seq
, bind
);
1399 gimple_set_body (child_fn
, seq
);
1402 /* Inform the callgraph about the new function. */
1403 cgraph_node::add_new_function (child_fn
, false);
1406 /* Destroy a omp_context data structures. Called through the splay tree
1407 value delete callback. */
1410 delete_omp_context (splay_tree_value value
)
1412 omp_context
*ctx
= (omp_context
*) value
;
1414 delete ctx
->cb
.decl_map
;
1417 splay_tree_delete (ctx
->field_map
);
1418 if (ctx
->sfield_map
)
1419 splay_tree_delete (ctx
->sfield_map
);
1421 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1422 it produces corrupt debug information. */
1423 if (ctx
->record_type
)
1426 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1427 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1429 if (ctx
->srecord_type
)
1432 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1433 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1436 if (is_task_ctx (ctx
))
1437 finalize_task_copyfn (ctx
->stmt
);
1442 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1446 fixup_child_record_type (omp_context
*ctx
)
1448 tree f
, type
= ctx
->record_type
;
1450 /* ??? It isn't sufficient to just call remap_type here, because
1451 variably_modified_type_p doesn't work the way we expect for
1452 record types. Testing each field for whether it needs remapping
1453 and creating a new record by hand works, however. */
1454 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1455 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1459 tree name
, new_fields
= NULL
;
1461 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1462 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1463 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1464 TYPE_DECL
, name
, type
);
1465 TYPE_NAME (type
) = name
;
1467 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1469 tree new_f
= copy_node (f
);
1470 DECL_CONTEXT (new_f
) = type
;
1471 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1472 DECL_CHAIN (new_f
) = new_fields
;
1473 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1474 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1476 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1480 /* Arrange to be able to look up the receiver field
1481 given the sender field. */
1482 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1483 (splay_tree_value
) new_f
);
1485 TYPE_FIELDS (type
) = nreverse (new_fields
);
1489 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1492 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1493 specified by CLAUSES. */
1496 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1499 bool scan_array_reductions
= false;
1501 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1505 switch (OMP_CLAUSE_CODE (c
))
1507 case OMP_CLAUSE_PRIVATE
:
1508 decl
= OMP_CLAUSE_DECL (c
);
1509 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1511 else if (!is_variable_sized (decl
))
1512 install_var_local (decl
, ctx
);
1515 case OMP_CLAUSE_SHARED
:
1516 decl
= OMP_CLAUSE_DECL (c
);
1517 /* Ignore shared directives in teams construct. */
1518 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1520 /* Global variables don't need to be copied,
1521 the receiver side will use them directly. */
1522 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1523 if (is_global_var (odecl
))
1525 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1528 gcc_assert (is_taskreg_ctx (ctx
));
1529 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1530 || !is_variable_sized (decl
));
1531 /* Global variables don't need to be copied,
1532 the receiver side will use them directly. */
1533 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1535 by_ref
= use_pointer_for_field (decl
, ctx
);
1536 if (! TREE_READONLY (decl
)
1537 || TREE_ADDRESSABLE (decl
)
1539 || is_reference (decl
))
1541 install_var_field (decl
, by_ref
, 3, ctx
);
1542 install_var_local (decl
, ctx
);
1545 /* We don't need to copy const scalar vars back. */
1546 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1549 case OMP_CLAUSE_LASTPRIVATE
:
1550 /* Let the corresponding firstprivate clause create
1552 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1556 case OMP_CLAUSE_FIRSTPRIVATE
:
1557 case OMP_CLAUSE_REDUCTION
:
1558 case OMP_CLAUSE_LINEAR
:
1559 decl
= OMP_CLAUSE_DECL (c
);
1561 if (is_variable_sized (decl
))
1563 if (is_task_ctx (ctx
))
1564 install_var_field (decl
, false, 1, ctx
);
1567 else if (is_taskreg_ctx (ctx
))
1570 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1571 by_ref
= use_pointer_for_field (decl
, NULL
);
1573 if (is_task_ctx (ctx
)
1574 && (global
|| by_ref
|| is_reference (decl
)))
1576 install_var_field (decl
, false, 1, ctx
);
1578 install_var_field (decl
, by_ref
, 2, ctx
);
1581 install_var_field (decl
, by_ref
, 3, ctx
);
1583 install_var_local (decl
, ctx
);
1586 case OMP_CLAUSE__LOOPTEMP_
:
1587 gcc_assert (is_parallel_ctx (ctx
));
1588 decl
= OMP_CLAUSE_DECL (c
);
1589 install_var_field (decl
, false, 3, ctx
);
1590 install_var_local (decl
, ctx
);
1593 case OMP_CLAUSE_COPYPRIVATE
:
1594 case OMP_CLAUSE_COPYIN
:
1595 decl
= OMP_CLAUSE_DECL (c
);
1596 by_ref
= use_pointer_for_field (decl
, NULL
);
1597 install_var_field (decl
, by_ref
, 3, ctx
);
1600 case OMP_CLAUSE_DEFAULT
:
1601 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1604 case OMP_CLAUSE_FINAL
:
1606 case OMP_CLAUSE_NUM_THREADS
:
1607 case OMP_CLAUSE_NUM_TEAMS
:
1608 case OMP_CLAUSE_THREAD_LIMIT
:
1609 case OMP_CLAUSE_DEVICE
:
1610 case OMP_CLAUSE_SCHEDULE
:
1611 case OMP_CLAUSE_DIST_SCHEDULE
:
1612 case OMP_CLAUSE_DEPEND
:
1613 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1615 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1619 case OMP_CLAUSE_FROM
:
1620 case OMP_CLAUSE_MAP
:
1622 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1623 decl
= OMP_CLAUSE_DECL (c
);
1624 /* Global variables with "omp declare target" attribute
1625 don't need to be copied, the receiver side will use them
1627 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1629 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1630 && lookup_attribute ("omp declare target",
1631 DECL_ATTRIBUTES (decl
)))
1633 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1634 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1636 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1637 #pragma omp target data, there is nothing to map for
1639 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1640 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1645 if (DECL_SIZE (decl
)
1646 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1648 tree decl2
= DECL_VALUE_EXPR (decl
);
1649 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1650 decl2
= TREE_OPERAND (decl2
, 0);
1651 gcc_assert (DECL_P (decl2
));
1652 install_var_field (decl2
, true, 3, ctx
);
1653 install_var_local (decl2
, ctx
);
1654 install_var_local (decl
, ctx
);
1658 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1659 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1660 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1661 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1662 install_var_field (decl
, true, 7, ctx
);
1664 install_var_field (decl
, true, 3, ctx
);
1665 if (gimple_omp_target_kind (ctx
->stmt
)
1666 == GF_OMP_TARGET_KIND_REGION
)
1667 install_var_local (decl
, ctx
);
1672 tree base
= get_base_address (decl
);
1673 tree nc
= OMP_CLAUSE_CHAIN (c
);
1676 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1677 && OMP_CLAUSE_DECL (nc
) == base
1678 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1679 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1681 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1682 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1688 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1689 decl
= OMP_CLAUSE_DECL (c
);
1691 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1692 (splay_tree_key
) decl
));
1694 = build_decl (OMP_CLAUSE_LOCATION (c
),
1695 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1696 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1697 insert_field_into_struct (ctx
->record_type
, field
);
1698 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1699 (splay_tree_value
) field
);
1704 case OMP_CLAUSE_NOWAIT
:
1705 case OMP_CLAUSE_ORDERED
:
1706 case OMP_CLAUSE_COLLAPSE
:
1707 case OMP_CLAUSE_UNTIED
:
1708 case OMP_CLAUSE_MERGEABLE
:
1709 case OMP_CLAUSE_PROC_BIND
:
1710 case OMP_CLAUSE_SAFELEN
:
1713 case OMP_CLAUSE_ALIGNED
:
1714 decl
= OMP_CLAUSE_DECL (c
);
1715 if (is_global_var (decl
)
1716 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1717 install_var_local (decl
, ctx
);
1725 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1727 switch (OMP_CLAUSE_CODE (c
))
1729 case OMP_CLAUSE_LASTPRIVATE
:
1730 /* Let the corresponding firstprivate clause create
1732 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1733 scan_array_reductions
= true;
1734 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1738 case OMP_CLAUSE_PRIVATE
:
1739 case OMP_CLAUSE_FIRSTPRIVATE
:
1740 case OMP_CLAUSE_REDUCTION
:
1741 case OMP_CLAUSE_LINEAR
:
1742 decl
= OMP_CLAUSE_DECL (c
);
1743 if (is_variable_sized (decl
))
1744 install_var_local (decl
, ctx
);
1745 fixup_remapped_decl (decl
, ctx
,
1746 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1747 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1748 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1749 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1750 scan_array_reductions
= true;
1751 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1752 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1753 scan_array_reductions
= true;
1756 case OMP_CLAUSE_SHARED
:
1757 /* Ignore shared directives in teams construct. */
1758 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1760 decl
= OMP_CLAUSE_DECL (c
);
1761 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1762 fixup_remapped_decl (decl
, ctx
, false);
1765 case OMP_CLAUSE_MAP
:
1766 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1768 decl
= OMP_CLAUSE_DECL (c
);
1770 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1771 && lookup_attribute ("omp declare target",
1772 DECL_ATTRIBUTES (decl
)))
1776 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1777 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1778 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1780 tree new_decl
= lookup_decl (decl
, ctx
);
1781 TREE_TYPE (new_decl
)
1782 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1784 else if (DECL_SIZE (decl
)
1785 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1787 tree decl2
= DECL_VALUE_EXPR (decl
);
1788 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1789 decl2
= TREE_OPERAND (decl2
, 0);
1790 gcc_assert (DECL_P (decl2
));
1791 fixup_remapped_decl (decl2
, ctx
, false);
1792 fixup_remapped_decl (decl
, ctx
, true);
1795 fixup_remapped_decl (decl
, ctx
, false);
1799 case OMP_CLAUSE_COPYPRIVATE
:
1800 case OMP_CLAUSE_COPYIN
:
1801 case OMP_CLAUSE_DEFAULT
:
1803 case OMP_CLAUSE_NUM_THREADS
:
1804 case OMP_CLAUSE_NUM_TEAMS
:
1805 case OMP_CLAUSE_THREAD_LIMIT
:
1806 case OMP_CLAUSE_DEVICE
:
1807 case OMP_CLAUSE_SCHEDULE
:
1808 case OMP_CLAUSE_DIST_SCHEDULE
:
1809 case OMP_CLAUSE_NOWAIT
:
1810 case OMP_CLAUSE_ORDERED
:
1811 case OMP_CLAUSE_COLLAPSE
:
1812 case OMP_CLAUSE_UNTIED
:
1813 case OMP_CLAUSE_FINAL
:
1814 case OMP_CLAUSE_MERGEABLE
:
1815 case OMP_CLAUSE_PROC_BIND
:
1816 case OMP_CLAUSE_SAFELEN
:
1817 case OMP_CLAUSE_ALIGNED
:
1818 case OMP_CLAUSE_DEPEND
:
1819 case OMP_CLAUSE__LOOPTEMP_
:
1821 case OMP_CLAUSE_FROM
:
1822 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1830 if (scan_array_reductions
)
1831 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1832 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1833 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1835 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1836 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1838 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1839 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1840 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1841 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1842 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1843 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1846 /* Create a new name for omp child function. Returns an identifier. If
1847 IS_CILK_FOR is true then the suffix for the child function is
1851 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
1854 return clone_function_name (current_function_decl
, "_cilk_for_fn");
1855 return clone_function_name (current_function_decl
,
1856 task_copy
? "_omp_cpyfn" : "_omp_fn");
1859 /* Returns the type of the induction variable for the child function for
1860 _Cilk_for and the types for _high and _low variables based on TYPE. */
1863 cilk_for_check_loop_diff_type (tree type
)
1865 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
1867 if (TYPE_UNSIGNED (type
))
1868 return uint32_type_node
;
1870 return integer_type_node
;
1874 if (TYPE_UNSIGNED (type
))
1875 return uint64_type_node
;
1877 return long_long_integer_type_node
;
1881 /* Build a decl for the omp child function. It'll not contain a body
1882 yet, just the bare decl. */
1885 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1887 tree decl
, type
, name
, t
;
1890 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
1891 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
1892 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
1893 tree cilk_var_type
= NULL_TREE
;
1895 name
= create_omp_child_function_name (task_copy
,
1896 cilk_for_count
!= NULL_TREE
);
1898 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1899 ptr_type_node
, NULL_TREE
);
1900 else if (cilk_for_count
)
1902 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
1903 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
1904 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1905 cilk_var_type
, cilk_var_type
, NULL_TREE
);
1908 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1910 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
1913 ctx
->cb
.dst_fn
= decl
;
1915 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1917 TREE_STATIC (decl
) = 1;
1918 TREE_USED (decl
) = 1;
1919 DECL_ARTIFICIAL (decl
) = 1;
1920 DECL_IGNORED_P (decl
) = 0;
1921 TREE_PUBLIC (decl
) = 0;
1922 DECL_UNINLINABLE (decl
) = 1;
1923 DECL_EXTERNAL (decl
) = 0;
1924 DECL_CONTEXT (decl
) = NULL_TREE
;
1925 DECL_INITIAL (decl
) = make_node (BLOCK
);
1926 bool target_p
= false;
1927 if (lookup_attribute ("omp declare target",
1928 DECL_ATTRIBUTES (current_function_decl
)))
1933 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1934 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1935 && gimple_omp_target_kind (octx
->stmt
)
1936 == GF_OMP_TARGET_KIND_REGION
)
1943 DECL_ATTRIBUTES (decl
)
1944 = tree_cons (get_identifier ("omp declare target"),
1945 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1947 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1948 RESULT_DECL
, NULL_TREE
, void_type_node
);
1949 DECL_ARTIFICIAL (t
) = 1;
1950 DECL_IGNORED_P (t
) = 1;
1951 DECL_CONTEXT (t
) = decl
;
1952 DECL_RESULT (decl
) = t
;
1954 /* _Cilk_for's child function requires two extra parameters called
1955 __low and __high that are set the by Cilk runtime when it calls this
1959 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1960 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
1961 DECL_ARTIFICIAL (t
) = 1;
1962 DECL_NAMELESS (t
) = 1;
1963 DECL_ARG_TYPE (t
) = ptr_type_node
;
1964 DECL_CONTEXT (t
) = current_function_decl
;
1966 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1967 DECL_ARGUMENTS (decl
) = t
;
1969 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1970 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
1971 DECL_ARTIFICIAL (t
) = 1;
1972 DECL_NAMELESS (t
) = 1;
1973 DECL_ARG_TYPE (t
) = ptr_type_node
;
1974 DECL_CONTEXT (t
) = current_function_decl
;
1976 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1977 DECL_ARGUMENTS (decl
) = t
;
1980 tree data_name
= get_identifier (".omp_data_i");
1981 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
1983 DECL_ARTIFICIAL (t
) = 1;
1984 DECL_NAMELESS (t
) = 1;
1985 DECL_ARG_TYPE (t
) = ptr_type_node
;
1986 DECL_CONTEXT (t
) = current_function_decl
;
1989 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1990 DECL_ARGUMENTS (decl
) = t
;
1992 ctx
->receiver_decl
= t
;
1995 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1996 PARM_DECL
, get_identifier (".omp_data_o"),
1998 DECL_ARTIFICIAL (t
) = 1;
1999 DECL_NAMELESS (t
) = 1;
2000 DECL_ARG_TYPE (t
) = ptr_type_node
;
2001 DECL_CONTEXT (t
) = current_function_decl
;
2003 TREE_ADDRESSABLE (t
) = 1;
2004 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2005 DECL_ARGUMENTS (decl
) = t
;
2008 /* Allocate memory for the function structure. The call to
2009 allocate_struct_function clobbers CFUN, so we need to restore
2011 push_struct_function (decl
);
2012 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2016 /* Callback for walk_gimple_seq. Check if combined parallel
2017 contains gimple_omp_for_combined_into_p OMP_FOR. */
2020 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2021 bool *handled_ops_p
,
2022 struct walk_stmt_info
*wi
)
2024 gimple stmt
= gsi_stmt (*gsi_p
);
2026 *handled_ops_p
= true;
2027 switch (gimple_code (stmt
))
2031 case GIMPLE_OMP_FOR
:
2032 if (gimple_omp_for_combined_into_p (stmt
)
2033 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2036 return integer_zero_node
;
2045 /* Scan an OpenMP parallel directive. */
2048 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2052 gimple stmt
= gsi_stmt (*gsi
);
2054 /* Ignore parallel directives with empty bodies, unless there
2055 are copyin clauses. */
2057 && empty_body_p (gimple_omp_body (stmt
))
2058 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2059 OMP_CLAUSE_COPYIN
) == NULL
)
2061 gsi_replace (gsi
, gimple_build_nop (), false);
2065 if (gimple_omp_parallel_combined_p (stmt
))
2068 struct walk_stmt_info wi
;
2070 memset (&wi
, 0, sizeof (wi
));
2072 walk_gimple_seq (gimple_omp_body (stmt
),
2073 find_combined_for
, NULL
, &wi
);
2074 for_stmt
= (gimple
) wi
.info
;
2077 struct omp_for_data fd
;
2078 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2079 /* We need two temporaries with fd.loop.v type (istart/iend)
2080 and then (fd.collapse - 1) temporaries with the same
2081 type for count2 ... countN-1 vars if not constant. */
2082 size_t count
= 2, i
;
2083 tree type
= fd
.iter_type
;
2085 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2086 count
+= fd
.collapse
- 1;
2087 for (i
= 0; i
< count
; i
++)
2089 tree temp
= create_tmp_var (type
, NULL
);
2090 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2091 OMP_CLAUSE__LOOPTEMP_
);
2092 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2093 OMP_CLAUSE_DECL (c
) = temp
;
2094 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2095 gimple_omp_parallel_set_clauses (stmt
, c
);
2100 ctx
= new_omp_context (stmt
, outer_ctx
);
2101 taskreg_contexts
.safe_push (ctx
);
2102 if (taskreg_nesting_level
> 1)
2103 ctx
->is_nested
= true;
2104 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2105 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2106 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2107 name
= create_tmp_var_name (".omp_data_s");
2108 name
= build_decl (gimple_location (stmt
),
2109 TYPE_DECL
, name
, ctx
->record_type
);
2110 DECL_ARTIFICIAL (name
) = 1;
2111 DECL_NAMELESS (name
) = 1;
2112 TYPE_NAME (ctx
->record_type
) = name
;
2113 create_omp_child_function (ctx
, false);
2114 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2116 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2117 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2119 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2120 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2123 /* Scan an OpenMP task directive. */
2126 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2130 gimple stmt
= gsi_stmt (*gsi
);
2132 /* Ignore task directives with empty bodies. */
2134 && empty_body_p (gimple_omp_body (stmt
)))
2136 gsi_replace (gsi
, gimple_build_nop (), false);
2140 ctx
= new_omp_context (stmt
, outer_ctx
);
2141 taskreg_contexts
.safe_push (ctx
);
2142 if (taskreg_nesting_level
> 1)
2143 ctx
->is_nested
= true;
2144 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2145 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2146 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2147 name
= create_tmp_var_name (".omp_data_s");
2148 name
= build_decl (gimple_location (stmt
),
2149 TYPE_DECL
, name
, ctx
->record_type
);
2150 DECL_ARTIFICIAL (name
) = 1;
2151 DECL_NAMELESS (name
) = 1;
2152 TYPE_NAME (ctx
->record_type
) = name
;
2153 create_omp_child_function (ctx
, false);
2154 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2156 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2158 if (ctx
->srecord_type
)
2160 name
= create_tmp_var_name (".omp_data_a");
2161 name
= build_decl (gimple_location (stmt
),
2162 TYPE_DECL
, name
, ctx
->srecord_type
);
2163 DECL_ARTIFICIAL (name
) = 1;
2164 DECL_NAMELESS (name
) = 1;
2165 TYPE_NAME (ctx
->srecord_type
) = name
;
2166 create_omp_child_function (ctx
, true);
2169 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2171 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2173 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2174 t
= build_int_cst (long_integer_type_node
, 0);
2175 gimple_omp_task_set_arg_size (stmt
, t
);
2176 t
= build_int_cst (long_integer_type_node
, 1);
2177 gimple_omp_task_set_arg_align (stmt
, t
);
2182 /* If any decls have been made addressable during scan_omp,
2183 adjust their fields if needed, and layout record types
2184 of parallel/task constructs. */
2187 finish_taskreg_scan (omp_context
*ctx
)
2189 if (ctx
->record_type
== NULL_TREE
)
2192 /* If any task_shared_vars were needed, verify all
2193 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2194 statements if use_pointer_for_field hasn't changed
2195 because of that. If it did, update field types now. */
2196 if (task_shared_vars
)
2200 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2201 c
; c
= OMP_CLAUSE_CHAIN (c
))
2202 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2204 tree decl
= OMP_CLAUSE_DECL (c
);
2206 /* Global variables don't need to be copied,
2207 the receiver side will use them directly. */
2208 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2210 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2211 || !use_pointer_for_field (decl
, ctx
))
2213 tree field
= lookup_field (decl
, ctx
);
2214 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2215 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2217 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2218 TREE_THIS_VOLATILE (field
) = 0;
2219 DECL_USER_ALIGN (field
) = 0;
2220 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2221 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2222 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2223 if (ctx
->srecord_type
)
2225 tree sfield
= lookup_sfield (decl
, ctx
);
2226 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2227 TREE_THIS_VOLATILE (sfield
) = 0;
2228 DECL_USER_ALIGN (sfield
) = 0;
2229 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2230 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2231 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2236 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2238 layout_type (ctx
->record_type
);
2239 fixup_child_record_type (ctx
);
2243 location_t loc
= gimple_location (ctx
->stmt
);
2244 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2245 /* Move VLA fields to the end. */
2246 p
= &TYPE_FIELDS (ctx
->record_type
);
2248 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2249 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2252 *p
= TREE_CHAIN (*p
);
2253 TREE_CHAIN (*q
) = NULL_TREE
;
2254 q
= &TREE_CHAIN (*q
);
2257 p
= &DECL_CHAIN (*p
);
2259 layout_type (ctx
->record_type
);
2260 fixup_child_record_type (ctx
);
2261 if (ctx
->srecord_type
)
2262 layout_type (ctx
->srecord_type
);
2263 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2264 TYPE_SIZE_UNIT (ctx
->record_type
));
2265 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2266 t
= build_int_cst (long_integer_type_node
,
2267 TYPE_ALIGN_UNIT (ctx
->record_type
));
2268 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2273 /* Scan an OpenMP loop directive. */
2276 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2281 ctx
= new_omp_context (stmt
, outer_ctx
);
2283 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2285 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2286 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2288 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2289 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2290 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2291 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2293 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2296 /* Scan an OpenMP sections directive. */
2299 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2303 ctx
= new_omp_context (stmt
, outer_ctx
);
2304 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2305 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2308 /* Scan an OpenMP single directive. */
2311 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2316 ctx
= new_omp_context (stmt
, outer_ctx
);
2317 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2318 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2319 name
= create_tmp_var_name (".omp_copy_s");
2320 name
= build_decl (gimple_location (stmt
),
2321 TYPE_DECL
, name
, ctx
->record_type
);
2322 TYPE_NAME (ctx
->record_type
) = name
;
2324 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2325 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2327 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2328 ctx
->record_type
= NULL
;
2330 layout_type (ctx
->record_type
);
2333 /* Scan an OpenMP target{, data, update} directive. */
2336 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2340 int kind
= gimple_omp_target_kind (stmt
);
2342 ctx
= new_omp_context (stmt
, outer_ctx
);
2343 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2344 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2345 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2346 name
= create_tmp_var_name (".omp_data_t");
2347 name
= build_decl (gimple_location (stmt
),
2348 TYPE_DECL
, name
, ctx
->record_type
);
2349 DECL_ARTIFICIAL (name
) = 1;
2350 DECL_NAMELESS (name
) = 1;
2351 TYPE_NAME (ctx
->record_type
) = name
;
2352 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2354 create_omp_child_function (ctx
, false);
2355 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2358 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2359 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2361 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2362 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2365 TYPE_FIELDS (ctx
->record_type
)
2366 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2367 #ifdef ENABLE_CHECKING
2369 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2370 for (field
= TYPE_FIELDS (ctx
->record_type
);
2372 field
= DECL_CHAIN (field
))
2373 gcc_assert (DECL_ALIGN (field
) == align
);
2375 layout_type (ctx
->record_type
);
2376 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2377 fixup_child_record_type (ctx
);
2381 /* Scan an OpenMP teams directive. */
2384 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2386 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2387 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2388 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2391 /* Check OpenMP nesting restrictions. */
2393 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2397 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2398 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2400 error_at (gimple_location (stmt
),
2401 "OpenMP constructs may not be nested inside simd region");
2404 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2406 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2407 || (gimple_omp_for_kind (stmt
)
2408 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2409 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2411 error_at (gimple_location (stmt
),
2412 "only distribute or parallel constructs are allowed to "
2413 "be closely nested inside teams construct");
2418 switch (gimple_code (stmt
))
2420 case GIMPLE_OMP_FOR
:
2421 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2423 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2425 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2427 error_at (gimple_location (stmt
),
2428 "distribute construct must be closely nested inside "
2436 if (is_gimple_call (stmt
)
2437 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2438 == BUILT_IN_GOMP_CANCEL
2439 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2440 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2442 const char *bad
= NULL
;
2443 const char *kind
= NULL
;
2446 error_at (gimple_location (stmt
), "orphaned %qs construct",
2447 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2448 == BUILT_IN_GOMP_CANCEL
2449 ? "#pragma omp cancel"
2450 : "#pragma omp cancellation point");
2453 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2454 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2458 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2459 bad
= "#pragma omp parallel";
2460 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2461 == BUILT_IN_GOMP_CANCEL
2462 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2463 ctx
->cancellable
= true;
2467 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2468 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2469 bad
= "#pragma omp for";
2470 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2471 == BUILT_IN_GOMP_CANCEL
2472 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2474 ctx
->cancellable
= true;
2475 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2477 warning_at (gimple_location (stmt
), 0,
2478 "%<#pragma omp cancel for%> inside "
2479 "%<nowait%> for construct");
2480 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2481 OMP_CLAUSE_ORDERED
))
2482 warning_at (gimple_location (stmt
), 0,
2483 "%<#pragma omp cancel for%> inside "
2484 "%<ordered%> for construct");
2489 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2490 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2491 bad
= "#pragma omp sections";
2492 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2493 == BUILT_IN_GOMP_CANCEL
2494 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2496 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2498 ctx
->cancellable
= true;
2499 if (find_omp_clause (gimple_omp_sections_clauses
2502 warning_at (gimple_location (stmt
), 0,
2503 "%<#pragma omp cancel sections%> inside "
2504 "%<nowait%> sections construct");
2508 gcc_assert (ctx
->outer
2509 && gimple_code (ctx
->outer
->stmt
)
2510 == GIMPLE_OMP_SECTIONS
);
2511 ctx
->outer
->cancellable
= true;
2512 if (find_omp_clause (gimple_omp_sections_clauses
2515 warning_at (gimple_location (stmt
), 0,
2516 "%<#pragma omp cancel sections%> inside "
2517 "%<nowait%> sections construct");
2523 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2524 bad
= "#pragma omp task";
2526 ctx
->cancellable
= true;
2530 error_at (gimple_location (stmt
), "invalid arguments");
2535 error_at (gimple_location (stmt
),
2536 "%<%s %s%> construct not closely nested inside of %qs",
2537 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2538 == BUILT_IN_GOMP_CANCEL
2539 ? "#pragma omp cancel"
2540 : "#pragma omp cancellation point", kind
, bad
);
2545 case GIMPLE_OMP_SECTIONS
:
2546 case GIMPLE_OMP_SINGLE
:
2547 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2548 switch (gimple_code (ctx
->stmt
))
2550 case GIMPLE_OMP_FOR
:
2551 case GIMPLE_OMP_SECTIONS
:
2552 case GIMPLE_OMP_SINGLE
:
2553 case GIMPLE_OMP_ORDERED
:
2554 case GIMPLE_OMP_MASTER
:
2555 case GIMPLE_OMP_TASK
:
2556 case GIMPLE_OMP_CRITICAL
:
2557 if (is_gimple_call (stmt
))
2559 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2560 != BUILT_IN_GOMP_BARRIER
)
2562 error_at (gimple_location (stmt
),
2563 "barrier region may not be closely nested inside "
2564 "of work-sharing, critical, ordered, master or "
2565 "explicit task region");
2568 error_at (gimple_location (stmt
),
2569 "work-sharing region may not be closely nested inside "
2570 "of work-sharing, critical, ordered, master or explicit "
2573 case GIMPLE_OMP_PARALLEL
:
2579 case GIMPLE_OMP_MASTER
:
2580 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2581 switch (gimple_code (ctx
->stmt
))
2583 case GIMPLE_OMP_FOR
:
2584 case GIMPLE_OMP_SECTIONS
:
2585 case GIMPLE_OMP_SINGLE
:
2586 case GIMPLE_OMP_TASK
:
2587 error_at (gimple_location (stmt
),
2588 "master region may not be closely nested inside "
2589 "of work-sharing or explicit task region");
2591 case GIMPLE_OMP_PARALLEL
:
2597 case GIMPLE_OMP_ORDERED
:
2598 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2599 switch (gimple_code (ctx
->stmt
))
2601 case GIMPLE_OMP_CRITICAL
:
2602 case GIMPLE_OMP_TASK
:
2603 error_at (gimple_location (stmt
),
2604 "ordered region may not be closely nested inside "
2605 "of critical or explicit task region");
2607 case GIMPLE_OMP_FOR
:
2608 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2609 OMP_CLAUSE_ORDERED
) == NULL
)
2611 error_at (gimple_location (stmt
),
2612 "ordered region must be closely nested inside "
2613 "a loop region with an ordered clause");
2617 case GIMPLE_OMP_PARALLEL
:
2618 error_at (gimple_location (stmt
),
2619 "ordered region must be closely nested inside "
2620 "a loop region with an ordered clause");
2626 case GIMPLE_OMP_CRITICAL
:
2627 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2628 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2629 && (gimple_omp_critical_name (stmt
)
2630 == gimple_omp_critical_name (ctx
->stmt
)))
2632 error_at (gimple_location (stmt
),
2633 "critical region may not be nested inside a critical "
2634 "region with the same name");
2638 case GIMPLE_OMP_TEAMS
:
2640 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2641 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2643 error_at (gimple_location (stmt
),
2644 "teams construct not closely nested inside of target "
2649 case GIMPLE_OMP_TARGET
:
2650 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2651 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2652 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2655 switch (gimple_omp_target_kind (stmt
))
2657 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2658 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2659 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2660 default: gcc_unreachable ();
2662 warning_at (gimple_location (stmt
), 0,
2663 "%s construct inside of target region", name
);
2673 /* Helper function scan_omp.
2675 Callback for walk_tree or operators in walk_gimple_stmt used to
2676 scan for OpenMP directives in TP. */
2679 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2681 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2682 omp_context
*ctx
= (omp_context
*) wi
->info
;
2685 switch (TREE_CODE (t
))
2692 *tp
= remap_decl (t
, &ctx
->cb
);
2696 if (ctx
&& TYPE_P (t
))
2697 *tp
= remap_type (t
, &ctx
->cb
);
2698 else if (!DECL_P (t
))
2703 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2704 if (tem
!= TREE_TYPE (t
))
2706 if (TREE_CODE (t
) == INTEGER_CST
)
2707 *tp
= wide_int_to_tree (tem
, t
);
2709 TREE_TYPE (t
) = tem
;
2719 /* Return true if FNDECL is a setjmp or a longjmp. */
2722 setjmp_or_longjmp_p (const_tree fndecl
)
2724 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2725 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2726 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2729 tree declname
= DECL_NAME (fndecl
);
2732 const char *name
= IDENTIFIER_POINTER (declname
);
2733 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2737 /* Helper function for scan_omp.
2739 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2740 the current statement in GSI. */
2743 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2744 struct walk_stmt_info
*wi
)
2746 gimple stmt
= gsi_stmt (*gsi
);
2747 omp_context
*ctx
= (omp_context
*) wi
->info
;
2749 if (gimple_has_location (stmt
))
2750 input_location
= gimple_location (stmt
);
2752 /* Check the OpenMP nesting restrictions. */
2753 bool remove
= false;
2754 if (is_gimple_omp (stmt
))
2755 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2756 else if (is_gimple_call (stmt
))
2758 tree fndecl
= gimple_call_fndecl (stmt
);
2761 if (setjmp_or_longjmp_p (fndecl
)
2763 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2764 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2767 error_at (gimple_location (stmt
),
2768 "setjmp/longjmp inside simd construct");
2770 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2771 switch (DECL_FUNCTION_CODE (fndecl
))
2773 case BUILT_IN_GOMP_BARRIER
:
2774 case BUILT_IN_GOMP_CANCEL
:
2775 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2776 case BUILT_IN_GOMP_TASKYIELD
:
2777 case BUILT_IN_GOMP_TASKWAIT
:
2778 case BUILT_IN_GOMP_TASKGROUP_START
:
2779 case BUILT_IN_GOMP_TASKGROUP_END
:
2780 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2789 stmt
= gimple_build_nop ();
2790 gsi_replace (gsi
, stmt
, false);
2793 *handled_ops_p
= true;
2795 switch (gimple_code (stmt
))
2797 case GIMPLE_OMP_PARALLEL
:
2798 taskreg_nesting_level
++;
2799 scan_omp_parallel (gsi
, ctx
);
2800 taskreg_nesting_level
--;
2803 case GIMPLE_OMP_TASK
:
2804 taskreg_nesting_level
++;
2805 scan_omp_task (gsi
, ctx
);
2806 taskreg_nesting_level
--;
2809 case GIMPLE_OMP_FOR
:
2810 scan_omp_for (stmt
, ctx
);
2813 case GIMPLE_OMP_SECTIONS
:
2814 scan_omp_sections (stmt
, ctx
);
2817 case GIMPLE_OMP_SINGLE
:
2818 scan_omp_single (stmt
, ctx
);
2821 case GIMPLE_OMP_SECTION
:
2822 case GIMPLE_OMP_MASTER
:
2823 case GIMPLE_OMP_TASKGROUP
:
2824 case GIMPLE_OMP_ORDERED
:
2825 case GIMPLE_OMP_CRITICAL
:
2826 ctx
= new_omp_context (stmt
, ctx
);
2827 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2830 case GIMPLE_OMP_TARGET
:
2831 scan_omp_target (stmt
, ctx
);
2834 case GIMPLE_OMP_TEAMS
:
2835 scan_omp_teams (stmt
, ctx
);
2842 *handled_ops_p
= false;
2844 for (var
= gimple_bind_vars (as_a
<gimple_bind
> (stmt
));
2846 var
= DECL_CHAIN (var
))
2847 insert_decl_map (&ctx
->cb
, var
, var
);
2851 *handled_ops_p
= false;
2859 /* Scan all the statements starting at the current statement. CTX
2860 contains context information about the OpenMP directives and
2861 clauses found during the scan. */
2864 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2866 location_t saved_location
;
2867 struct walk_stmt_info wi
;
2869 memset (&wi
, 0, sizeof (wi
));
2871 wi
.want_locations
= true;
2873 saved_location
= input_location
;
2874 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2875 input_location
= saved_location
;
2878 /* Re-gimplification and code generation routines. */
2880 /* Build a call to GOMP_barrier. */
2883 build_omp_barrier (tree lhs
)
2885 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2886 : BUILT_IN_GOMP_BARRIER
);
2887 gimple_call g
= gimple_build_call (fndecl
, 0);
2889 gimple_call_set_lhs (g
, lhs
);
2893 /* If a context was created for STMT when it was scanned, return it. */
2895 static omp_context
*
2896 maybe_lookup_ctx (gimple stmt
)
2899 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2900 return n
? (omp_context
*) n
->value
: NULL
;
2904 /* Find the mapping for DECL in CTX or the immediately enclosing
2905 context that has a mapping for DECL.
2907 If CTX is a nested parallel directive, we may have to use the decl
2908 mappings created in CTX's parent context. Suppose that we have the
2909 following parallel nesting (variable UIDs showed for clarity):
2912 #omp parallel shared(iD.1562) -> outer parallel
2913 iD.1562 = iD.1562 + 1;
2915 #omp parallel shared (iD.1562) -> inner parallel
2916 iD.1562 = iD.1562 - 1;
2918 Each parallel structure will create a distinct .omp_data_s structure
2919 for copying iD.1562 in/out of the directive:
2921 outer parallel .omp_data_s.1.i -> iD.1562
2922 inner parallel .omp_data_s.2.i -> iD.1562
2924 A shared variable mapping will produce a copy-out operation before
2925 the parallel directive and a copy-in operation after it. So, in
2926 this case we would have:
2929 .omp_data_o.1.i = iD.1562;
2930 #omp parallel shared(iD.1562) -> outer parallel
2931 .omp_data_i.1 = &.omp_data_o.1
2932 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2934 .omp_data_o.2.i = iD.1562; -> **
2935 #omp parallel shared(iD.1562) -> inner parallel
2936 .omp_data_i.2 = &.omp_data_o.2
2937 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2940 ** This is a problem. The symbol iD.1562 cannot be referenced
2941 inside the body of the outer parallel region. But since we are
2942 emitting this copy operation while expanding the inner parallel
2943 directive, we need to access the CTX structure of the outer
2944 parallel directive to get the correct mapping:
2946 .omp_data_o.2.i = .omp_data_i.1->i
2948 Since there may be other workshare or parallel directives enclosing
2949 the parallel directive, it may be necessary to walk up the context
2950 parent chain. This is not a problem in general because nested
2951 parallelism happens only rarely. */
2954 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2959 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2960 t
= maybe_lookup_decl (decl
, up
);
2962 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2964 return t
? t
: decl
;
2968 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2969 in outer contexts. */
2972 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2977 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2978 t
= maybe_lookup_decl (decl
, up
);
2980 return t
? t
: decl
;
2984 /* Construct the initialization value for reduction CLAUSE. */
2987 omp_reduction_init (tree clause
, tree type
)
2989 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2990 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2997 case TRUTH_ORIF_EXPR
:
2998 case TRUTH_XOR_EXPR
:
3000 return build_zero_cst (type
);
3003 case TRUTH_AND_EXPR
:
3004 case TRUTH_ANDIF_EXPR
:
3006 return fold_convert_loc (loc
, type
, integer_one_node
);
3009 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3012 if (SCALAR_FLOAT_TYPE_P (type
))
3014 REAL_VALUE_TYPE max
, min
;
3015 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3018 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3021 real_maxval (&min
, 1, TYPE_MODE (type
));
3022 return build_real (type
, min
);
3026 gcc_assert (INTEGRAL_TYPE_P (type
));
3027 return TYPE_MIN_VALUE (type
);
3031 if (SCALAR_FLOAT_TYPE_P (type
))
3033 REAL_VALUE_TYPE max
;
3034 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3037 real_maxval (&max
, 0, TYPE_MODE (type
));
3038 return build_real (type
, max
);
3042 gcc_assert (INTEGRAL_TYPE_P (type
));
3043 return TYPE_MAX_VALUE (type
);
3051 /* Return alignment to be assumed for var in CLAUSE, which should be
3052 OMP_CLAUSE_ALIGNED. */
3055 omp_clause_aligned_alignment (tree clause
)
3057 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3058 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3060 /* Otherwise return implementation defined alignment. */
3061 unsigned int al
= 1;
3062 enum machine_mode mode
, vmode
;
3063 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3065 vs
= 1 << floor_log2 (vs
);
3066 static enum mode_class classes
[]
3067 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3068 for (int i
= 0; i
< 4; i
+= 2)
3069 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3071 mode
= GET_MODE_WIDER_MODE (mode
))
3073 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3074 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3077 && GET_MODE_SIZE (vmode
) < vs
3078 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3079 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3081 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3082 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3084 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3085 / GET_MODE_SIZE (mode
));
3086 if (TYPE_MODE (type
) != vmode
)
3088 if (TYPE_ALIGN_UNIT (type
) > al
)
3089 al
= TYPE_ALIGN_UNIT (type
);
3091 return build_int_cst (integer_type_node
, al
);
3094 /* Return maximum possible vectorization factor for the target. */
3101 || !flag_tree_loop_optimize
3102 || (!flag_tree_loop_vectorize
3103 && (global_options_set
.x_flag_tree_loop_vectorize
3104 || global_options_set
.x_flag_tree_vectorize
)))
3107 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3110 vs
= 1 << floor_log2 (vs
);
3113 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3114 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3115 return GET_MODE_NUNITS (vqimode
);
3119 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3123 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3124 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3128 max_vf
= omp_max_vf ();
3131 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3132 OMP_CLAUSE_SAFELEN
);
3133 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3135 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3137 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3141 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3142 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3148 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3149 tree avar
= create_tmp_var_raw (atype
, NULL
);
3150 if (TREE_ADDRESSABLE (new_var
))
3151 TREE_ADDRESSABLE (avar
) = 1;
3152 DECL_ATTRIBUTES (avar
)
3153 = tree_cons (get_identifier ("omp simd array"), NULL
,
3154 DECL_ATTRIBUTES (avar
));
3155 gimple_add_tmp_var (avar
);
3156 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3157 NULL_TREE
, NULL_TREE
);
3158 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3159 NULL_TREE
, NULL_TREE
);
3160 if (DECL_P (new_var
))
3162 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3163 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3168 /* Helper function of lower_rec_input_clauses. For a reference
3169 in simd reduction, add an underlying variable it will reference. */
3172 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3174 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3175 if (TREE_CONSTANT (z
))
3177 const char *name
= NULL
;
3178 if (DECL_NAME (new_vard
))
3179 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3181 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3182 gimple_add_tmp_var (z
);
3183 TREE_ADDRESSABLE (z
) = 1;
3184 z
= build_fold_addr_expr_loc (loc
, z
);
3185 gimplify_assign (new_vard
, z
, ilist
);
3189 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3190 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3191 private variables. Initialization statements go in ILIST, while calls
3192 to destructors go in DLIST. */
3195 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3196 omp_context
*ctx
, struct omp_for_data
*fd
)
3198 tree c
, dtor
, copyin_seq
, x
, ptr
;
3199 bool copyin_by_ref
= false;
3200 bool lastprivate_firstprivate
= false;
3201 bool reduction_omp_orig_ref
= false;
3203 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3204 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3206 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3207 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3208 gimple_seq llist
[2] = { NULL
, NULL
};
3212 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3213 with data sharing clauses referencing variable sized vars. That
3214 is unnecessarily hard to support and very unlikely to result in
3215 vectorized code anyway. */
3217 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3218 switch (OMP_CLAUSE_CODE (c
))
3220 case OMP_CLAUSE_LINEAR
:
3221 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3224 case OMP_CLAUSE_REDUCTION
:
3225 case OMP_CLAUSE_PRIVATE
:
3226 case OMP_CLAUSE_FIRSTPRIVATE
:
3227 case OMP_CLAUSE_LASTPRIVATE
:
3228 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3235 /* Do all the fixed sized types in the first pass, and the variable sized
3236 types in the second pass. This makes sure that the scalar arguments to
3237 the variable sized types are processed before we use them in the
3238 variable sized operations. */
3239 for (pass
= 0; pass
< 2; ++pass
)
3241 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3243 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3246 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3250 case OMP_CLAUSE_PRIVATE
:
3251 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3254 case OMP_CLAUSE_SHARED
:
3255 /* Ignore shared directives in teams construct. */
3256 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3258 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3260 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3263 case OMP_CLAUSE_FIRSTPRIVATE
:
3264 case OMP_CLAUSE_COPYIN
:
3265 case OMP_CLAUSE_LINEAR
:
3267 case OMP_CLAUSE_REDUCTION
:
3268 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3269 reduction_omp_orig_ref
= true;
3271 case OMP_CLAUSE__LOOPTEMP_
:
3272 /* Handle _looptemp_ clauses only on parallel. */
3276 case OMP_CLAUSE_LASTPRIVATE
:
3277 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3279 lastprivate_firstprivate
= true;
3283 /* Even without corresponding firstprivate, if
3284 decl is Fortran allocatable, it needs outer var
3287 && lang_hooks
.decls
.omp_private_outer_ref
3288 (OMP_CLAUSE_DECL (c
)))
3289 lastprivate_firstprivate
= true;
3291 case OMP_CLAUSE_ALIGNED
:
3294 var
= OMP_CLAUSE_DECL (c
);
3295 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3296 && !is_global_var (var
))
3298 new_var
= maybe_lookup_decl (var
, ctx
);
3299 if (new_var
== NULL_TREE
)
3300 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3301 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3302 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3303 omp_clause_aligned_alignment (c
));
3304 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3305 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3306 gimplify_and_add (x
, ilist
);
3308 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3309 && is_global_var (var
))
3311 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3312 new_var
= lookup_decl (var
, ctx
);
3313 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3314 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3315 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3316 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3317 omp_clause_aligned_alignment (c
));
3318 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3319 x
= create_tmp_var (ptype
, NULL
);
3320 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3321 gimplify_and_add (t
, ilist
);
3322 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3323 SET_DECL_VALUE_EXPR (new_var
, t
);
3324 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3331 new_var
= var
= OMP_CLAUSE_DECL (c
);
3332 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3333 new_var
= lookup_decl (var
, ctx
);
3335 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3340 else if (is_variable_sized (var
))
3342 /* For variable sized types, we need to allocate the
3343 actual storage here. Call alloca and store the
3344 result in the pointer decl that we created elsewhere. */
3348 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3353 ptr
= DECL_VALUE_EXPR (new_var
);
3354 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3355 ptr
= TREE_OPERAND (ptr
, 0);
3356 gcc_assert (DECL_P (ptr
));
3357 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3359 /* void *tmp = __builtin_alloca */
3360 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3361 stmt
= gimple_build_call (atmp
, 1, x
);
3362 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3363 gimple_add_tmp_var (tmp
);
3364 gimple_call_set_lhs (stmt
, tmp
);
3366 gimple_seq_add_stmt (ilist
, stmt
);
3368 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3369 gimplify_assign (ptr
, x
, ilist
);
3372 else if (is_reference (var
))
3374 /* For references that are being privatized for Fortran,
3375 allocate new backing storage for the new pointer
3376 variable. This allows us to avoid changing all the
3377 code that expects a pointer to something that expects
3378 a direct variable. */
3382 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3383 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3385 x
= build_receiver_ref (var
, false, ctx
);
3386 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3388 else if (TREE_CONSTANT (x
))
3390 /* For reduction in SIMD loop, defer adding the
3391 initialization of the reference, because if we decide
3392 to use SIMD array for it, the initilization could cause
3394 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3398 const char *name
= NULL
;
3399 if (DECL_NAME (var
))
3400 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3402 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3404 gimple_add_tmp_var (x
);
3405 TREE_ADDRESSABLE (x
) = 1;
3406 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3411 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3412 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3417 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3418 gimplify_assign (new_var
, x
, ilist
);
3421 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3423 else if (c_kind
== OMP_CLAUSE_REDUCTION
3424 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3432 switch (OMP_CLAUSE_CODE (c
))
3434 case OMP_CLAUSE_SHARED
:
3435 /* Ignore shared directives in teams construct. */
3436 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3438 /* Shared global vars are just accessed directly. */
3439 if (is_global_var (new_var
))
3441 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3442 needs to be delayed until after fixup_child_record_type so
3443 that we get the correct type during the dereference. */
3444 by_ref
= use_pointer_for_field (var
, ctx
);
3445 x
= build_receiver_ref (var
, by_ref
, ctx
);
3446 SET_DECL_VALUE_EXPR (new_var
, x
);
3447 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3449 /* ??? If VAR is not passed by reference, and the variable
3450 hasn't been initialized yet, then we'll get a warning for
3451 the store into the omp_data_s structure. Ideally, we'd be
3452 able to notice this and not store anything at all, but
3453 we're generating code too early. Suppress the warning. */
3455 TREE_NO_WARNING (var
) = 1;
3458 case OMP_CLAUSE_LASTPRIVATE
:
3459 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3463 case OMP_CLAUSE_PRIVATE
:
3464 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3465 x
= build_outer_var_ref (var
, ctx
);
3466 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3468 if (is_task_ctx (ctx
))
3469 x
= build_receiver_ref (var
, false, ctx
);
3471 x
= build_outer_var_ref (var
, ctx
);
3477 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3480 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3481 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3482 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3483 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3484 idx
, lane
, ivar
, lvar
))
3487 x
= lang_hooks
.decls
.omp_clause_default_ctor
3488 (c
, unshare_expr (ivar
), x
);
3490 gimplify_and_add (x
, &llist
[0]);
3493 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3496 gimple_seq tseq
= NULL
;
3499 gimplify_stmt (&dtor
, &tseq
);
3500 gimple_seq_add_seq (&llist
[1], tseq
);
3507 gimplify_and_add (nx
, ilist
);
3511 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3514 gimple_seq tseq
= NULL
;
3517 gimplify_stmt (&dtor
, &tseq
);
3518 gimple_seq_add_seq (dlist
, tseq
);
3522 case OMP_CLAUSE_LINEAR
:
3523 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3524 goto do_firstprivate
;
3525 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3528 x
= build_outer_var_ref (var
, ctx
);
3531 case OMP_CLAUSE_FIRSTPRIVATE
:
3532 if (is_task_ctx (ctx
))
3534 if (is_reference (var
) || is_variable_sized (var
))
3536 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3538 || use_pointer_for_field (var
, NULL
))
3540 x
= build_receiver_ref (var
, false, ctx
);
3541 SET_DECL_VALUE_EXPR (new_var
, x
);
3542 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3547 x
= build_outer_var_ref (var
, ctx
);
3550 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3551 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3553 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3554 tree stept
= TREE_TYPE (t
);
3555 tree ct
= find_omp_clause (clauses
,
3556 OMP_CLAUSE__LOOPTEMP_
);
3558 tree l
= OMP_CLAUSE_DECL (ct
);
3559 tree n1
= fd
->loop
.n1
;
3560 tree step
= fd
->loop
.step
;
3561 tree itype
= TREE_TYPE (l
);
3562 if (POINTER_TYPE_P (itype
))
3563 itype
= signed_type_for (itype
);
3564 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3565 if (TYPE_UNSIGNED (itype
)
3566 && fd
->loop
.cond_code
== GT_EXPR
)
3567 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3568 fold_build1 (NEGATE_EXPR
, itype
, l
),
3569 fold_build1 (NEGATE_EXPR
,
3572 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3573 t
= fold_build2 (MULT_EXPR
, stept
,
3574 fold_convert (stept
, l
), t
);
3576 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3578 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3580 gimplify_and_add (x
, ilist
);
3584 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3585 x
= fold_build2 (POINTER_PLUS_EXPR
,
3586 TREE_TYPE (x
), x
, t
);
3588 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3591 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3592 || TREE_ADDRESSABLE (new_var
))
3593 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3594 idx
, lane
, ivar
, lvar
))
3596 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3598 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3599 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3600 gimplify_and_add (x
, ilist
);
3601 gimple_stmt_iterator gsi
3602 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3604 = gimple_build_assign (unshare_expr (lvar
), iv
);
3605 gsi_insert_before_without_update (&gsi
, g
,
3607 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3608 enum tree_code code
= PLUS_EXPR
;
3609 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3610 code
= POINTER_PLUS_EXPR
;
3611 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3612 gsi_insert_before_without_update (&gsi
, g
,
3616 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3617 (c
, unshare_expr (ivar
), x
);
3618 gimplify_and_add (x
, &llist
[0]);
3619 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3622 gimple_seq tseq
= NULL
;
3625 gimplify_stmt (&dtor
, &tseq
);
3626 gimple_seq_add_seq (&llist
[1], tseq
);
3631 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3632 gimplify_and_add (x
, ilist
);
3635 case OMP_CLAUSE__LOOPTEMP_
:
3636 gcc_assert (is_parallel_ctx (ctx
));
3637 x
= build_outer_var_ref (var
, ctx
);
3638 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3639 gimplify_and_add (x
, ilist
);
3642 case OMP_CLAUSE_COPYIN
:
3643 by_ref
= use_pointer_for_field (var
, NULL
);
3644 x
= build_receiver_ref (var
, by_ref
, ctx
);
3645 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3646 append_to_statement_list (x
, ©in_seq
);
3647 copyin_by_ref
|= by_ref
;
3650 case OMP_CLAUSE_REDUCTION
:
3651 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3653 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3655 x
= build_outer_var_ref (var
, ctx
);
3657 if (is_reference (var
)
3658 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3660 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3661 SET_DECL_VALUE_EXPR (placeholder
, x
);
3662 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3663 tree new_vard
= new_var
;
3664 if (is_reference (var
))
3666 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3667 new_vard
= TREE_OPERAND (new_var
, 0);
3668 gcc_assert (DECL_P (new_vard
));
3671 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3672 idx
, lane
, ivar
, lvar
))
3674 if (new_vard
== new_var
)
3676 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3677 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3681 SET_DECL_VALUE_EXPR (new_vard
,
3682 build_fold_addr_expr (ivar
));
3683 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3685 x
= lang_hooks
.decls
.omp_clause_default_ctor
3686 (c
, unshare_expr (ivar
),
3687 build_outer_var_ref (var
, ctx
));
3689 gimplify_and_add (x
, &llist
[0]);
3690 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3692 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3693 lower_omp (&tseq
, ctx
);
3694 gimple_seq_add_seq (&llist
[0], tseq
);
3696 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3697 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3698 lower_omp (&tseq
, ctx
);
3699 gimple_seq_add_seq (&llist
[1], tseq
);
3700 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3701 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3702 if (new_vard
== new_var
)
3703 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3705 SET_DECL_VALUE_EXPR (new_vard
,
3706 build_fold_addr_expr (lvar
));
3707 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3712 gimplify_stmt (&dtor
, &tseq
);
3713 gimple_seq_add_seq (&llist
[1], tseq
);
3717 /* If this is a reference to constant size reduction var
3718 with placeholder, we haven't emitted the initializer
3719 for it because it is undesirable if SIMD arrays are used.
3720 But if they aren't used, we need to emit the deferred
3721 initialization now. */
3722 else if (is_reference (var
) && is_simd
)
3723 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3724 x
= lang_hooks
.decls
.omp_clause_default_ctor
3725 (c
, unshare_expr (new_var
),
3726 build_outer_var_ref (var
, ctx
));
3728 gimplify_and_add (x
, ilist
);
3729 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3731 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3732 lower_omp (&tseq
, ctx
);
3733 gimple_seq_add_seq (ilist
, tseq
);
3735 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3738 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3739 lower_omp (&tseq
, ctx
);
3740 gimple_seq_add_seq (dlist
, tseq
);
3741 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3743 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3748 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3749 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3750 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3752 /* reduction(-:var) sums up the partial results, so it
3753 acts identically to reduction(+:var). */
3754 if (code
== MINUS_EXPR
)
3757 tree new_vard
= new_var
;
3758 if (is_simd
&& is_reference (var
))
3760 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3761 new_vard
= TREE_OPERAND (new_var
, 0);
3762 gcc_assert (DECL_P (new_vard
));
3765 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3766 idx
, lane
, ivar
, lvar
))
3768 tree ref
= build_outer_var_ref (var
, ctx
);
3770 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3772 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3773 ref
= build_outer_var_ref (var
, ctx
);
3774 gimplify_assign (ref
, x
, &llist
[1]);
3776 if (new_vard
!= new_var
)
3778 SET_DECL_VALUE_EXPR (new_vard
,
3779 build_fold_addr_expr (lvar
));
3780 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3785 if (is_reference (var
) && is_simd
)
3786 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3787 gimplify_assign (new_var
, x
, ilist
);
3790 tree ref
= build_outer_var_ref (var
, ctx
);
3792 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3793 ref
= build_outer_var_ref (var
, ctx
);
3794 gimplify_assign (ref
, x
, dlist
);
3808 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3809 /* Don't want uninit warnings on simduid, it is always uninitialized,
3810 but we use it not for the value, but for the DECL_UID only. */
3811 TREE_NO_WARNING (uid
) = 1;
3813 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3814 gimple_call_set_lhs (g
, lane
);
3815 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3816 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3817 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3818 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3819 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3820 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3821 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3822 build_int_cst (unsigned_type_node
, 0),
3824 gimple_seq_add_stmt (ilist
, g
);
3825 for (int i
= 0; i
< 2; i
++)
3828 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3829 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3830 gimple_call_set_lhs (g
, vf
);
3831 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3832 gimple_seq_add_stmt (seq
, g
);
3833 tree t
= build_int_cst (unsigned_type_node
, 0);
3834 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3835 gimple_seq_add_stmt (seq
, g
);
3836 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3837 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3838 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3839 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3840 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3841 gimple_seq_add_seq (seq
, llist
[i
]);
3842 t
= build_int_cst (unsigned_type_node
, 1);
3843 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3844 gimple_seq_add_stmt (seq
, g
);
3845 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3846 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3847 gimple_seq_add_stmt (seq
, g
);
3848 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3852 /* The copyin sequence is not to be executed by the main thread, since
3853 that would result in self-copies. Perhaps not visible to scalars,
3854 but it certainly is to C++ operator=. */
3857 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3859 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3860 build_int_cst (TREE_TYPE (x
), 0));
3861 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3862 gimplify_and_add (x
, ilist
);
3865 /* If any copyin variable is passed by reference, we must ensure the
3866 master thread doesn't modify it before it is copied over in all
3867 threads. Similarly for variables in both firstprivate and
3868 lastprivate clauses we need to ensure the lastprivate copying
3869 happens after firstprivate copying in all threads. And similarly
3870 for UDRs if initializer expression refers to omp_orig. */
3871 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3873 /* Don't add any barrier for #pragma omp simd or
3874 #pragma omp distribute. */
3875 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3876 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3877 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3880 /* If max_vf is non-zero, then we can use only a vectorization factor
3881 up to the max_vf we chose. So stick it into the safelen clause. */
3884 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3885 OMP_CLAUSE_SAFELEN
);
3887 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3888 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3891 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3892 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3894 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3895 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3901 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3902 both parallel and workshare constructs. PREDICATE may be NULL if it's
3906 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3909 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3910 bool par_clauses
= false;
3911 tree simduid
= NULL
, lastlane
= NULL
;
3913 /* Early exit if there are no lastprivate or linear clauses. */
3914 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3915 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3916 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3917 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3919 if (clauses
== NULL
)
3921 /* If this was a workshare clause, see if it had been combined
3922 with its parallel. In that case, look for the clauses on the
3923 parallel statement itself. */
3924 if (is_parallel_ctx (ctx
))
3928 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3931 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3932 OMP_CLAUSE_LASTPRIVATE
);
3933 if (clauses
== NULL
)
3941 tree label_true
, arm1
, arm2
;
3943 label
= create_artificial_label (UNKNOWN_LOCATION
);
3944 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3945 arm1
= TREE_OPERAND (predicate
, 0);
3946 arm2
= TREE_OPERAND (predicate
, 1);
3947 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3948 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3949 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3951 gimple_seq_add_stmt (stmt_list
, stmt
);
3952 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3955 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3956 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3958 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3960 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3963 for (c
= clauses
; c
;)
3966 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3968 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3969 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3970 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3972 var
= OMP_CLAUSE_DECL (c
);
3973 new_var
= lookup_decl (var
, ctx
);
3975 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3977 tree val
= DECL_VALUE_EXPR (new_var
);
3978 if (TREE_CODE (val
) == ARRAY_REF
3979 && VAR_P (TREE_OPERAND (val
, 0))
3980 && lookup_attribute ("omp simd array",
3981 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3984 if (lastlane
== NULL
)
3986 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3988 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3990 TREE_OPERAND (val
, 1));
3991 gimple_call_set_lhs (g
, lastlane
);
3992 gimple_seq_add_stmt (stmt_list
, g
);
3994 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3995 TREE_OPERAND (val
, 0), lastlane
,
3996 NULL_TREE
, NULL_TREE
);
4000 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4001 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4003 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4004 gimple_seq_add_seq (stmt_list
,
4005 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4006 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4008 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4009 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4011 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4012 gimple_seq_add_seq (stmt_list
,
4013 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4014 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4017 x
= build_outer_var_ref (var
, ctx
);
4018 if (is_reference (var
))
4019 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4020 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4021 gimplify_and_add (x
, stmt_list
);
4023 c
= OMP_CLAUSE_CHAIN (c
);
4024 if (c
== NULL
&& !par_clauses
)
4026 /* If this was a workshare clause, see if it had been combined
4027 with its parallel. In that case, continue looking for the
4028 clauses also on the parallel statement itself. */
4029 if (is_parallel_ctx (ctx
))
4033 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4036 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4037 OMP_CLAUSE_LASTPRIVATE
);
4043 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4047 /* Generate code to implement the REDUCTION clauses. */
4050 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4052 gimple_seq sub_seq
= NULL
;
4057 /* SIMD reductions are handled in lower_rec_input_clauses. */
4058 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4059 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4062 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4063 update in that case, otherwise use a lock. */
4064 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4065 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4067 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4069 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4079 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4081 tree var
, ref
, new_var
;
4082 enum tree_code code
;
4083 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4085 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4088 var
= OMP_CLAUSE_DECL (c
);
4089 new_var
= lookup_decl (var
, ctx
);
4090 if (is_reference (var
))
4091 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4092 ref
= build_outer_var_ref (var
, ctx
);
4093 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4095 /* reduction(-:var) sums up the partial results, so it acts
4096 identically to reduction(+:var). */
4097 if (code
== MINUS_EXPR
)
4102 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4104 addr
= save_expr (addr
);
4105 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4106 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4107 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4108 gimplify_and_add (x
, stmt_seqp
);
4112 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4114 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4116 if (is_reference (var
)
4117 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4119 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4120 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4121 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4122 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4123 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4124 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4125 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4129 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4130 ref
= build_outer_var_ref (var
, ctx
);
4131 gimplify_assign (ref
, x
, &sub_seq
);
4135 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4137 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4139 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4141 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4143 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4147 /* Generate code to implement the COPYPRIVATE clauses. */
4150 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4155 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4157 tree var
, new_var
, ref
, x
;
4159 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4161 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4164 var
= OMP_CLAUSE_DECL (c
);
4165 by_ref
= use_pointer_for_field (var
, NULL
);
4167 ref
= build_sender_ref (var
, ctx
);
4168 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4171 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4172 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4174 gimplify_assign (ref
, x
, slist
);
4176 ref
= build_receiver_ref (var
, false, ctx
);
4179 ref
= fold_convert_loc (clause_loc
,
4180 build_pointer_type (TREE_TYPE (new_var
)),
4182 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4184 if (is_reference (var
))
4186 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4187 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4188 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4190 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4191 gimplify_and_add (x
, rlist
);
4196 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4197 and REDUCTION from the sender (aka parent) side. */
4200 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4205 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4207 tree val
, ref
, x
, var
;
4208 bool by_ref
, do_in
= false, do_out
= false;
4209 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4211 switch (OMP_CLAUSE_CODE (c
))
4213 case OMP_CLAUSE_PRIVATE
:
4214 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4217 case OMP_CLAUSE_FIRSTPRIVATE
:
4218 case OMP_CLAUSE_COPYIN
:
4219 case OMP_CLAUSE_LASTPRIVATE
:
4220 case OMP_CLAUSE_REDUCTION
:
4221 case OMP_CLAUSE__LOOPTEMP_
:
4227 val
= OMP_CLAUSE_DECL (c
);
4228 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4230 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4231 && is_global_var (var
))
4233 if (is_variable_sized (val
))
4235 by_ref
= use_pointer_for_field (val
, NULL
);
4237 switch (OMP_CLAUSE_CODE (c
))
4239 case OMP_CLAUSE_PRIVATE
:
4240 case OMP_CLAUSE_FIRSTPRIVATE
:
4241 case OMP_CLAUSE_COPYIN
:
4242 case OMP_CLAUSE__LOOPTEMP_
:
4246 case OMP_CLAUSE_LASTPRIVATE
:
4247 if (by_ref
|| is_reference (val
))
4249 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4256 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4261 case OMP_CLAUSE_REDUCTION
:
4263 do_out
= !(by_ref
|| is_reference (val
));
4272 ref
= build_sender_ref (val
, ctx
);
4273 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4274 gimplify_assign (ref
, x
, ilist
);
4275 if (is_task_ctx (ctx
))
4276 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4281 ref
= build_sender_ref (val
, ctx
);
4282 gimplify_assign (var
, ref
, olist
);
4287 /* Generate code to implement SHARED from the sender (aka parent)
4288 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4289 list things that got automatically shared. */
4292 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4294 tree var
, ovar
, nvar
, f
, x
, record_type
;
4296 if (ctx
->record_type
== NULL
)
4299 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4300 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4302 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4303 nvar
= maybe_lookup_decl (ovar
, ctx
);
4304 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4307 /* If CTX is a nested parallel directive. Find the immediately
4308 enclosing parallel or workshare construct that contains a
4309 mapping for OVAR. */
4310 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4312 if (use_pointer_for_field (ovar
, ctx
))
4314 x
= build_sender_ref (ovar
, ctx
);
4315 var
= build_fold_addr_expr (var
);
4316 gimplify_assign (x
, var
, ilist
);
4320 x
= build_sender_ref (ovar
, ctx
);
4321 gimplify_assign (x
, var
, ilist
);
4323 if (!TREE_READONLY (var
)
4324 /* We don't need to receive a new reference to a result
4325 or parm decl. In fact we may not store to it as we will
4326 invalidate any pending RSO and generate wrong gimple
4328 && !((TREE_CODE (var
) == RESULT_DECL
4329 || TREE_CODE (var
) == PARM_DECL
)
4330 && DECL_BY_REFERENCE (var
)))
4332 x
= build_sender_ref (ovar
, ctx
);
4333 gimplify_assign (var
, x
, olist
);
4340 /* A convenience function to build an empty GIMPLE_COND with just the
4344 gimple_build_cond_empty (tree cond
)
4346 enum tree_code pred_code
;
4349 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4350 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4354 /* Build the function calls to GOMP_parallel_start etc to actually
4355 generate the parallel operation. REGION is the parallel region
4356 being expanded. BB is the block where to insert the code. WS_ARGS
4357 will be set if this is a call to a combined parallel+workshare
4358 construct, it contains the list of additional arguments needed by
4359 the workshare construct. */
4362 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4363 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4365 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4366 gimple_stmt_iterator gsi
;
4368 enum built_in_function start_ix
;
4370 location_t clause_loc
;
4371 vec
<tree
, va_gc
> *args
;
4373 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4375 /* Determine what flavor of GOMP_parallel we will be
4377 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4378 if (is_combined_parallel (region
))
4380 switch (region
->inner
->type
)
4382 case GIMPLE_OMP_FOR
:
4383 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4384 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4385 + (region
->inner
->sched_kind
4386 == OMP_CLAUSE_SCHEDULE_RUNTIME
4387 ? 3 : region
->inner
->sched_kind
));
4388 start_ix
= (enum built_in_function
)start_ix2
;
4390 case GIMPLE_OMP_SECTIONS
:
4391 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4398 /* By default, the value of NUM_THREADS is zero (selected at run time)
4399 and there is no conditional. */
4401 val
= build_int_cst (unsigned_type_node
, 0);
4402 flags
= build_int_cst (unsigned_type_node
, 0);
4404 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4406 cond
= OMP_CLAUSE_IF_EXPR (c
);
4408 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4411 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4412 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4415 clause_loc
= gimple_location (entry_stmt
);
4417 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4419 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4421 /* Ensure 'val' is of the correct type. */
4422 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4424 /* If we found the clause 'if (cond)', build either
4425 (cond != 0) or (cond ? val : 1u). */
4428 cond
= gimple_boolify (cond
);
4430 if (integer_zerop (val
))
4431 val
= fold_build2_loc (clause_loc
,
4432 EQ_EXPR
, unsigned_type_node
, cond
,
4433 build_int_cst (TREE_TYPE (cond
), 0));
4436 basic_block cond_bb
, then_bb
, else_bb
;
4437 edge e
, e_then
, e_else
;
4438 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4440 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4441 if (gimple_in_ssa_p (cfun
))
4443 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4444 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4445 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4454 e
= split_block (bb
, NULL
);
4459 then_bb
= create_empty_bb (cond_bb
);
4460 else_bb
= create_empty_bb (then_bb
);
4461 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4462 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4464 stmt
= gimple_build_cond_empty (cond
);
4465 gsi
= gsi_start_bb (cond_bb
);
4466 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4468 gsi
= gsi_start_bb (then_bb
);
4469 stmt
= gimple_build_assign (tmp_then
, val
);
4470 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4472 gsi
= gsi_start_bb (else_bb
);
4473 stmt
= gimple_build_assign
4474 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4475 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4477 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4478 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4479 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4480 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4481 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4482 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4484 if (gimple_in_ssa_p (cfun
))
4486 gimple_phi phi
= create_phi_node (tmp_join
, bb
);
4487 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4488 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4494 gsi
= gsi_start_bb (bb
);
4495 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4496 false, GSI_CONTINUE_LINKING
);
4499 gsi
= gsi_last_bb (bb
);
4500 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4502 t1
= null_pointer_node
;
4504 t1
= build_fold_addr_expr (t
);
4505 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4507 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4508 args
->quick_push (t2
);
4509 args
->quick_push (t1
);
4510 args
->quick_push (val
);
4512 args
->splice (*ws_args
);
4513 args
->quick_push (flags
);
4515 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4516 builtin_decl_explicit (start_ix
), args
);
4518 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4519 false, GSI_CONTINUE_LINKING
);
4522 /* Insert a function call whose name is FUNC_NAME with the information from
4523 ENTRY_STMT into the basic_block BB. */
4526 expand_cilk_for_call (basic_block bb
, gimple entry_stmt
,
4527 vec
<tree
, va_gc
> *ws_args
)
4530 gimple_stmt_iterator gsi
;
4531 vec
<tree
, va_gc
> *args
;
4533 gcc_assert (vec_safe_length (ws_args
) == 2);
4534 tree func_name
= (*ws_args
)[0];
4535 tree grain
= (*ws_args
)[1];
4537 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4538 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
4539 gcc_assert (count
!= NULL_TREE
);
4540 count
= OMP_CLAUSE_OPERAND (count
, 0);
4542 gsi
= gsi_last_bb (bb
);
4543 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4545 t1
= null_pointer_node
;
4547 t1
= build_fold_addr_expr (t
);
4548 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4550 vec_alloc (args
, 4);
4551 args
->quick_push (t2
);
4552 args
->quick_push (t1
);
4553 args
->quick_push (count
);
4554 args
->quick_push (grain
);
4555 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
4557 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
4558 GSI_CONTINUE_LINKING
);
4561 /* Build the function call to GOMP_task to actually
4562 generate the task operation. BB is the block where to insert the code. */
4565 expand_task_call (basic_block bb
, gimple entry_stmt
)
4567 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4568 gimple_stmt_iterator gsi
;
4569 location_t loc
= gimple_location (entry_stmt
);
4571 clauses
= gimple_omp_task_clauses (entry_stmt
);
4573 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4575 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4577 cond
= boolean_true_node
;
4579 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4580 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4581 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4582 flags
= build_int_cst (unsigned_type_node
,
4583 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4585 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4588 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4589 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4590 build_int_cst (unsigned_type_node
, 2),
4591 build_int_cst (unsigned_type_node
, 0));
4592 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4595 depend
= OMP_CLAUSE_DECL (depend
);
4597 depend
= build_int_cst (ptr_type_node
, 0);
4599 gsi
= gsi_last_bb (bb
);
4600 t
= gimple_omp_task_data_arg (entry_stmt
);
4602 t2
= null_pointer_node
;
4604 t2
= build_fold_addr_expr_loc (loc
, t
);
4605 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4606 t
= gimple_omp_task_copy_fn (entry_stmt
);
4608 t3
= null_pointer_node
;
4610 t3
= build_fold_addr_expr_loc (loc
, t
);
4612 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4614 gimple_omp_task_arg_size (entry_stmt
),
4615 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4618 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4619 false, GSI_CONTINUE_LINKING
);
4623 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4624 catch handler and return it. This prevents programs from violating the
4625 structured block semantics with throws. */
4628 maybe_catch_exception (gimple_seq body
)
4633 if (!flag_exceptions
)
4636 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4637 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4639 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4641 g
= gimple_build_eh_must_not_throw (decl
);
4642 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4645 return gimple_seq_alloc_with_stmt (g
);
4648 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4651 vec2chain (vec
<tree
, va_gc
> *v
)
4653 tree chain
= NULL_TREE
, t
;
4656 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4658 DECL_CHAIN (t
) = chain
;
4666 /* Remove barriers in REGION->EXIT's block. Note that this is only
4667 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4668 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4669 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4673 remove_exit_barrier (struct omp_region
*region
)
4675 gimple_stmt_iterator gsi
;
4676 basic_block exit_bb
;
4680 int any_addressable_vars
= -1;
4682 exit_bb
= region
->exit
;
4684 /* If the parallel region doesn't return, we don't have REGION->EXIT
4689 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4690 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4691 statements that can appear in between are extremely limited -- no
4692 memory operations at all. Here, we allow nothing at all, so the
4693 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4694 gsi
= gsi_last_bb (exit_bb
);
4695 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4697 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4700 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4702 gsi
= gsi_last_bb (e
->src
);
4703 if (gsi_end_p (gsi
))
4705 stmt
= gsi_stmt (gsi
);
4706 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4707 && !gimple_omp_return_nowait_p (stmt
))
4709 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4710 in many cases. If there could be tasks queued, the barrier
4711 might be needed to let the tasks run before some local
4712 variable of the parallel that the task uses as shared
4713 runs out of scope. The task can be spawned either
4714 from within current function (this would be easy to check)
4715 or from some function it calls and gets passed an address
4716 of such a variable. */
4717 if (any_addressable_vars
< 0)
4719 gimple parallel_stmt
= last_stmt (region
->entry
);
4720 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4721 tree local_decls
, block
, decl
;
4724 any_addressable_vars
= 0;
4725 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4726 if (TREE_ADDRESSABLE (decl
))
4728 any_addressable_vars
= 1;
4731 for (block
= gimple_block (stmt
);
4732 !any_addressable_vars
4734 && TREE_CODE (block
) == BLOCK
;
4735 block
= BLOCK_SUPERCONTEXT (block
))
4737 for (local_decls
= BLOCK_VARS (block
);
4739 local_decls
= DECL_CHAIN (local_decls
))
4740 if (TREE_ADDRESSABLE (local_decls
))
4742 any_addressable_vars
= 1;
4745 if (block
== gimple_block (parallel_stmt
))
4749 if (!any_addressable_vars
)
4750 gimple_omp_return_set_nowait (stmt
);
4756 remove_exit_barriers (struct omp_region
*region
)
4758 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4759 remove_exit_barrier (region
);
4763 region
= region
->inner
;
4764 remove_exit_barriers (region
);
4765 while (region
->next
)
4767 region
= region
->next
;
4768 remove_exit_barriers (region
);
4773 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4774 calls. These can't be declared as const functions, but
4775 within one parallel body they are constant, so they can be
4776 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4777 which are declared const. Similarly for task body, except
4778 that in untied task omp_get_thread_num () can change at any task
4779 scheduling point. */
4782 optimize_omp_library_calls (gimple entry_stmt
)
4785 gimple_stmt_iterator gsi
;
4786 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4787 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4788 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4789 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4790 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4791 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4792 OMP_CLAUSE_UNTIED
) != NULL
);
4794 FOR_EACH_BB_FN (bb
, cfun
)
4795 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4797 gimple call
= gsi_stmt (gsi
);
4800 if (is_gimple_call (call
)
4801 && (decl
= gimple_call_fndecl (call
))
4802 && DECL_EXTERNAL (decl
)
4803 && TREE_PUBLIC (decl
)
4804 && DECL_INITIAL (decl
) == NULL
)
4808 if (DECL_NAME (decl
) == thr_num_id
)
4810 /* In #pragma omp task untied omp_get_thread_num () can change
4811 during the execution of the task region. */
4814 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4816 else if (DECL_NAME (decl
) == num_thr_id
)
4817 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4821 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4822 || gimple_call_num_args (call
) != 0)
4825 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4828 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4829 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4830 TREE_TYPE (TREE_TYPE (built_in
))))
4833 gimple_call_set_fndecl (call
, built_in
);
4838 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4842 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4846 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4847 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4850 if (TREE_CODE (t
) == ADDR_EXPR
)
4851 recompute_tree_invariant_for_addr_expr (t
);
4853 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4857 /* Prepend TO = FROM assignment before *GSI_P. */
4860 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4862 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4863 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4864 true, GSI_SAME_STMT
);
4865 gimple stmt
= gimple_build_assign (to
, from
);
4866 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4867 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4868 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4870 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4871 gimple_regimplify_operands (stmt
, &gsi
);
4875 /* Expand the OpenMP parallel or task directive starting at REGION. */
4878 expand_omp_taskreg (struct omp_region
*region
)
4880 basic_block entry_bb
, exit_bb
, new_bb
;
4881 struct function
*child_cfun
;
4882 tree child_fn
, block
, t
;
4883 gimple_stmt_iterator gsi
;
4884 gimple entry_stmt
, stmt
;
4886 vec
<tree
, va_gc
> *ws_args
;
4888 entry_stmt
= last_stmt (region
->entry
);
4889 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4890 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4892 entry_bb
= region
->entry
;
4893 exit_bb
= region
->exit
;
4897 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
4898 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
4899 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
4902 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
4903 and the inner statement contains the name of the built-in function
4905 ws_args
= region
->inner
->ws_args
;
4906 else if (is_combined_parallel (region
))
4907 ws_args
= region
->ws_args
;
4911 if (child_cfun
->cfg
)
4913 /* Due to inlining, it may happen that we have already outlined
4914 the region, in which case all we need to do is make the
4915 sub-graph unreachable and emit the parallel call. */
4916 edge entry_succ_e
, exit_succ_e
;
4918 entry_succ_e
= single_succ_edge (entry_bb
);
4920 gsi
= gsi_last_bb (entry_bb
);
4921 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4922 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4923 gsi_remove (&gsi
, true);
4928 exit_succ_e
= single_succ_edge (exit_bb
);
4929 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4931 remove_edge_and_dominated_blocks (entry_succ_e
);
4935 unsigned srcidx
, dstidx
, num
;
4937 /* If the parallel region needs data sent from the parent
4938 function, then the very first statement (except possible
4939 tree profile counter updates) of the parallel body
4940 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4941 &.OMP_DATA_O is passed as an argument to the child function,
4942 we need to replace it with the argument as seen by the child
4945 In most cases, this will end up being the identity assignment
4946 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4947 a function call that has been inlined, the original PARM_DECL
4948 .OMP_DATA_I may have been converted into a different local
4949 variable. In which case, we need to keep the assignment. */
4950 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4952 basic_block entry_succ_bb
= single_succ (entry_bb
);
4954 gimple parcopy_stmt
= NULL
;
4956 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4960 gcc_assert (!gsi_end_p (gsi
));
4961 stmt
= gsi_stmt (gsi
);
4962 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4965 if (gimple_num_ops (stmt
) == 2)
4967 tree arg
= gimple_assign_rhs1 (stmt
);
4969 /* We're ignore the subcode because we're
4970 effectively doing a STRIP_NOPS. */
4972 if (TREE_CODE (arg
) == ADDR_EXPR
4973 && TREE_OPERAND (arg
, 0)
4974 == gimple_omp_taskreg_data_arg (entry_stmt
))
4976 parcopy_stmt
= stmt
;
4982 gcc_assert (parcopy_stmt
!= NULL
);
4983 arg
= DECL_ARGUMENTS (child_fn
);
4985 if (!gimple_in_ssa_p (cfun
))
4987 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4988 gsi_remove (&gsi
, true);
4991 /* ?? Is setting the subcode really necessary ?? */
4992 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4993 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4998 /* If we are in ssa form, we must load the value from the default
4999 definition of the argument. That should not be defined now,
5000 since the argument is not used uninitialized. */
5001 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5002 narg
= make_ssa_name (arg
, gimple_build_nop ());
5003 set_ssa_default_def (cfun
, arg
, narg
);
5004 /* ?? Is setting the subcode really necessary ?? */
5005 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5006 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5007 update_stmt (parcopy_stmt
);
5011 /* Declare local variables needed in CHILD_CFUN. */
5012 block
= DECL_INITIAL (child_fn
);
5013 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5014 /* The gimplifier could record temporaries in parallel/task block
5015 rather than in containing function's local_decls chain,
5016 which would mean cgraph missed finalizing them. Do it now. */
5017 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5018 if (TREE_CODE (t
) == VAR_DECL
5020 && !DECL_EXTERNAL (t
))
5021 varpool_node::finalize_decl (t
);
5022 DECL_SAVED_TREE (child_fn
) = NULL
;
5023 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5024 gimple_set_body (child_fn
, NULL
);
5025 TREE_USED (block
) = 1;
5027 /* Reset DECL_CONTEXT on function arguments. */
5028 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5029 DECL_CONTEXT (t
) = child_fn
;
5031 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5032 so that it can be moved to the child function. */
5033 gsi
= gsi_last_bb (entry_bb
);
5034 stmt
= gsi_stmt (gsi
);
5035 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5036 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5037 gsi_remove (&gsi
, true);
5038 e
= split_block (entry_bb
, stmt
);
5040 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5042 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5045 gsi
= gsi_last_bb (exit_bb
);
5046 gcc_assert (!gsi_end_p (gsi
)
5047 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5048 stmt
= gimple_build_return (NULL
);
5049 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5050 gsi_remove (&gsi
, true);
5053 /* Move the parallel region into CHILD_CFUN. */
5055 if (gimple_in_ssa_p (cfun
))
5057 init_tree_ssa (child_cfun
);
5058 init_ssa_operands (child_cfun
);
5059 child_cfun
->gimple_df
->in_ssa_p
= true;
5063 block
= gimple_block (entry_stmt
);
5065 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5067 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5068 /* When the OMP expansion process cannot guarantee an up-to-date
5069 loop tree arrange for the child function to fixup loops. */
5070 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5071 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5073 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5074 num
= vec_safe_length (child_cfun
->local_decls
);
5075 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5077 t
= (*child_cfun
->local_decls
)[srcidx
];
5078 if (DECL_CONTEXT (t
) == cfun
->decl
)
5080 if (srcidx
!= dstidx
)
5081 (*child_cfun
->local_decls
)[dstidx
] = t
;
5085 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5087 /* Inform the callgraph about the new function. */
5088 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5089 cgraph_node::add_new_function (child_fn
, true);
5091 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5092 fixed in a following pass. */
5093 push_cfun (child_cfun
);
5095 optimize_omp_library_calls (entry_stmt
);
5096 cgraph_edge::rebuild_edges ();
5098 /* Some EH regions might become dead, see PR34608. If
5099 pass_cleanup_cfg isn't the first pass to happen with the
5100 new child, these dead EH edges might cause problems.
5101 Clean them up now. */
5102 if (flag_exceptions
)
5105 bool changed
= false;
5107 FOR_EACH_BB_FN (bb
, cfun
)
5108 changed
|= gimple_purge_dead_eh_edges (bb
);
5110 cleanup_tree_cfg ();
5112 if (gimple_in_ssa_p (cfun
))
5113 update_ssa (TODO_update_ssa
);
5117 /* Emit a library call to launch the children threads. */
5119 expand_cilk_for_call (new_bb
, entry_stmt
, ws_args
);
5120 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5121 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5123 expand_task_call (new_bb
, entry_stmt
);
5124 if (gimple_in_ssa_p (cfun
))
5125 update_ssa (TODO_update_ssa_only_virtuals
);
5129 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5130 of the combined collapse > 1 loop constructs, generate code like:
5131 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5136 count3 = (adj + N32 - N31) / STEP3;
5137 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5142 count2 = (adj + N22 - N21) / STEP2;
5143 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5148 count1 = (adj + N12 - N11) / STEP1;
5149 count = count1 * count2 * count3;
5150 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5152 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5153 of the combined loop constructs, just initialize COUNTS array
5154 from the _looptemp_ clauses. */
5156 /* NOTE: It *could* be better to moosh all of the BBs together,
5157 creating one larger BB with all the computation and the unexpected
5158 jump at the end. I.e.
5160 bool zero3, zero2, zero1, zero;
5163 count3 = (N32 - N31) /[cl] STEP3;
5165 count2 = (N22 - N21) /[cl] STEP2;
5167 count1 = (N12 - N11) /[cl] STEP1;
5168 zero = zero3 || zero2 || zero1;
5169 count = count1 * count2 * count3;
5170 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5172 After all, we expect the zero=false, and thus we expect to have to
5173 evaluate all of the comparison expressions, so short-circuiting
5174 oughtn't be a win. Since the condition isn't protecting a
5175 denominator, we're not concerned about divide-by-zero, so we can
5176 fully evaluate count even if a numerator turned out to be wrong.
5178 It seems like putting this all together would create much better
5179 scheduling opportunities, and less pressure on the chip's branch
5183 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5184 basic_block
&entry_bb
, tree
*counts
,
5185 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5186 basic_block
&l2_dom_bb
)
5188 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5193 /* Collapsed loops need work for expansion into SSA form. */
5194 gcc_assert (!gimple_in_ssa_p (cfun
));
5196 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5197 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5199 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5200 isn't supposed to be handled, as the inner loop doesn't
5202 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5203 OMP_CLAUSE__LOOPTEMP_
);
5204 gcc_assert (innerc
);
5205 for (i
= 0; i
< fd
->collapse
; i
++)
5207 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5208 OMP_CLAUSE__LOOPTEMP_
);
5209 gcc_assert (innerc
);
5211 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5213 counts
[0] = NULL_TREE
;
5218 for (i
= 0; i
< fd
->collapse
; i
++)
5220 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5222 if (SSA_VAR_P (fd
->loop
.n2
)
5223 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5224 fold_convert (itype
, fd
->loops
[i
].n1
),
5225 fold_convert (itype
, fd
->loops
[i
].n2
)))
5226 == NULL_TREE
|| !integer_onep (t
)))
5229 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5230 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5231 true, GSI_SAME_STMT
);
5232 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5233 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5234 true, GSI_SAME_STMT
);
5235 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5236 NULL_TREE
, NULL_TREE
);
5237 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5238 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5239 expand_omp_regimplify_p
, NULL
, NULL
)
5240 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5241 expand_omp_regimplify_p
, NULL
, NULL
))
5243 *gsi
= gsi_for_stmt (stmt
);
5244 gimple_regimplify_operands (stmt
, gsi
);
5246 e
= split_block (entry_bb
, stmt
);
5247 if (zero_iter_bb
== NULL
)
5249 first_zero_iter
= i
;
5250 zero_iter_bb
= create_empty_bb (entry_bb
);
5251 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5252 *gsi
= gsi_after_labels (zero_iter_bb
);
5253 stmt
= gimple_build_assign (fd
->loop
.n2
,
5254 build_zero_cst (type
));
5255 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5256 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5259 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5260 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5261 e
->flags
= EDGE_TRUE_VALUE
;
5262 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5263 if (l2_dom_bb
== NULL
)
5264 l2_dom_bb
= entry_bb
;
5266 *gsi
= gsi_last_bb (entry_bb
);
5269 if (POINTER_TYPE_P (itype
))
5270 itype
= signed_type_for (itype
);
5271 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5273 t
= fold_build2 (PLUS_EXPR
, itype
,
5274 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5275 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5276 fold_convert (itype
, fd
->loops
[i
].n2
));
5277 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5278 fold_convert (itype
, fd
->loops
[i
].n1
));
5279 /* ?? We could probably use CEIL_DIV_EXPR instead of
5280 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5281 generate the same code in the end because generically we
5282 don't know that the values involved must be negative for
5284 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5285 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5286 fold_build1 (NEGATE_EXPR
, itype
, t
),
5287 fold_build1 (NEGATE_EXPR
, itype
,
5288 fold_convert (itype
,
5289 fd
->loops
[i
].step
)));
5291 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5292 fold_convert (itype
, fd
->loops
[i
].step
));
5293 t
= fold_convert (type
, t
);
5294 if (TREE_CODE (t
) == INTEGER_CST
)
5298 counts
[i
] = create_tmp_reg (type
, ".count");
5299 expand_omp_build_assign (gsi
, counts
[i
], t
);
5301 if (SSA_VAR_P (fd
->loop
.n2
))
5306 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5307 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5313 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5315 V3 = N31 + (T % count3) * STEP3;
5317 V2 = N21 + (T % count2) * STEP2;
5319 V1 = N11 + T * STEP1;
5320 if this loop doesn't have an inner loop construct combined with it.
5321 If it does have an inner loop construct combined with it and the
5322 iteration count isn't known constant, store values from counts array
5323 into its _looptemp_ temporaries instead. */
5326 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5327 tree
*counts
, gimple inner_stmt
, tree startvar
)
5330 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5332 /* If fd->loop.n2 is constant, then no propagation of the counts
5333 is needed, they are constant. */
5334 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5337 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5338 ? gimple_omp_parallel_clauses (inner_stmt
)
5339 : gimple_omp_for_clauses (inner_stmt
);
5340 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5341 isn't supposed to be handled, as the inner loop doesn't
5343 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5344 gcc_assert (innerc
);
5345 for (i
= 0; i
< fd
->collapse
; i
++)
5347 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5348 OMP_CLAUSE__LOOPTEMP_
);
5349 gcc_assert (innerc
);
5352 tree tem
= OMP_CLAUSE_DECL (innerc
);
5353 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5354 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5355 false, GSI_CONTINUE_LINKING
);
5356 gimple stmt
= gimple_build_assign (tem
, t
);
5357 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5363 tree type
= TREE_TYPE (fd
->loop
.v
);
5364 tree tem
= create_tmp_reg (type
, ".tem");
5365 gimple stmt
= gimple_build_assign (tem
, startvar
);
5366 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5368 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5370 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5372 if (POINTER_TYPE_P (vtype
))
5373 itype
= signed_type_for (vtype
);
5375 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5378 t
= fold_convert (itype
, t
);
5379 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5380 fold_convert (itype
, fd
->loops
[i
].step
));
5381 if (POINTER_TYPE_P (vtype
))
5382 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5384 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5385 t
= force_gimple_operand_gsi (gsi
, t
,
5386 DECL_P (fd
->loops
[i
].v
)
5387 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5389 GSI_CONTINUE_LINKING
);
5390 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5391 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5394 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5395 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5396 false, GSI_CONTINUE_LINKING
);
5397 stmt
= gimple_build_assign (tem
, t
);
5398 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5404 /* Helper function for expand_omp_for_*. Generate code like:
5407 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5411 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5418 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5419 basic_block body_bb
)
5421 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5423 gimple_stmt_iterator gsi
;
5429 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5431 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5433 bb
= create_empty_bb (last_bb
);
5434 add_bb_to_loop (bb
, last_bb
->loop_father
);
5435 gsi
= gsi_start_bb (bb
);
5437 if (i
< fd
->collapse
- 1)
5439 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5440 e
->probability
= REG_BR_PROB_BASE
/ 8;
5442 t
= fd
->loops
[i
+ 1].n1
;
5443 t
= force_gimple_operand_gsi (&gsi
, t
,
5444 DECL_P (fd
->loops
[i
+ 1].v
)
5445 && TREE_ADDRESSABLE (fd
->loops
[i
5448 GSI_CONTINUE_LINKING
);
5449 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5450 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5455 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5457 if (POINTER_TYPE_P (vtype
))
5458 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5460 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5461 t
= force_gimple_operand_gsi (&gsi
, t
,
5462 DECL_P (fd
->loops
[i
].v
)
5463 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5464 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5465 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5466 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5470 t
= fd
->loops
[i
].n2
;
5471 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5472 false, GSI_CONTINUE_LINKING
);
5473 tree v
= fd
->loops
[i
].v
;
5474 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5475 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5476 false, GSI_CONTINUE_LINKING
);
5477 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5478 stmt
= gimple_build_cond_empty (t
);
5479 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5480 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5481 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5484 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5492 /* A subroutine of expand_omp_for. Generate code for a parallel
5493 loop with any schedule. Given parameters:
5495 for (V = N1; V cond N2; V += STEP) BODY;
5497 where COND is "<" or ">", we generate pseudocode
5499 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5500 if (more) goto L0; else goto L3;
5507 if (V cond iend) goto L1; else goto L2;
5509 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5512 If this is a combined omp parallel loop, instead of the call to
5513 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5514 If this is gimple_omp_for_combined_p loop, then instead of assigning
5515 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5516 inner GIMPLE_OMP_FOR and V += STEP; and
5517 if (V cond iend) goto L1; else goto L2; are removed.
5519 For collapsed loops, given parameters:
5521 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5522 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5523 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5526 we generate pseudocode
5528 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5533 count3 = (adj + N32 - N31) / STEP3;
5534 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5539 count2 = (adj + N22 - N21) / STEP2;
5540 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5545 count1 = (adj + N12 - N11) / STEP1;
5546 count = count1 * count2 * count3;
5551 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5552 if (more) goto L0; else goto L3;
5556 V3 = N31 + (T % count3) * STEP3;
5558 V2 = N21 + (T % count2) * STEP2;
5560 V1 = N11 + T * STEP1;
5565 if (V < iend) goto L10; else goto L2;
5568 if (V3 cond3 N32) goto L1; else goto L11;
5572 if (V2 cond2 N22) goto L1; else goto L12;
5578 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5584 expand_omp_for_generic (struct omp_region
*region
,
5585 struct omp_for_data
*fd
,
5586 enum built_in_function start_fn
,
5587 enum built_in_function next_fn
,
5590 tree type
, istart0
, iend0
, iend
;
5591 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5592 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5593 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5594 gimple_stmt_iterator gsi
;
5595 gimple_assign assign_stmt
;
5596 bool in_combined_parallel
= is_combined_parallel (region
);
5597 bool broken_loop
= region
->cont
== NULL
;
5599 tree
*counts
= NULL
;
5602 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5603 gcc_assert (fd
->iter_type
== long_integer_type_node
5604 || !in_combined_parallel
);
5606 type
= TREE_TYPE (fd
->loop
.v
);
5607 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5608 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5609 TREE_ADDRESSABLE (istart0
) = 1;
5610 TREE_ADDRESSABLE (iend0
) = 1;
5612 /* See if we need to bias by LLONG_MIN. */
5613 if (fd
->iter_type
== long_long_unsigned_type_node
5614 && TREE_CODE (type
) == INTEGER_TYPE
5615 && !TYPE_UNSIGNED (type
))
5619 if (fd
->loop
.cond_code
== LT_EXPR
)
5622 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5626 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5629 if (TREE_CODE (n1
) != INTEGER_CST
5630 || TREE_CODE (n2
) != INTEGER_CST
5631 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5632 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5635 entry_bb
= region
->entry
;
5636 cont_bb
= region
->cont
;
5638 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5639 gcc_assert (broken_loop
5640 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5641 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5642 l1_bb
= single_succ (l0_bb
);
5645 l2_bb
= create_empty_bb (cont_bb
);
5646 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5647 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5651 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5652 exit_bb
= region
->exit
;
5654 gsi
= gsi_last_bb (entry_bb
);
5656 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5657 if (fd
->collapse
> 1)
5659 int first_zero_iter
= -1;
5660 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5662 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5663 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5664 zero_iter_bb
, first_zero_iter
,
5669 /* Some counts[i] vars might be uninitialized if
5670 some loop has zero iterations. But the body shouldn't
5671 be executed in that case, so just avoid uninit warnings. */
5672 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5673 if (SSA_VAR_P (counts
[i
]))
5674 TREE_NO_WARNING (counts
[i
]) = 1;
5676 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5678 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5679 gsi
= gsi_last_bb (entry_bb
);
5680 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5681 get_immediate_dominator (CDI_DOMINATORS
,
5685 if (in_combined_parallel
)
5687 /* In a combined parallel loop, emit a call to
5688 GOMP_loop_foo_next. */
5689 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5690 build_fold_addr_expr (istart0
),
5691 build_fold_addr_expr (iend0
));
5695 tree t0
, t1
, t2
, t3
, t4
;
5696 /* If this is not a combined parallel loop, emit a call to
5697 GOMP_loop_foo_start in ENTRY_BB. */
5698 t4
= build_fold_addr_expr (iend0
);
5699 t3
= build_fold_addr_expr (istart0
);
5700 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5703 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5705 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5706 OMP_CLAUSE__LOOPTEMP_
);
5707 gcc_assert (innerc
);
5708 t0
= OMP_CLAUSE_DECL (innerc
);
5709 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5710 OMP_CLAUSE__LOOPTEMP_
);
5711 gcc_assert (innerc
);
5712 t1
= OMP_CLAUSE_DECL (innerc
);
5714 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5715 && TYPE_PRECISION (TREE_TYPE (t0
))
5716 != TYPE_PRECISION (fd
->iter_type
))
5718 /* Avoid casting pointers to integer of a different size. */
5719 tree itype
= signed_type_for (type
);
5720 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5721 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5725 t1
= fold_convert (fd
->iter_type
, t1
);
5726 t0
= fold_convert (fd
->iter_type
, t0
);
5730 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5731 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5733 if (fd
->iter_type
== long_integer_type_node
)
5737 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5738 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5739 6, t0
, t1
, t2
, t
, t3
, t4
);
5742 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5743 5, t0
, t1
, t2
, t3
, t4
);
5751 /* The GOMP_loop_ull_*start functions have additional boolean
5752 argument, true for < loops and false for > loops.
5753 In Fortran, the C bool type can be different from
5754 boolean_type_node. */
5755 bfn_decl
= builtin_decl_explicit (start_fn
);
5756 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5757 t5
= build_int_cst (c_bool_type
,
5758 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5761 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5762 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5763 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5766 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5767 6, t5
, t0
, t1
, t2
, t3
, t4
);
5770 if (TREE_TYPE (t
) != boolean_type_node
)
5771 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5772 t
, build_int_cst (TREE_TYPE (t
), 0));
5773 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5774 true, GSI_SAME_STMT
);
5775 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5777 /* Remove the GIMPLE_OMP_FOR statement. */
5778 gsi_remove (&gsi
, true);
5780 /* Iteration setup for sequential loop goes in L0_BB. */
5781 tree startvar
= fd
->loop
.v
;
5782 tree endvar
= NULL_TREE
;
5784 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5786 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5787 && gimple_omp_for_kind (inner_stmt
)
5788 == GF_OMP_FOR_KIND_SIMD
);
5789 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5790 OMP_CLAUSE__LOOPTEMP_
);
5791 gcc_assert (innerc
);
5792 startvar
= OMP_CLAUSE_DECL (innerc
);
5793 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5794 OMP_CLAUSE__LOOPTEMP_
);
5795 gcc_assert (innerc
);
5796 endvar
= OMP_CLAUSE_DECL (innerc
);
5799 gsi
= gsi_start_bb (l0_bb
);
5802 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5803 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5804 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5805 t
= fold_convert (TREE_TYPE (startvar
), t
);
5806 t
= force_gimple_operand_gsi (&gsi
, t
,
5808 && TREE_ADDRESSABLE (startvar
),
5809 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5810 assign_stmt
= gimple_build_assign (startvar
, t
);
5811 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5815 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5816 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5817 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5818 t
= fold_convert (TREE_TYPE (startvar
), t
);
5819 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5820 false, GSI_CONTINUE_LINKING
);
5823 assign_stmt
= gimple_build_assign (endvar
, iend
);
5824 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5825 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5826 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5828 assign_stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5830 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5832 if (fd
->collapse
> 1)
5833 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5837 /* Code to control the increment and predicate for the sequential
5838 loop goes in the CONT_BB. */
5839 gsi
= gsi_last_bb (cont_bb
);
5840 gimple_omp_continue cont_stmt
=
5841 as_a
<gimple_omp_continue
> (gsi_stmt (gsi
));
5842 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
5843 vmain
= gimple_omp_continue_control_use (cont_stmt
);
5844 vback
= gimple_omp_continue_control_def (cont_stmt
);
5846 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5848 if (POINTER_TYPE_P (type
))
5849 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5851 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5852 t
= force_gimple_operand_gsi (&gsi
, t
,
5854 && TREE_ADDRESSABLE (vback
),
5855 NULL_TREE
, true, GSI_SAME_STMT
);
5856 assign_stmt
= gimple_build_assign (vback
, t
);
5857 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
5859 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5860 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5862 gimple_cond cond_stmt
= gimple_build_cond_empty (t
);
5863 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
5866 /* Remove GIMPLE_OMP_CONTINUE. */
5867 gsi_remove (&gsi
, true);
5869 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5870 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5872 /* Emit code to get the next parallel iteration in L2_BB. */
5873 gsi
= gsi_start_bb (l2_bb
);
5875 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5876 build_fold_addr_expr (istart0
),
5877 build_fold_addr_expr (iend0
));
5878 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5879 false, GSI_CONTINUE_LINKING
);
5880 if (TREE_TYPE (t
) != boolean_type_node
)
5881 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5882 t
, build_int_cst (TREE_TYPE (t
), 0));
5883 gimple_cond cond_stmt
= gimple_build_cond_empty (t
);
5884 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
5887 /* Add the loop cleanup function. */
5888 gsi
= gsi_last_bb (exit_bb
);
5889 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5890 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5891 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5892 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5894 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5895 gimple_call call_stmt
= gimple_build_call (t
, 0);
5896 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5897 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5898 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
5899 gsi_remove (&gsi
, true);
5901 /* Connect the new blocks. */
5902 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5903 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5909 e
= find_edge (cont_bb
, l3_bb
);
5910 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5912 phis
= phi_nodes (l3_bb
);
5913 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5915 gimple phi
= gsi_stmt (gsi
);
5916 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5917 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5921 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5922 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5923 e
= find_edge (cont_bb
, l1_bb
);
5924 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5929 else if (fd
->collapse
> 1)
5932 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5935 e
->flags
= EDGE_TRUE_VALUE
;
5938 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5939 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5943 e
= find_edge (cont_bb
, l2_bb
);
5944 e
->flags
= EDGE_FALLTHRU
;
5946 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5948 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5949 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5950 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5951 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5952 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5953 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5954 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5955 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5957 struct loop
*outer_loop
= alloc_loop ();
5958 outer_loop
->header
= l0_bb
;
5959 outer_loop
->latch
= l2_bb
;
5960 add_loop (outer_loop
, l0_bb
->loop_father
);
5962 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5964 struct loop
*loop
= alloc_loop ();
5965 loop
->header
= l1_bb
;
5966 /* The loop may have multiple latches. */
5967 add_loop (loop
, outer_loop
);
5973 /* A subroutine of expand_omp_for. Generate code for a parallel
5974 loop with static schedule and no specified chunk size. Given
5977 for (V = N1; V cond N2; V += STEP) BODY;
5979 where COND is "<" or ">", we generate pseudocode
5981 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5986 if ((__typeof (V)) -1 > 0 && cond is >)
5987 n = -(adj + N2 - N1) / -STEP;
5989 n = (adj + N2 - N1) / STEP;
5992 if (threadid < tt) goto L3; else goto L4;
5997 s0 = q * threadid + tt;
6000 if (s0 >= e0) goto L2; else goto L0;
6006 if (V cond e) goto L1;
6011 expand_omp_for_static_nochunk (struct omp_region
*region
,
6012 struct omp_for_data
*fd
,
6015 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6016 tree type
, itype
, vmain
, vback
;
6017 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6018 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6020 gimple_stmt_iterator gsi
;
6022 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6023 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6024 bool broken_loop
= region
->cont
== NULL
;
6025 tree
*counts
= NULL
;
6028 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6029 if (POINTER_TYPE_P (type
))
6030 itype
= signed_type_for (type
);
6032 entry_bb
= region
->entry
;
6033 cont_bb
= region
->cont
;
6034 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6035 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6036 gcc_assert (broken_loop
6037 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6038 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6039 body_bb
= single_succ (seq_start_bb
);
6042 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6043 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6045 exit_bb
= region
->exit
;
6047 /* Iteration space partitioning goes in ENTRY_BB. */
6048 gsi
= gsi_last_bb (entry_bb
);
6049 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6051 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6053 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6054 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6057 if (fd
->collapse
> 1)
6059 int first_zero_iter
= -1;
6060 basic_block l2_dom_bb
= NULL
;
6062 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6063 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6064 fin_bb
, first_zero_iter
,
6068 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6069 t
= integer_one_node
;
6071 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6072 fold_convert (type
, fd
->loop
.n1
),
6073 fold_convert (type
, fd
->loop
.n2
));
6074 if (fd
->collapse
== 1
6075 && TYPE_UNSIGNED (type
)
6076 && (t
== NULL_TREE
|| !integer_onep (t
)))
6078 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6079 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6080 true, GSI_SAME_STMT
);
6081 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6082 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6083 true, GSI_SAME_STMT
);
6084 gimple_cond cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6085 NULL_TREE
, NULL_TREE
);
6086 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6087 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6088 expand_omp_regimplify_p
, NULL
, NULL
)
6089 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6090 expand_omp_regimplify_p
, NULL
, NULL
))
6092 gsi
= gsi_for_stmt (cond_stmt
);
6093 gimple_regimplify_operands (cond_stmt
, &gsi
);
6095 ep
= split_block (entry_bb
, cond_stmt
);
6096 ep
->flags
= EDGE_TRUE_VALUE
;
6097 entry_bb
= ep
->dest
;
6098 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6099 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6100 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6101 if (gimple_in_ssa_p (cfun
))
6103 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6104 for (gsi
= gsi_start_phis (fin_bb
);
6105 !gsi_end_p (gsi
); gsi_next (&gsi
))
6107 gimple phi
= gsi_stmt (gsi
);
6108 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6109 ep
, UNKNOWN_LOCATION
);
6112 gsi
= gsi_last_bb (entry_bb
);
6115 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6116 t
= fold_convert (itype
, t
);
6117 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6118 true, GSI_SAME_STMT
);
6120 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6121 t
= fold_convert (itype
, t
);
6122 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6123 true, GSI_SAME_STMT
);
6127 step
= fd
->loop
.step
;
6128 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6130 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6131 OMP_CLAUSE__LOOPTEMP_
);
6132 gcc_assert (innerc
);
6133 n1
= OMP_CLAUSE_DECL (innerc
);
6134 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6135 OMP_CLAUSE__LOOPTEMP_
);
6136 gcc_assert (innerc
);
6137 n2
= OMP_CLAUSE_DECL (innerc
);
6139 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6140 true, NULL_TREE
, true, GSI_SAME_STMT
);
6141 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6142 true, NULL_TREE
, true, GSI_SAME_STMT
);
6143 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6144 true, NULL_TREE
, true, GSI_SAME_STMT
);
6146 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6147 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6148 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6149 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6150 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6151 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6152 fold_build1 (NEGATE_EXPR
, itype
, t
),
6153 fold_build1 (NEGATE_EXPR
, itype
, step
));
6155 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6156 t
= fold_convert (itype
, t
);
6157 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6159 q
= create_tmp_reg (itype
, "q");
6160 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6161 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6162 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6164 tt
= create_tmp_reg (itype
, "tt");
6165 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6166 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6167 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6169 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6170 gimple_cond cond_stmt
= gimple_build_cond_empty (t
);
6171 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6173 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6174 gsi
= gsi_last_bb (second_bb
);
6175 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6177 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6179 gimple_assign assign_stmt
=
6180 gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6181 build_int_cst (itype
, 1));
6182 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6184 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6185 gsi
= gsi_last_bb (third_bb
);
6186 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6188 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6189 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6190 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6192 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6193 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6195 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6196 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6198 /* Remove the GIMPLE_OMP_FOR statement. */
6199 gsi_remove (&gsi
, true);
6201 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6202 gsi
= gsi_start_bb (seq_start_bb
);
6204 tree startvar
= fd
->loop
.v
;
6205 tree endvar
= NULL_TREE
;
6207 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6209 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6210 ? gimple_omp_parallel_clauses (inner_stmt
)
6211 : gimple_omp_for_clauses (inner_stmt
);
6212 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6213 gcc_assert (innerc
);
6214 startvar
= OMP_CLAUSE_DECL (innerc
);
6215 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6216 OMP_CLAUSE__LOOPTEMP_
);
6217 gcc_assert (innerc
);
6218 endvar
= OMP_CLAUSE_DECL (innerc
);
6220 t
= fold_convert (itype
, s0
);
6221 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6222 if (POINTER_TYPE_P (type
))
6223 t
= fold_build_pointer_plus (n1
, t
);
6225 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6226 t
= fold_convert (TREE_TYPE (startvar
), t
);
6227 t
= force_gimple_operand_gsi (&gsi
, t
,
6229 && TREE_ADDRESSABLE (startvar
),
6230 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6231 assign_stmt
= gimple_build_assign (startvar
, t
);
6232 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6234 t
= fold_convert (itype
, e0
);
6235 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6236 if (POINTER_TYPE_P (type
))
6237 t
= fold_build_pointer_plus (n1
, t
);
6239 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6240 t
= fold_convert (TREE_TYPE (startvar
), t
);
6241 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6242 false, GSI_CONTINUE_LINKING
);
6245 assign_stmt
= gimple_build_assign (endvar
, e
);
6246 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6247 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6248 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6250 assign_stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6252 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6254 if (fd
->collapse
> 1)
6255 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6259 /* The code controlling the sequential loop replaces the
6260 GIMPLE_OMP_CONTINUE. */
6261 gsi
= gsi_last_bb (cont_bb
);
6262 gimple_omp_continue cont_stmt
=
6263 as_a
<gimple_omp_continue
> (gsi_stmt (gsi
));
6264 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6265 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6266 vback
= gimple_omp_continue_control_def (cont_stmt
);
6268 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6270 if (POINTER_TYPE_P (type
))
6271 t
= fold_build_pointer_plus (vmain
, step
);
6273 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6274 t
= force_gimple_operand_gsi (&gsi
, t
,
6276 && TREE_ADDRESSABLE (vback
),
6277 NULL_TREE
, true, GSI_SAME_STMT
);
6278 assign_stmt
= gimple_build_assign (vback
, t
);
6279 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6281 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6282 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6284 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6287 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6288 gsi_remove (&gsi
, true);
6290 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6291 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6294 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6295 gsi
= gsi_last_bb (exit_bb
);
6296 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6298 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6299 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6301 gsi_remove (&gsi
, true);
6303 /* Connect all the blocks. */
6304 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6305 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6306 ep
= find_edge (entry_bb
, second_bb
);
6307 ep
->flags
= EDGE_TRUE_VALUE
;
6308 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6309 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6310 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6314 ep
= find_edge (cont_bb
, body_bb
);
6315 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6320 else if (fd
->collapse
> 1)
6323 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6326 ep
->flags
= EDGE_TRUE_VALUE
;
6327 find_edge (cont_bb
, fin_bb
)->flags
6328 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6331 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6332 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6333 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6335 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6336 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6337 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6338 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6340 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6342 struct loop
*loop
= alloc_loop ();
6343 loop
->header
= body_bb
;
6344 if (collapse_bb
== NULL
)
6345 loop
->latch
= cont_bb
;
6346 add_loop (loop
, body_bb
->loop_father
);
6351 /* A subroutine of expand_omp_for. Generate code for a parallel
6352 loop with static schedule and a specified chunk size. Given
6355 for (V = N1; V cond N2; V += STEP) BODY;
6357 where COND is "<" or ">", we generate pseudocode
6359 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6364 if ((__typeof (V)) -1 > 0 && cond is >)
6365 n = -(adj + N2 - N1) / -STEP;
6367 n = (adj + N2 - N1) / STEP;
6369 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6370 here so that V is defined
6371 if the loop is not entered
6373 s0 = (trip * nthreads + threadid) * CHUNK;
6374 e0 = min(s0 + CHUNK, n);
6375 if (s0 < n) goto L1; else goto L4;
6382 if (V cond e) goto L2; else goto L3;
6390 expand_omp_for_static_chunk (struct omp_region
*region
,
6391 struct omp_for_data
*fd
, gimple inner_stmt
)
6393 tree n
, s0
, e0
, e
, t
;
6394 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6395 tree type
, itype
, vmain
, vback
, vextra
;
6396 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6397 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6398 gimple_stmt_iterator gsi
;
6400 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6401 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6402 bool broken_loop
= region
->cont
== NULL
;
6403 tree
*counts
= NULL
;
6406 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6407 if (POINTER_TYPE_P (type
))
6408 itype
= signed_type_for (type
);
6410 entry_bb
= region
->entry
;
6411 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6413 iter_part_bb
= se
->dest
;
6414 cont_bb
= region
->cont
;
6415 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6416 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6417 gcc_assert (broken_loop
6418 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6419 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6420 body_bb
= single_succ (seq_start_bb
);
6423 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6424 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6425 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6427 exit_bb
= region
->exit
;
6429 /* Trip and adjustment setup goes in ENTRY_BB. */
6430 gsi
= gsi_last_bb (entry_bb
);
6431 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6433 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6435 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6436 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6439 if (fd
->collapse
> 1)
6441 int first_zero_iter
= -1;
6442 basic_block l2_dom_bb
= NULL
;
6444 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6445 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6446 fin_bb
, first_zero_iter
,
6450 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6451 t
= integer_one_node
;
6453 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6454 fold_convert (type
, fd
->loop
.n1
),
6455 fold_convert (type
, fd
->loop
.n2
));
6456 if (fd
->collapse
== 1
6457 && TYPE_UNSIGNED (type
)
6458 && (t
== NULL_TREE
|| !integer_onep (t
)))
6460 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6461 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6462 true, GSI_SAME_STMT
);
6463 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6464 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6465 true, GSI_SAME_STMT
);
6466 gimple_cond cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6467 NULL_TREE
, NULL_TREE
);
6468 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6469 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6470 expand_omp_regimplify_p
, NULL
, NULL
)
6471 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6472 expand_omp_regimplify_p
, NULL
, NULL
))
6474 gsi
= gsi_for_stmt (cond_stmt
);
6475 gimple_regimplify_operands (cond_stmt
, &gsi
);
6477 se
= split_block (entry_bb
, cond_stmt
);
6478 se
->flags
= EDGE_TRUE_VALUE
;
6479 entry_bb
= se
->dest
;
6480 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6481 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6482 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6483 if (gimple_in_ssa_p (cfun
))
6485 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6486 for (gsi
= gsi_start_phis (fin_bb
);
6487 !gsi_end_p (gsi
); gsi_next (&gsi
))
6489 gimple phi
= gsi_stmt (gsi
);
6490 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6491 se
, UNKNOWN_LOCATION
);
6494 gsi
= gsi_last_bb (entry_bb
);
6497 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6498 t
= fold_convert (itype
, t
);
6499 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6500 true, GSI_SAME_STMT
);
6502 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6503 t
= fold_convert (itype
, t
);
6504 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6505 true, GSI_SAME_STMT
);
6509 step
= fd
->loop
.step
;
6510 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6512 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6513 OMP_CLAUSE__LOOPTEMP_
);
6514 gcc_assert (innerc
);
6515 n1
= OMP_CLAUSE_DECL (innerc
);
6516 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6517 OMP_CLAUSE__LOOPTEMP_
);
6518 gcc_assert (innerc
);
6519 n2
= OMP_CLAUSE_DECL (innerc
);
6521 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6522 true, NULL_TREE
, true, GSI_SAME_STMT
);
6523 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6524 true, NULL_TREE
, true, GSI_SAME_STMT
);
6525 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6526 true, NULL_TREE
, true, GSI_SAME_STMT
);
6528 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6529 true, NULL_TREE
, true, GSI_SAME_STMT
);
6531 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6532 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6533 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6534 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6535 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6536 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6537 fold_build1 (NEGATE_EXPR
, itype
, t
),
6538 fold_build1 (NEGATE_EXPR
, itype
, step
));
6540 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6541 t
= fold_convert (itype
, t
);
6542 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6543 true, GSI_SAME_STMT
);
6545 trip_var
= create_tmp_reg (itype
, ".trip");
6546 if (gimple_in_ssa_p (cfun
))
6548 trip_init
= make_ssa_name (trip_var
, NULL
);
6549 trip_main
= make_ssa_name (trip_var
, NULL
);
6550 trip_back
= make_ssa_name (trip_var
, NULL
);
6554 trip_init
= trip_var
;
6555 trip_main
= trip_var
;
6556 trip_back
= trip_var
;
6559 gimple_assign assign_stmt
=
6560 gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6561 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6563 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6564 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6565 if (POINTER_TYPE_P (type
))
6566 t
= fold_build_pointer_plus (n1
, t
);
6568 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6569 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6570 true, GSI_SAME_STMT
);
6572 /* Remove the GIMPLE_OMP_FOR. */
6573 gsi_remove (&gsi
, true);
6575 /* Iteration space partitioning goes in ITER_PART_BB. */
6576 gsi
= gsi_last_bb (iter_part_bb
);
6578 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6579 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6580 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6581 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6582 false, GSI_CONTINUE_LINKING
);
6584 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6585 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6586 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6587 false, GSI_CONTINUE_LINKING
);
6589 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6590 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6592 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6593 gsi
= gsi_start_bb (seq_start_bb
);
6595 tree startvar
= fd
->loop
.v
;
6596 tree endvar
= NULL_TREE
;
6598 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6600 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6601 ? gimple_omp_parallel_clauses (inner_stmt
)
6602 : gimple_omp_for_clauses (inner_stmt
);
6603 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6604 gcc_assert (innerc
);
6605 startvar
= OMP_CLAUSE_DECL (innerc
);
6606 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6607 OMP_CLAUSE__LOOPTEMP_
);
6608 gcc_assert (innerc
);
6609 endvar
= OMP_CLAUSE_DECL (innerc
);
6612 t
= fold_convert (itype
, s0
);
6613 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6614 if (POINTER_TYPE_P (type
))
6615 t
= fold_build_pointer_plus (n1
, t
);
6617 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6618 t
= fold_convert (TREE_TYPE (startvar
), t
);
6619 t
= force_gimple_operand_gsi (&gsi
, t
,
6621 && TREE_ADDRESSABLE (startvar
),
6622 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6623 assign_stmt
= gimple_build_assign (startvar
, t
);
6624 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6626 t
= fold_convert (itype
, e0
);
6627 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6628 if (POINTER_TYPE_P (type
))
6629 t
= fold_build_pointer_plus (n1
, t
);
6631 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6632 t
= fold_convert (TREE_TYPE (startvar
), t
);
6633 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6634 false, GSI_CONTINUE_LINKING
);
6637 assign_stmt
= gimple_build_assign (endvar
, e
);
6638 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6639 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6640 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6642 assign_stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6644 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6646 if (fd
->collapse
> 1)
6647 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6651 /* The code controlling the sequential loop goes in CONT_BB,
6652 replacing the GIMPLE_OMP_CONTINUE. */
6653 gsi
= gsi_last_bb (cont_bb
);
6654 gimple_omp_continue cont_stmt
=
6655 as_a
<gimple_omp_continue
> (gsi_stmt (gsi
));
6656 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6657 vback
= gimple_omp_continue_control_def (cont_stmt
);
6659 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6661 if (POINTER_TYPE_P (type
))
6662 t
= fold_build_pointer_plus (vmain
, step
);
6664 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6665 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6666 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6667 true, GSI_SAME_STMT
);
6668 assign_stmt
= gimple_build_assign (vback
, t
);
6669 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6671 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6672 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6674 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6677 /* Remove GIMPLE_OMP_CONTINUE. */
6678 gsi_remove (&gsi
, true);
6680 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6681 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6683 /* Trip update code goes into TRIP_UPDATE_BB. */
6684 gsi
= gsi_start_bb (trip_update_bb
);
6686 t
= build_int_cst (itype
, 1);
6687 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6688 assign_stmt
= gimple_build_assign (trip_back
, t
);
6689 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6692 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6693 gsi
= gsi_last_bb (exit_bb
);
6694 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6696 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6697 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6699 gsi_remove (&gsi
, true);
6701 /* Connect the new blocks. */
6702 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6703 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6707 se
= find_edge (cont_bb
, body_bb
);
6708 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6713 else if (fd
->collapse
> 1)
6716 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6719 se
->flags
= EDGE_TRUE_VALUE
;
6720 find_edge (cont_bb
, trip_update_bb
)->flags
6721 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6723 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6726 if (gimple_in_ssa_p (cfun
))
6728 gimple_stmt_iterator psi
;
6734 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6736 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6737 remove arguments of the phi nodes in fin_bb. We need to create
6738 appropriate phi nodes in iter_part_bb instead. */
6739 se
= single_pred_edge (fin_bb
);
6740 re
= single_succ_edge (trip_update_bb
);
6741 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
6742 ene
= single_succ_edge (entry_bb
);
6744 psi
= gsi_start_phis (fin_bb
);
6745 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6746 gsi_next (&psi
), ++i
)
6749 source_location locus
;
6751 phi
= gsi_stmt (psi
);
6752 t
= gimple_phi_result (phi
);
6753 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6754 nphi
= create_phi_node (t
, iter_part_bb
);
6756 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6757 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6759 /* A special case -- fd->loop.v is not yet computed in
6760 iter_part_bb, we need to use vextra instead. */
6761 if (t
== fd
->loop
.v
)
6763 add_phi_arg (nphi
, t
, ene
, locus
);
6764 locus
= redirect_edge_var_map_location (vm
);
6765 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6767 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6768 redirect_edge_var_map_clear (re
);
6771 psi
= gsi_start_phis (fin_bb
);
6772 if (gsi_end_p (psi
))
6774 remove_phi_node (&psi
, false);
6777 /* Make phi node for trip. */
6778 phi
= create_phi_node (trip_main
, iter_part_bb
);
6779 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6781 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6786 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6787 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6788 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6789 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6790 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6791 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6792 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6793 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6794 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6798 struct loop
*trip_loop
= alloc_loop ();
6799 trip_loop
->header
= iter_part_bb
;
6800 trip_loop
->latch
= trip_update_bb
;
6801 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6803 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6805 struct loop
*loop
= alloc_loop ();
6806 loop
->header
= body_bb
;
6807 if (collapse_bb
== NULL
)
6808 loop
->latch
= cont_bb
;
6809 add_loop (loop
, trip_loop
);
6814 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
6816 for (V = N1; V cond N2; V += STEP) BODY;
6818 where COND is "<" or ">" or "!=", we generate pseudocode
6820 for (ind_var = low; ind_var < high; ind_var++)
6822 V = n1 + (ind_var * STEP)
6827 In the above pseudocode, low and high are function parameters of the
6828 child function. In the function below, we are inserting a temp.
6829 variable that will be making a call to two OMP functions that will not be
6830 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
6831 with _Cilk_for). These functions are replaced with low and high
6832 by the function that handles taskreg. */
6836 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
6838 bool broken_loop
= region
->cont
== NULL
;
6839 basic_block entry_bb
= region
->entry
;
6840 basic_block cont_bb
= region
->cont
;
6842 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6843 gcc_assert (broken_loop
6844 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6845 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6846 basic_block l1_bb
, l2_bb
;
6850 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6851 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6852 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6853 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6857 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6858 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6859 l2_bb
= single_succ (l1_bb
);
6861 basic_block exit_bb
= region
->exit
;
6862 basic_block l2_dom_bb
= NULL
;
6864 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
6866 /* Below statements until the "tree high_val = ..." are pseudo statements
6867 used to pass information to be used by expand_omp_taskreg.
6868 low_val and high_val will be replaced by the __low and __high
6869 parameter from the child function.
6871 The call_exprs part is a place-holder, it is mainly used
6872 to distinctly identify to the top-level part that this is
6873 where we should put low and high (reasoning given in header
6877 = gimple_omp_parallel_child_fn (last_stmt (region
->outer
->entry
));
6878 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
6879 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
6881 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
6883 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
6886 gcc_assert (low_val
&& high_val
);
6888 tree type
= TREE_TYPE (low_val
);
6889 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
6890 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6892 /* Not needed in SSA form right now. */
6893 gcc_assert (!gimple_in_ssa_p (cfun
));
6894 if (l2_dom_bb
== NULL
)
6900 gimple stmt
= gimple_build_assign (ind_var
, n1
);
6902 /* Replace the GIMPLE_OMP_FOR statement. */
6903 gsi_replace (&gsi
, stmt
, true);
6907 /* Code to control the increment goes in the CONT_BB. */
6908 gsi
= gsi_last_bb (cont_bb
);
6909 stmt
= gsi_stmt (gsi
);
6910 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6911 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ind_var
, ind_var
,
6912 build_one_cst (type
));
6914 /* Replace GIMPLE_OMP_CONTINUE. */
6915 gsi_replace (&gsi
, stmt
, true);
6918 /* Emit the condition in L1_BB. */
6919 gsi
= gsi_after_labels (l1_bb
);
6920 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
6921 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
6923 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
6924 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6925 fd
->loop
.n1
, fold_convert (sizetype
, t
));
6927 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6928 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
6929 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
6930 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6932 /* The condition is always '<' since the runtime will fill in the low
6934 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
6935 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6937 /* Remove GIMPLE_OMP_RETURN. */
6938 gsi
= gsi_last_bb (exit_bb
);
6939 gsi_remove (&gsi
, true);
6941 /* Connect the new blocks. */
6942 remove_edge (FALLTHRU_EDGE (entry_bb
));
6947 remove_edge (BRANCH_EDGE (entry_bb
));
6948 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6950 e
= BRANCH_EDGE (l1_bb
);
6951 ne
= FALLTHRU_EDGE (l1_bb
);
6952 e
->flags
= EDGE_TRUE_VALUE
;
6956 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6958 ne
= single_succ_edge (l1_bb
);
6959 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6962 ne
->flags
= EDGE_FALSE_VALUE
;
6963 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6964 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6966 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6967 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6968 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6972 struct loop
*loop
= alloc_loop ();
6973 loop
->header
= l1_bb
;
6974 loop
->latch
= cont_bb
;
6975 add_loop (loop
, l1_bb
->loop_father
);
6976 loop
->safelen
= INT_MAX
;
6979 /* Pick the correct library function based on the precision of the
6980 induction variable type. */
6981 tree lib_fun
= NULL_TREE
;
6982 if (TYPE_PRECISION (type
) == 32)
6983 lib_fun
= cilk_for_32_fndecl
;
6984 else if (TYPE_PRECISION (type
) == 64)
6985 lib_fun
= cilk_for_64_fndecl
;
6989 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
6991 /* WS_ARGS contains the library function flavor to call:
6992 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
6993 user-defined grain value. If the user does not define one, then zero
6994 is passed in by the parser. */
6995 vec_alloc (region
->ws_args
, 2);
6996 region
->ws_args
->quick_push (lib_fun
);
6997 region
->ws_args
->quick_push (fd
->chunk_size
);
7000 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7001 loop. Given parameters:
7003 for (V = N1; V cond N2; V += STEP) BODY;
7005 where COND is "<" or ">", we generate pseudocode
7013 if (V cond N2) goto L0; else goto L2;
7016 For collapsed loops, given parameters:
7018 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7019 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7020 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7023 we generate pseudocode
7029 count3 = (adj + N32 - N31) / STEP3;
7034 count2 = (adj + N22 - N21) / STEP2;
7039 count1 = (adj + N12 - N11) / STEP1;
7040 count = count1 * count2 * count3;
7050 V2 += (V3 cond3 N32) ? 0 : STEP2;
7051 V3 = (V3 cond3 N32) ? V3 : N31;
7052 V1 += (V2 cond2 N22) ? 0 : STEP1;
7053 V2 = (V2 cond2 N22) ? V2 : N21;
7055 if (V < count) goto L0; else goto L2;
7061 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7064 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7065 gimple_stmt_iterator gsi
;
7067 bool broken_loop
= region
->cont
== NULL
;
7069 tree
*counts
= NULL
;
7071 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7072 OMP_CLAUSE_SAFELEN
);
7073 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7074 OMP_CLAUSE__SIMDUID_
);
7077 type
= TREE_TYPE (fd
->loop
.v
);
7078 entry_bb
= region
->entry
;
7079 cont_bb
= region
->cont
;
7080 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7081 gcc_assert (broken_loop
7082 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7083 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7086 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7087 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7088 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7089 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7093 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7094 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7095 l2_bb
= single_succ (l1_bb
);
7097 exit_bb
= region
->exit
;
7100 gsi
= gsi_last_bb (entry_bb
);
7102 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7103 /* Not needed in SSA form right now. */
7104 gcc_assert (!gimple_in_ssa_p (cfun
));
7105 if (fd
->collapse
> 1)
7107 int first_zero_iter
= -1;
7108 basic_block zero_iter_bb
= l2_bb
;
7110 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7111 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7112 zero_iter_bb
, first_zero_iter
,
7115 if (l2_dom_bb
== NULL
)
7120 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7122 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7123 OMP_CLAUSE__LOOPTEMP_
);
7124 gcc_assert (innerc
);
7125 n1
= OMP_CLAUSE_DECL (innerc
);
7126 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7127 OMP_CLAUSE__LOOPTEMP_
);
7128 gcc_assert (innerc
);
7129 n2
= OMP_CLAUSE_DECL (innerc
);
7130 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7131 fold_convert (type
, n1
));
7132 if (fd
->collapse
> 1)
7135 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7141 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7142 fold_convert (type
, fd
->loop
.n1
));
7143 if (fd
->collapse
> 1)
7144 for (i
= 0; i
< fd
->collapse
; i
++)
7146 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7147 if (POINTER_TYPE_P (itype
))
7148 itype
= signed_type_for (itype
);
7149 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7150 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7154 /* Remove the GIMPLE_OMP_FOR statement. */
7155 gsi_remove (&gsi
, true);
7159 /* Code to control the increment goes in the CONT_BB. */
7160 gsi
= gsi_last_bb (cont_bb
);
7161 stmt
= gsi_stmt (gsi
);
7162 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7164 if (POINTER_TYPE_P (type
))
7165 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7167 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7168 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7170 if (fd
->collapse
> 1)
7172 i
= fd
->collapse
- 1;
7173 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7175 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7176 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7180 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7182 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7185 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7187 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7189 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7190 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7191 if (POINTER_TYPE_P (itype2
))
7192 itype2
= signed_type_for (itype2
);
7193 t
= build3 (COND_EXPR
, itype2
,
7194 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7196 fold_convert (itype
, fd
->loops
[i
].n2
)),
7197 build_int_cst (itype2
, 0),
7198 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7199 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7200 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7202 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7203 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7205 t
= build3 (COND_EXPR
, itype
,
7206 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7208 fold_convert (itype
, fd
->loops
[i
].n2
)),
7210 fold_convert (itype
, fd
->loops
[i
].n1
));
7211 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7215 /* Remove GIMPLE_OMP_CONTINUE. */
7216 gsi_remove (&gsi
, true);
7219 /* Emit the condition in L1_BB. */
7220 gsi
= gsi_start_bb (l1_bb
);
7222 t
= fold_convert (type
, n2
);
7223 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7224 false, GSI_CONTINUE_LINKING
);
7225 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7226 stmt
= gimple_build_cond_empty (t
);
7227 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7228 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
7230 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
7233 gsi
= gsi_for_stmt (stmt
);
7234 gimple_regimplify_operands (stmt
, &gsi
);
7237 /* Remove GIMPLE_OMP_RETURN. */
7238 gsi
= gsi_last_bb (exit_bb
);
7239 gsi_remove (&gsi
, true);
7241 /* Connect the new blocks. */
7242 remove_edge (FALLTHRU_EDGE (entry_bb
));
7246 remove_edge (BRANCH_EDGE (entry_bb
));
7247 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7249 e
= BRANCH_EDGE (l1_bb
);
7250 ne
= FALLTHRU_EDGE (l1_bb
);
7251 e
->flags
= EDGE_TRUE_VALUE
;
7255 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7257 ne
= single_succ_edge (l1_bb
);
7258 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7261 ne
->flags
= EDGE_FALSE_VALUE
;
7262 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7263 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7265 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7266 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7267 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7271 struct loop
*loop
= alloc_loop ();
7272 loop
->header
= l1_bb
;
7273 loop
->latch
= cont_bb
;
7274 add_loop (loop
, l1_bb
->loop_father
);
7275 if (safelen
== NULL_TREE
)
7276 loop
->safelen
= INT_MAX
;
7279 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7280 if (TREE_CODE (safelen
) != INTEGER_CST
)
7282 else if (!tree_fits_uhwi_p (safelen
)
7283 || tree_to_uhwi (safelen
) > INT_MAX
)
7284 loop
->safelen
= INT_MAX
;
7286 loop
->safelen
= tree_to_uhwi (safelen
);
7287 if (loop
->safelen
== 1)
7292 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7293 cfun
->has_simduid_loops
= true;
7295 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7297 if ((flag_tree_loop_vectorize
7298 || (!global_options_set
.x_flag_tree_loop_vectorize
7299 && !global_options_set
.x_flag_tree_vectorize
))
7300 && flag_tree_loop_optimize
7301 && loop
->safelen
> 1)
7303 loop
->force_vectorize
= true;
7304 cfun
->has_force_vectorize_loops
= true;
7310 /* Expand the OpenMP loop defined by REGION. */
7313 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7315 struct omp_for_data fd
;
7316 struct omp_for_data_loop
*loops
;
7319 = (struct omp_for_data_loop
*)
7320 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7321 * sizeof (struct omp_for_data_loop
));
7322 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7323 region
->sched_kind
= fd
.sched_kind
;
7325 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7326 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7327 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7330 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7331 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7332 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7335 /* If there isn't a continue then this is a degerate case where
7336 the introduction of abnormal edges during lowering will prevent
7337 original loops from being detected. Fix that up. */
7338 loops_state_set (LOOPS_NEED_FIXUP
);
7340 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7341 expand_omp_simd (region
, &fd
);
7342 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7343 expand_cilk_for (region
, &fd
);
7344 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7345 && !fd
.have_ordered
)
7347 if (fd
.chunk_size
== NULL
)
7348 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7350 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7354 int fn_index
, start_ix
, next_ix
;
7356 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7357 == GF_OMP_FOR_KIND_FOR
);
7358 if (fd
.chunk_size
== NULL
7359 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7360 fd
.chunk_size
= integer_zero_node
;
7361 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7362 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7363 ? 3 : fd
.sched_kind
;
7364 fn_index
+= fd
.have_ordered
* 4;
7365 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7366 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7367 if (fd
.iter_type
== long_long_unsigned_type_node
)
7369 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7370 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7371 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7372 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7374 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7375 (enum built_in_function
) next_ix
, inner_stmt
);
7378 if (gimple_in_ssa_p (cfun
))
7379 update_ssa (TODO_update_ssa_only_virtuals
);
7383 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7385 v = GOMP_sections_start (n);
7402 v = GOMP_sections_next ();
7407 If this is a combined parallel sections, replace the call to
7408 GOMP_sections_start with call to GOMP_sections_next. */
7411 expand_omp_sections (struct omp_region
*region
)
7413 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7415 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7416 gimple_stmt_iterator si
, switch_si
;
7417 gimple sections_stmt
, stmt
;
7418 gimple_omp_continue cont
;
7421 struct omp_region
*inner
;
7423 bool exit_reachable
= region
->cont
!= NULL
;
7425 gcc_assert (region
->exit
!= NULL
);
7426 entry_bb
= region
->entry
;
7427 l0_bb
= single_succ (entry_bb
);
7428 l1_bb
= region
->cont
;
7429 l2_bb
= region
->exit
;
7430 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7431 l2
= gimple_block_label (l2_bb
);
7434 /* This can happen if there are reductions. */
7435 len
= EDGE_COUNT (l0_bb
->succs
);
7436 gcc_assert (len
> 0);
7437 e
= EDGE_SUCC (l0_bb
, len
- 1);
7438 si
= gsi_last_bb (e
->dest
);
7441 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7442 l2
= gimple_block_label (e
->dest
);
7444 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7446 si
= gsi_last_bb (e
->dest
);
7448 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7450 l2
= gimple_block_label (e
->dest
);
7456 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7458 default_bb
= create_empty_bb (l0_bb
);
7460 /* We will build a switch() with enough cases for all the
7461 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7462 and a default case to abort if something goes wrong. */
7463 len
= EDGE_COUNT (l0_bb
->succs
);
7465 /* Use vec::quick_push on label_vec throughout, since we know the size
7467 auto_vec
<tree
> label_vec (len
);
7469 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7470 GIMPLE_OMP_SECTIONS statement. */
7471 si
= gsi_last_bb (entry_bb
);
7472 sections_stmt
= gsi_stmt (si
);
7473 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7474 vin
= gimple_omp_sections_control (sections_stmt
);
7475 if (!is_combined_parallel (region
))
7477 /* If we are not inside a combined parallel+sections region,
7478 call GOMP_sections_start. */
7479 t
= build_int_cst (unsigned_type_node
, len
- 1);
7480 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7481 stmt
= gimple_build_call (u
, 1, t
);
7485 /* Otherwise, call GOMP_sections_next. */
7486 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7487 stmt
= gimple_build_call (u
, 0);
7489 gimple_call_set_lhs (stmt
, vin
);
7490 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7491 gsi_remove (&si
, true);
7493 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7495 switch_si
= gsi_last_bb (l0_bb
);
7496 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7499 cont
= as_a
<gimple_omp_continue
> (last_stmt (l1_bb
));
7500 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7501 vmain
= gimple_omp_continue_control_use (cont
);
7502 vnext
= gimple_omp_continue_control_def (cont
);
7510 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7511 label_vec
.quick_push (t
);
7514 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7515 for (inner
= region
->inner
, casei
= 1;
7517 inner
= inner
->next
, i
++, casei
++)
7519 basic_block s_entry_bb
, s_exit_bb
;
7521 /* Skip optional reduction region. */
7522 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7529 s_entry_bb
= inner
->entry
;
7530 s_exit_bb
= inner
->exit
;
7532 t
= gimple_block_label (s_entry_bb
);
7533 u
= build_int_cst (unsigned_type_node
, casei
);
7534 u
= build_case_label (u
, NULL
, t
);
7535 label_vec
.quick_push (u
);
7537 si
= gsi_last_bb (s_entry_bb
);
7538 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7539 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7540 gsi_remove (&si
, true);
7541 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7543 if (s_exit_bb
== NULL
)
7546 si
= gsi_last_bb (s_exit_bb
);
7547 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7548 gsi_remove (&si
, true);
7550 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7553 /* Error handling code goes in DEFAULT_BB. */
7554 t
= gimple_block_label (default_bb
);
7555 u
= build_case_label (NULL
, NULL
, t
);
7556 make_edge (l0_bb
, default_bb
, 0);
7557 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7559 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7560 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7561 gsi_remove (&switch_si
, true);
7563 si
= gsi_start_bb (default_bb
);
7564 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7565 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7571 /* Code to get the next section goes in L1_BB. */
7572 si
= gsi_last_bb (l1_bb
);
7573 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7575 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7576 stmt
= gimple_build_call (bfn_decl
, 0);
7577 gimple_call_set_lhs (stmt
, vnext
);
7578 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7579 gsi_remove (&si
, true);
7581 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7584 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7585 si
= gsi_last_bb (l2_bb
);
7586 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7587 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7588 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7589 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7591 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7592 stmt
= gimple_build_call (t
, 0);
7593 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7594 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7595 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7596 gsi_remove (&si
, true);
7598 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7602 /* Expand code for an OpenMP single directive. We've already expanded
7603 much of the code, here we simply place the GOMP_barrier call. */
7606 expand_omp_single (struct omp_region
*region
)
7608 basic_block entry_bb
, exit_bb
;
7609 gimple_stmt_iterator si
;
7611 entry_bb
= region
->entry
;
7612 exit_bb
= region
->exit
;
7614 si
= gsi_last_bb (entry_bb
);
7615 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7616 gsi_remove (&si
, true);
7617 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7619 si
= gsi_last_bb (exit_bb
);
7620 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7622 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7623 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7625 gsi_remove (&si
, true);
7626 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7630 /* Generic expansion for OpenMP synchronization directives: master,
7631 ordered and critical. All we need to do here is remove the entry
7632 and exit markers for REGION. */
7635 expand_omp_synch (struct omp_region
*region
)
7637 basic_block entry_bb
, exit_bb
;
7638 gimple_stmt_iterator si
;
7640 entry_bb
= region
->entry
;
7641 exit_bb
= region
->exit
;
7643 si
= gsi_last_bb (entry_bb
);
7644 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7645 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7646 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7647 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7648 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7649 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7650 gsi_remove (&si
, true);
7651 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7655 si
= gsi_last_bb (exit_bb
);
7656 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7657 gsi_remove (&si
, true);
7658 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7662 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7663 operation as a normal volatile load. */
7666 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7667 tree loaded_val
, int index
)
7669 enum built_in_function tmpbase
;
7670 gimple_stmt_iterator gsi
;
7671 basic_block store_bb
;
7674 tree decl
, call
, type
, itype
;
7676 gsi
= gsi_last_bb (load_bb
);
7677 stmt
= gsi_stmt (gsi
);
7678 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7679 loc
= gimple_location (stmt
);
7681 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7682 is smaller than word size, then expand_atomic_load assumes that the load
7683 is atomic. We could avoid the builtin entirely in this case. */
7685 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7686 decl
= builtin_decl_explicit (tmpbase
);
7687 if (decl
== NULL_TREE
)
7690 type
= TREE_TYPE (loaded_val
);
7691 itype
= TREE_TYPE (TREE_TYPE (decl
));
7693 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7694 build_int_cst (NULL
,
7695 gimple_omp_atomic_seq_cst_p (stmt
)
7697 : MEMMODEL_RELAXED
));
7698 if (!useless_type_conversion_p (type
, itype
))
7699 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7700 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7702 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7703 gsi_remove (&gsi
, true);
7705 store_bb
= single_succ (load_bb
);
7706 gsi
= gsi_last_bb (store_bb
);
7707 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7708 gsi_remove (&gsi
, true);
7710 if (gimple_in_ssa_p (cfun
))
7711 update_ssa (TODO_update_ssa_no_phi
);
7716 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7717 operation as a normal volatile store. */
7720 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7721 tree loaded_val
, tree stored_val
, int index
)
7723 enum built_in_function tmpbase
;
7724 gimple_stmt_iterator gsi
;
7725 basic_block store_bb
= single_succ (load_bb
);
7728 tree decl
, call
, type
, itype
;
7729 enum machine_mode imode
;
7732 gsi
= gsi_last_bb (load_bb
);
7733 stmt
= gsi_stmt (gsi
);
7734 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7736 /* If the load value is needed, then this isn't a store but an exchange. */
7737 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7739 gsi
= gsi_last_bb (store_bb
);
7740 stmt
= gsi_stmt (gsi
);
7741 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7742 loc
= gimple_location (stmt
);
7744 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7745 is smaller than word size, then expand_atomic_store assumes that the store
7746 is atomic. We could avoid the builtin entirely in this case. */
7748 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7749 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7750 decl
= builtin_decl_explicit (tmpbase
);
7751 if (decl
== NULL_TREE
)
7754 type
= TREE_TYPE (stored_val
);
7756 /* Dig out the type of the function's second argument. */
7757 itype
= TREE_TYPE (decl
);
7758 itype
= TYPE_ARG_TYPES (itype
);
7759 itype
= TREE_CHAIN (itype
);
7760 itype
= TREE_VALUE (itype
);
7761 imode
= TYPE_MODE (itype
);
7763 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7766 if (!useless_type_conversion_p (itype
, type
))
7767 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7768 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7769 build_int_cst (NULL
,
7770 gimple_omp_atomic_seq_cst_p (stmt
)
7772 : MEMMODEL_RELAXED
));
7775 if (!useless_type_conversion_p (type
, itype
))
7776 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7777 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7780 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7781 gsi_remove (&gsi
, true);
7783 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7784 gsi
= gsi_last_bb (load_bb
);
7785 gsi_remove (&gsi
, true);
7787 if (gimple_in_ssa_p (cfun
))
7788 update_ssa (TODO_update_ssa_no_phi
);
7793 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7794 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7795 size of the data type, and thus usable to find the index of the builtin
7796 decl. Returns false if the expression is not of the proper form. */
7799 expand_omp_atomic_fetch_op (basic_block load_bb
,
7800 tree addr
, tree loaded_val
,
7801 tree stored_val
, int index
)
7803 enum built_in_function oldbase
, newbase
, tmpbase
;
7804 tree decl
, itype
, call
;
7806 basic_block store_bb
= single_succ (load_bb
);
7807 gimple_stmt_iterator gsi
;
7810 enum tree_code code
;
7811 bool need_old
, need_new
;
7812 enum machine_mode imode
;
7815 /* We expect to find the following sequences:
7818 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7821 val = tmp OP something; (or: something OP tmp)
7822 GIMPLE_OMP_STORE (val)
7824 ???FIXME: Allow a more flexible sequence.
7825 Perhaps use data flow to pick the statements.
7829 gsi
= gsi_after_labels (store_bb
);
7830 stmt
= gsi_stmt (gsi
);
7831 loc
= gimple_location (stmt
);
7832 if (!is_gimple_assign (stmt
))
7835 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7837 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7838 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7839 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7840 gcc_checking_assert (!need_old
|| !need_new
);
7842 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7845 /* Check for one of the supported fetch-op operations. */
7846 code
= gimple_assign_rhs_code (stmt
);
7850 case POINTER_PLUS_EXPR
:
7851 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7852 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7855 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7856 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7859 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7860 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7863 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7864 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7867 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7868 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7874 /* Make sure the expression is of the proper form. */
7875 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7876 rhs
= gimple_assign_rhs2 (stmt
);
7877 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7878 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7879 rhs
= gimple_assign_rhs1 (stmt
);
7883 tmpbase
= ((enum built_in_function
)
7884 ((need_new
? newbase
: oldbase
) + index
+ 1));
7885 decl
= builtin_decl_explicit (tmpbase
);
7886 if (decl
== NULL_TREE
)
7888 itype
= TREE_TYPE (TREE_TYPE (decl
));
7889 imode
= TYPE_MODE (itype
);
7891 /* We could test all of the various optabs involved, but the fact of the
7892 matter is that (with the exception of i486 vs i586 and xadd) all targets
7893 that support any atomic operaton optab also implements compare-and-swap.
7894 Let optabs.c take care of expanding any compare-and-swap loop. */
7895 if (!can_compare_and_swap_p (imode
, true))
7898 gsi
= gsi_last_bb (load_bb
);
7899 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7901 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7902 It only requires that the operation happen atomically. Thus we can
7903 use the RELAXED memory model. */
7904 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7905 fold_convert_loc (loc
, itype
, rhs
),
7906 build_int_cst (NULL
,
7907 seq_cst
? MEMMODEL_SEQ_CST
7908 : MEMMODEL_RELAXED
));
7910 if (need_old
|| need_new
)
7912 lhs
= need_old
? loaded_val
: stored_val
;
7913 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7914 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7917 call
= fold_convert_loc (loc
, void_type_node
, call
);
7918 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7919 gsi_remove (&gsi
, true);
7921 gsi
= gsi_last_bb (store_bb
);
7922 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7923 gsi_remove (&gsi
, true);
7924 gsi
= gsi_last_bb (store_bb
);
7925 gsi_remove (&gsi
, true);
7927 if (gimple_in_ssa_p (cfun
))
7928 update_ssa (TODO_update_ssa_no_phi
);
7933 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7937 newval = rhs; // with oldval replacing *addr in rhs
7938 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7939 if (oldval != newval)
7942 INDEX is log2 of the size of the data type, and thus usable to find the
7943 index of the builtin decl. */
7946 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7947 tree addr
, tree loaded_val
, tree stored_val
,
7950 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7951 tree type
, itype
, cmpxchg
, iaddr
;
7952 gimple_stmt_iterator si
;
7953 basic_block loop_header
= single_succ (load_bb
);
7956 enum built_in_function fncode
;
7958 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7959 order to use the RELAXED memory model effectively. */
7960 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7962 cmpxchg
= builtin_decl_explicit (fncode
);
7963 if (cmpxchg
== NULL_TREE
)
7965 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7966 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7968 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7971 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7972 si
= gsi_last_bb (load_bb
);
7973 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7975 /* For floating-point values, we'll need to view-convert them to integers
7976 so that we can perform the atomic compare and swap. Simplify the
7977 following code by always setting up the "i"ntegral variables. */
7978 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7982 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7985 = force_gimple_operand_gsi (&si
,
7986 fold_convert (TREE_TYPE (iaddr
), addr
),
7987 false, NULL_TREE
, true, GSI_SAME_STMT
);
7988 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7989 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7990 loadedi
= create_tmp_var (itype
, NULL
);
7991 if (gimple_in_ssa_p (cfun
))
7992 loadedi
= make_ssa_name (loadedi
, NULL
);
7997 loadedi
= loaded_val
;
8000 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8001 tree loaddecl
= builtin_decl_explicit (fncode
);
8004 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8005 build_call_expr (loaddecl
, 2, iaddr
,
8006 build_int_cst (NULL_TREE
,
8007 MEMMODEL_RELAXED
)));
8009 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8010 build_int_cst (TREE_TYPE (iaddr
), 0));
8013 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8016 /* Move the value to the LOADEDI temporary. */
8017 if (gimple_in_ssa_p (cfun
))
8019 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8020 phi
= create_phi_node (loadedi
, loop_header
);
8021 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8025 gsi_insert_before (&si
,
8026 gimple_build_assign (loadedi
, initial
),
8028 if (loadedi
!= loaded_val
)
8030 gimple_stmt_iterator gsi2
;
8033 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8034 gsi2
= gsi_start_bb (loop_header
);
8035 if (gimple_in_ssa_p (cfun
))
8038 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8039 true, GSI_SAME_STMT
);
8040 stmt
= gimple_build_assign (loaded_val
, x
);
8041 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8045 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8046 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8047 true, GSI_SAME_STMT
);
8050 gsi_remove (&si
, true);
8052 si
= gsi_last_bb (store_bb
);
8053 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8056 storedi
= stored_val
;
8059 force_gimple_operand_gsi (&si
,
8060 build1 (VIEW_CONVERT_EXPR
, itype
,
8061 stored_val
), true, NULL_TREE
, true,
8064 /* Build the compare&swap statement. */
8065 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8066 new_storedi
= force_gimple_operand_gsi (&si
,
8067 fold_convert (TREE_TYPE (loadedi
),
8070 true, GSI_SAME_STMT
);
8072 if (gimple_in_ssa_p (cfun
))
8076 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
8077 stmt
= gimple_build_assign (old_vali
, loadedi
);
8078 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8080 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8081 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8084 /* Note that we always perform the comparison as an integer, even for
8085 floating point. This allows the atomic operation to properly
8086 succeed even with NaNs and -0.0. */
8087 stmt
= gimple_build_cond_empty
8088 (build2 (NE_EXPR
, boolean_type_node
,
8089 new_storedi
, old_vali
));
8090 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8093 e
= single_succ_edge (store_bb
);
8094 e
->flags
&= ~EDGE_FALLTHRU
;
8095 e
->flags
|= EDGE_FALSE_VALUE
;
8097 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8099 /* Copy the new value to loadedi (we already did that before the condition
8100 if we are not in SSA). */
8101 if (gimple_in_ssa_p (cfun
))
8103 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8104 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8107 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8108 gsi_remove (&si
, true);
8110 struct loop
*loop
= alloc_loop ();
8111 loop
->header
= loop_header
;
8112 loop
->latch
= store_bb
;
8113 add_loop (loop
, loop_header
->loop_father
);
8115 if (gimple_in_ssa_p (cfun
))
8116 update_ssa (TODO_update_ssa_no_phi
);
8121 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8123 GOMP_atomic_start ();
8127 The result is not globally atomic, but works so long as all parallel
8128 references are within #pragma omp atomic directives. According to
8129 responses received from omp@openmp.org, appears to be within spec.
8130 Which makes sense, since that's how several other compilers handle
8131 this situation as well.
8132 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8133 expanding. STORED_VAL is the operand of the matching
8134 GIMPLE_OMP_ATOMIC_STORE.
8137 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8141 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8146 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8147 tree addr
, tree loaded_val
, tree stored_val
)
8149 gimple_stmt_iterator si
;
8153 si
= gsi_last_bb (load_bb
);
8154 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8156 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8157 t
= build_call_expr (t
, 0);
8158 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8160 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8161 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8162 gsi_remove (&si
, true);
8164 si
= gsi_last_bb (store_bb
);
8165 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8167 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8169 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8171 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8172 t
= build_call_expr (t
, 0);
8173 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8174 gsi_remove (&si
, true);
8176 if (gimple_in_ssa_p (cfun
))
8177 update_ssa (TODO_update_ssa_no_phi
);
8181 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8182 using expand_omp_atomic_fetch_op. If it failed, we try to
8183 call expand_omp_atomic_pipeline, and if it fails too, the
8184 ultimate fallback is wrapping the operation in a mutex
8185 (expand_omp_atomic_mutex). REGION is the atomic region built
8186 by build_omp_regions_1(). */
8189 expand_omp_atomic (struct omp_region
*region
)
8191 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8192 gimple_omp_atomic_load load
=
8193 as_a
<gimple_omp_atomic_load
> (last_stmt (load_bb
));
8194 gimple_omp_atomic_store store
=
8195 as_a
<gimple_omp_atomic_store
> (last_stmt (store_bb
));
8196 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8197 tree addr
= gimple_omp_atomic_load_rhs (load
);
8198 tree stored_val
= gimple_omp_atomic_store_val (store
);
8199 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8200 HOST_WIDE_INT index
;
8202 /* Make sure the type is one of the supported sizes. */
8203 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8204 index
= exact_log2 (index
);
8205 if (index
>= 0 && index
<= 4)
8207 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8209 /* __sync builtins require strict data alignment. */
8210 if (exact_log2 (align
) >= index
)
8213 if (loaded_val
== stored_val
8214 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8215 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8216 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8217 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8221 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8222 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8223 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8224 && store_bb
== single_succ (load_bb
)
8225 && first_stmt (store_bb
) == store
8226 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8230 /* When possible, use specialized atomic update functions. */
8231 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8232 && store_bb
== single_succ (load_bb
)
8233 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8234 loaded_val
, stored_val
, index
))
8237 /* If we don't have specialized __sync builtins, try and implement
8238 as a compare and swap loop. */
8239 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8240 loaded_val
, stored_val
, index
))
8245 /* The ultimate fallback is wrapping the operation in a mutex. */
8246 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8250 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
8253 expand_omp_target (struct omp_region
*region
)
8255 basic_block entry_bb
, exit_bb
, new_bb
;
8256 struct function
*child_cfun
= NULL
;
8257 tree child_fn
= NULL_TREE
, block
, t
;
8258 gimple_stmt_iterator gsi
;
8259 gimple entry_stmt
, stmt
;
8262 entry_stmt
= last_stmt (region
->entry
);
8263 new_bb
= region
->entry
;
8264 int kind
= gimple_omp_target_kind (entry_stmt
);
8265 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8267 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8268 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8271 entry_bb
= region
->entry
;
8272 exit_bb
= region
->exit
;
8274 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8276 unsigned srcidx
, dstidx
, num
;
8278 /* If the target region needs data sent from the parent
8279 function, then the very first statement (except possible
8280 tree profile counter updates) of the parallel body
8281 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8282 &.OMP_DATA_O is passed as an argument to the child function,
8283 we need to replace it with the argument as seen by the child
8286 In most cases, this will end up being the identity assignment
8287 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
8288 a function call that has been inlined, the original PARM_DECL
8289 .OMP_DATA_I may have been converted into a different local
8290 variable. In which case, we need to keep the assignment. */
8291 if (gimple_omp_target_data_arg (entry_stmt
))
8293 basic_block entry_succ_bb
= single_succ (entry_bb
);
8294 gimple_stmt_iterator gsi
;
8296 gimple tgtcopy_stmt
= NULL
;
8298 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
8300 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8302 gcc_assert (!gsi_end_p (gsi
));
8303 stmt
= gsi_stmt (gsi
);
8304 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8307 if (gimple_num_ops (stmt
) == 2)
8309 tree arg
= gimple_assign_rhs1 (stmt
);
8311 /* We're ignoring the subcode because we're
8312 effectively doing a STRIP_NOPS. */
8314 if (TREE_CODE (arg
) == ADDR_EXPR
8315 && TREE_OPERAND (arg
, 0) == sender
)
8317 tgtcopy_stmt
= stmt
;
8323 gcc_assert (tgtcopy_stmt
!= NULL
);
8324 arg
= DECL_ARGUMENTS (child_fn
);
8326 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8327 gsi_remove (&gsi
, true);
8330 /* Declare local variables needed in CHILD_CFUN. */
8331 block
= DECL_INITIAL (child_fn
);
8332 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8333 /* The gimplifier could record temporaries in target block
8334 rather than in containing function's local_decls chain,
8335 which would mean cgraph missed finalizing them. Do it now. */
8336 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8337 if (TREE_CODE (t
) == VAR_DECL
8339 && !DECL_EXTERNAL (t
))
8340 varpool_node::finalize_decl (t
);
8341 DECL_SAVED_TREE (child_fn
) = NULL
;
8342 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8343 gimple_set_body (child_fn
, NULL
);
8344 TREE_USED (block
) = 1;
8346 /* Reset DECL_CONTEXT on function arguments. */
8347 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8348 DECL_CONTEXT (t
) = child_fn
;
8350 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
8351 so that it can be moved to the child function. */
8352 gsi
= gsi_last_bb (entry_bb
);
8353 stmt
= gsi_stmt (gsi
);
8354 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
8355 && gimple_omp_target_kind (stmt
)
8356 == GF_OMP_TARGET_KIND_REGION
);
8357 gsi_remove (&gsi
, true);
8358 e
= split_block (entry_bb
, stmt
);
8360 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8362 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8365 gsi
= gsi_last_bb (exit_bb
);
8366 gcc_assert (!gsi_end_p (gsi
)
8367 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8368 stmt
= gimple_build_return (NULL
);
8369 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8370 gsi_remove (&gsi
, true);
8373 /* Move the target region into CHILD_CFUN. */
8375 block
= gimple_block (entry_stmt
);
8377 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8379 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8380 /* When the OMP expansion process cannot guarantee an up-to-date
8381 loop tree arrange for the child function to fixup loops. */
8382 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8383 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8385 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8386 num
= vec_safe_length (child_cfun
->local_decls
);
8387 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8389 t
= (*child_cfun
->local_decls
)[srcidx
];
8390 if (DECL_CONTEXT (t
) == cfun
->decl
)
8392 if (srcidx
!= dstidx
)
8393 (*child_cfun
->local_decls
)[dstidx
] = t
;
8397 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8399 /* Inform the callgraph about the new function. */
8400 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8401 cgraph_node::add_new_function (child_fn
, true);
8403 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8404 fixed in a following pass. */
8405 push_cfun (child_cfun
);
8406 cgraph_edge::rebuild_edges ();
8408 /* Some EH regions might become dead, see PR34608. If
8409 pass_cleanup_cfg isn't the first pass to happen with the
8410 new child, these dead EH edges might cause problems.
8411 Clean them up now. */
8412 if (flag_exceptions
)
8415 bool changed
= false;
8417 FOR_EACH_BB_FN (bb
, cfun
)
8418 changed
|= gimple_purge_dead_eh_edges (bb
);
8420 cleanup_tree_cfg ();
8425 /* Emit a library call to launch the target region, or do data
8427 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8428 enum built_in_function start_ix
;
8429 location_t clause_loc
;
8431 clauses
= gimple_omp_target_clauses (entry_stmt
);
8433 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8434 start_ix
= BUILT_IN_GOMP_TARGET
;
8435 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8436 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8438 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8440 /* By default, the value of DEVICE is -1 (let runtime library choose)
8441 and there is no conditional. */
8443 device
= build_int_cst (integer_type_node
, -1);
8445 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8447 cond
= OMP_CLAUSE_IF_EXPR (c
);
8449 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8452 device
= OMP_CLAUSE_DEVICE_ID (c
);
8453 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8456 clause_loc
= gimple_location (entry_stmt
);
8458 /* Ensure 'device' is of the correct type. */
8459 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8461 /* If we found the clause 'if (cond)', build
8462 (cond ? device : -2). */
8465 cond
= gimple_boolify (cond
);
8467 basic_block cond_bb
, then_bb
, else_bb
;
8471 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8472 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8474 gsi
= gsi_last_bb (new_bb
);
8476 e
= split_block (new_bb
, gsi_stmt (gsi
));
8479 e
= split_block (new_bb
, NULL
);
8484 then_bb
= create_empty_bb (cond_bb
);
8485 else_bb
= create_empty_bb (then_bb
);
8486 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8487 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8489 stmt
= gimple_build_cond_empty (cond
);
8490 gsi
= gsi_last_bb (cond_bb
);
8491 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8493 gsi
= gsi_start_bb (then_bb
);
8494 stmt
= gimple_build_assign (tmp_var
, device
);
8495 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8497 gsi
= gsi_start_bb (else_bb
);
8498 stmt
= gimple_build_assign (tmp_var
,
8499 build_int_cst (integer_type_node
, -2));
8500 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8502 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8503 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8504 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8505 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8506 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8507 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8512 gsi
= gsi_last_bb (new_bb
);
8513 t
= gimple_omp_target_data_arg (entry_stmt
);
8516 t1
= size_zero_node
;
8517 t2
= build_zero_cst (ptr_type_node
);
8523 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8524 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8525 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8526 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8527 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8531 /* FIXME: This will be address of
8532 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8533 symbol, as soon as the linker plugin is able to create it for us. */
8534 tree openmp_target
= build_zero_cst (ptr_type_node
);
8535 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8537 tree fnaddr
= build_fold_addr_expr (child_fn
);
8538 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8539 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8542 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8543 device
, openmp_target
, t1
, t2
, t3
, t4
);
8544 gimple_set_location (g
, gimple_location (entry_stmt
));
8545 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8546 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8549 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8550 gsi_remove (&gsi
, true);
8552 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8554 gsi
= gsi_last_bb (region
->exit
);
8556 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8557 gsi_remove (&gsi
, true);
8562 /* Expand the parallel region tree rooted at REGION. Expansion
8563 proceeds in depth-first order. Innermost regions are expanded
8564 first. This way, parallel regions that require a new function to
8565 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8566 internal dependencies in their body. */
8569 expand_omp (struct omp_region
*region
)
8573 location_t saved_location
;
8574 gimple inner_stmt
= NULL
;
8576 /* First, determine whether this is a combined parallel+workshare
8578 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8579 determine_parallel_type (region
);
8581 if (region
->type
== GIMPLE_OMP_FOR
8582 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8583 inner_stmt
= last_stmt (region
->inner
->entry
);
8586 expand_omp (region
->inner
);
8588 saved_location
= input_location
;
8589 if (gimple_has_location (last_stmt (region
->entry
)))
8590 input_location
= gimple_location (last_stmt (region
->entry
));
8592 switch (region
->type
)
8594 case GIMPLE_OMP_PARALLEL
:
8595 case GIMPLE_OMP_TASK
:
8596 expand_omp_taskreg (region
);
8599 case GIMPLE_OMP_FOR
:
8600 expand_omp_for (region
, inner_stmt
);
8603 case GIMPLE_OMP_SECTIONS
:
8604 expand_omp_sections (region
);
8607 case GIMPLE_OMP_SECTION
:
8608 /* Individual omp sections are handled together with their
8609 parent GIMPLE_OMP_SECTIONS region. */
8612 case GIMPLE_OMP_SINGLE
:
8613 expand_omp_single (region
);
8616 case GIMPLE_OMP_MASTER
:
8617 case GIMPLE_OMP_TASKGROUP
:
8618 case GIMPLE_OMP_ORDERED
:
8619 case GIMPLE_OMP_CRITICAL
:
8620 case GIMPLE_OMP_TEAMS
:
8621 expand_omp_synch (region
);
8624 case GIMPLE_OMP_ATOMIC_LOAD
:
8625 expand_omp_atomic (region
);
8628 case GIMPLE_OMP_TARGET
:
8629 expand_omp_target (region
);
8636 input_location
= saved_location
;
8637 region
= region
->next
;
8642 /* Helper for build_omp_regions. Scan the dominator tree starting at
8643 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8644 true, the function ends once a single tree is built (otherwise, whole
8645 forest of OMP constructs may be built). */
8648 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8651 gimple_stmt_iterator gsi
;
8655 gsi
= gsi_last_bb (bb
);
8656 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8658 struct omp_region
*region
;
8659 enum gimple_code code
;
8661 stmt
= gsi_stmt (gsi
);
8662 code
= gimple_code (stmt
);
8663 if (code
== GIMPLE_OMP_RETURN
)
8665 /* STMT is the return point out of region PARENT. Mark it
8666 as the exit point and make PARENT the immediately
8667 enclosing region. */
8668 gcc_assert (parent
);
8671 parent
= parent
->outer
;
8673 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8675 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8676 GIMPLE_OMP_RETURN, but matches with
8677 GIMPLE_OMP_ATOMIC_LOAD. */
8678 gcc_assert (parent
);
8679 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8682 parent
= parent
->outer
;
8685 else if (code
== GIMPLE_OMP_CONTINUE
)
8687 gcc_assert (parent
);
8690 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8692 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8693 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8696 else if (code
== GIMPLE_OMP_TARGET
8697 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8698 new_omp_region (bb
, code
, parent
);
8701 /* Otherwise, this directive becomes the parent for a new
8703 region
= new_omp_region (bb
, code
, parent
);
8708 if (single_tree
&& !parent
)
8711 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8713 son
= next_dom_son (CDI_DOMINATORS
, son
))
8714 build_omp_regions_1 (son
, parent
, single_tree
);
8717 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8721 build_omp_regions_root (basic_block root
)
8723 gcc_assert (root_omp_region
== NULL
);
8724 build_omp_regions_1 (root
, NULL
, true);
8725 gcc_assert (root_omp_region
!= NULL
);
8728 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8731 omp_expand_local (basic_block head
)
8733 build_omp_regions_root (head
);
8734 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8736 fprintf (dump_file
, "\nOMP region tree\n\n");
8737 dump_omp_region (dump_file
, root_omp_region
, 0);
8738 fprintf (dump_file
, "\n");
8741 remove_exit_barriers (root_omp_region
);
8742 expand_omp (root_omp_region
);
8744 free_omp_regions ();
8747 /* Scan the CFG and build a tree of OMP regions. Return the root of
8748 the OMP region tree. */
8751 build_omp_regions (void)
8753 gcc_assert (root_omp_region
== NULL
);
8754 calculate_dominance_info (CDI_DOMINATORS
);
8755 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8758 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8761 execute_expand_omp (void)
8763 build_omp_regions ();
8765 if (!root_omp_region
)
8770 fprintf (dump_file
, "\nOMP region tree\n\n");
8771 dump_omp_region (dump_file
, root_omp_region
, 0);
8772 fprintf (dump_file
, "\n");
8775 remove_exit_barriers (root_omp_region
);
8777 expand_omp (root_omp_region
);
8779 cleanup_tree_cfg ();
8781 free_omp_regions ();
8786 /* OMP expansion -- the default pass, run before creation of SSA form. */
8790 const pass_data pass_data_expand_omp
=
8792 GIMPLE_PASS
, /* type */
8793 "ompexp", /* name */
8794 OPTGROUP_NONE
, /* optinfo_flags */
8795 TV_NONE
, /* tv_id */
8796 PROP_gimple_any
, /* properties_required */
8797 0, /* properties_provided */
8798 0, /* properties_destroyed */
8799 0, /* todo_flags_start */
8800 0, /* todo_flags_finish */
8803 class pass_expand_omp
: public gimple_opt_pass
8806 pass_expand_omp (gcc::context
*ctxt
)
8807 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8810 /* opt_pass methods: */
8811 virtual bool gate (function
*)
8813 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8814 || flag_cilkplus
!= 0) && !seen_error ());
8817 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8819 }; // class pass_expand_omp
8824 make_pass_expand_omp (gcc::context
*ctxt
)
8826 return new pass_expand_omp (ctxt
);
8829 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8831 /* If ctx is a worksharing context inside of a cancellable parallel
8832 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8833 and conditional branch to parallel's cancel_label to handle
8834 cancellation in the implicit barrier. */
8837 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8839 gimple omp_return
= gimple_seq_last_stmt (*body
);
8840 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8841 if (gimple_omp_return_nowait_p (omp_return
))
8844 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8845 && ctx
->outer
->cancellable
)
8847 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8848 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8849 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8850 gimple_omp_return_set_lhs (omp_return
, lhs
);
8851 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8852 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8853 fold_convert (c_bool_type
,
8854 boolean_false_node
),
8855 ctx
->outer
->cancel_label
, fallthru_label
);
8856 gimple_seq_add_stmt (body
, g
);
8857 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8861 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8862 CTX is the enclosing OMP context for the current statement. */
8865 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8867 tree block
, control
;
8868 gimple_stmt_iterator tgsi
;
8870 gimple_bind new_stmt
, bind
;
8871 gimple_seq ilist
, dlist
, olist
, new_body
;
8873 stmt
= gsi_stmt (*gsi_p
);
8875 push_gimplify_context ();
8879 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8880 &ilist
, &dlist
, ctx
, NULL
);
8882 new_body
= gimple_omp_body (stmt
);
8883 gimple_omp_set_body (stmt
, NULL
);
8884 tgsi
= gsi_start (new_body
);
8885 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8890 sec_start
= gsi_stmt (tgsi
);
8891 sctx
= maybe_lookup_ctx (sec_start
);
8894 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8895 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8896 GSI_CONTINUE_LINKING
);
8897 gimple_omp_set_body (sec_start
, NULL
);
8899 if (gsi_one_before_end_p (tgsi
))
8901 gimple_seq l
= NULL
;
8902 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8904 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8905 gimple_omp_section_set_last (sec_start
);
8908 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8909 GSI_CONTINUE_LINKING
);
8912 block
= make_node (BLOCK
);
8913 bind
= gimple_build_bind (NULL
, new_body
, block
);
8916 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8918 block
= make_node (BLOCK
);
8919 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8920 gsi_replace (gsi_p
, new_stmt
, true);
8922 pop_gimplify_context (new_stmt
);
8923 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8924 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8925 if (BLOCK_VARS (block
))
8926 TREE_USED (block
) = 1;
8929 gimple_seq_add_seq (&new_body
, ilist
);
8930 gimple_seq_add_stmt (&new_body
, stmt
);
8931 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8932 gimple_seq_add_stmt (&new_body
, bind
);
8934 control
= create_tmp_var (unsigned_type_node
, ".section");
8935 t
= gimple_build_omp_continue (control
, control
);
8936 gimple_omp_sections_set_control (stmt
, control
);
8937 gimple_seq_add_stmt (&new_body
, t
);
8939 gimple_seq_add_seq (&new_body
, olist
);
8940 if (ctx
->cancellable
)
8941 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8942 gimple_seq_add_seq (&new_body
, dlist
);
8944 new_body
= maybe_catch_exception (new_body
);
8946 t
= gimple_build_omp_return
8947 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8948 OMP_CLAUSE_NOWAIT
));
8949 gimple_seq_add_stmt (&new_body
, t
);
8950 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8952 gimple_bind_set_body (new_stmt
, new_body
);
8956 /* A subroutine of lower_omp_single. Expand the simple form of
8957 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8959 if (GOMP_single_start ())
8961 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8963 FIXME. It may be better to delay expanding the logic of this until
8964 pass_expand_omp. The expanded logic may make the job more difficult
8965 to a synchronization analysis pass. */
8968 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8970 location_t loc
= gimple_location (single_stmt
);
8971 tree tlabel
= create_artificial_label (loc
);
8972 tree flabel
= create_artificial_label (loc
);
8976 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8977 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8978 call
= gimple_build_call (decl
, 0);
8979 gimple_call_set_lhs (call
, lhs
);
8980 gimple_seq_add_stmt (pre_p
, call
);
8982 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8983 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8986 gimple_seq_add_stmt (pre_p
, cond
);
8987 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8988 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8989 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8993 /* A subroutine of lower_omp_single. Expand the simple form of
8994 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8996 #pragma omp single copyprivate (a, b, c)
8998 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
9001 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
9007 GOMP_single_copy_end (©out);
9018 FIXME. It may be better to delay expanding the logic of this until
9019 pass_expand_omp. The expanded logic may make the job more difficult
9020 to a synchronization analysis pass. */
9023 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
9025 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
9026 gimple_seq copyin_seq
;
9027 location_t loc
= gimple_location (single_stmt
);
9029 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
9031 ptr_type
= build_pointer_type (ctx
->record_type
);
9032 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
9034 l0
= create_artificial_label (loc
);
9035 l1
= create_artificial_label (loc
);
9036 l2
= create_artificial_label (loc
);
9038 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
9039 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
9040 t
= fold_convert_loc (loc
, ptr_type
, t
);
9041 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
9043 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
9044 build_int_cst (ptr_type
, 0));
9045 t
= build3 (COND_EXPR
, void_type_node
, t
,
9046 build_and_jump (&l0
), build_and_jump (&l1
));
9047 gimplify_and_add (t
, pre_p
);
9049 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
9051 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
9054 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
9057 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9058 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
9059 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
9060 gimplify_and_add (t
, pre_p
);
9062 t
= build_and_jump (&l2
);
9063 gimplify_and_add (t
, pre_p
);
9065 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
9067 gimple_seq_add_seq (pre_p
, copyin_seq
);
9069 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
9073 /* Expand code for an OpenMP single directive. */
9076 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9079 gimple t
, single_stmt
= gsi_stmt (*gsi_p
);
9081 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
9083 push_gimplify_context ();
9085 block
= make_node (BLOCK
);
9086 bind
= gimple_build_bind (NULL
, NULL
, block
);
9087 gsi_replace (gsi_p
, bind
, true);
9090 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
9091 &bind_body
, &dlist
, ctx
, NULL
);
9092 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
9094 gimple_seq_add_stmt (&bind_body
, single_stmt
);
9096 if (ctx
->record_type
)
9097 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
9099 lower_omp_single_simple (single_stmt
, &bind_body
);
9101 gimple_omp_set_body (single_stmt
, NULL
);
9103 gimple_seq_add_seq (&bind_body
, dlist
);
9105 bind_body
= maybe_catch_exception (bind_body
);
9107 t
= gimple_build_omp_return
9108 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
9109 OMP_CLAUSE_NOWAIT
));
9110 gimple_seq_add_stmt (&bind_body_tail
, t
);
9111 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
9112 if (ctx
->record_type
)
9114 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
9115 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9116 TREE_THIS_VOLATILE (clobber
) = 1;
9117 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
9118 clobber
), GSI_SAME_STMT
);
9120 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
9121 gimple_bind_set_body (bind
, bind_body
);
9123 pop_gimplify_context (bind
);
9125 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9126 BLOCK_VARS (block
) = ctx
->block_vars
;
9127 if (BLOCK_VARS (block
))
9128 TREE_USED (block
) = 1;
9132 /* Expand code for an OpenMP master directive. */
9135 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9137 tree block
, lab
= NULL
, x
, bfn_decl
;
9138 gimple stmt
= gsi_stmt (*gsi_p
);
9140 location_t loc
= gimple_location (stmt
);
9143 push_gimplify_context ();
9145 block
= make_node (BLOCK
);
9146 bind
= gimple_build_bind (NULL
, NULL
, block
);
9147 gsi_replace (gsi_p
, bind
, true);
9148 gimple_bind_add_stmt (bind
, stmt
);
9150 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9151 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
9152 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
9153 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
9155 gimplify_and_add (x
, &tseq
);
9156 gimple_bind_add_seq (bind
, tseq
);
9158 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9159 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9160 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9161 gimple_omp_set_body (stmt
, NULL
);
9163 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
9165 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9167 pop_gimplify_context (bind
);
9169 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9170 BLOCK_VARS (block
) = ctx
->block_vars
;
9174 /* Expand code for an OpenMP taskgroup directive. */
9177 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9179 gimple stmt
= gsi_stmt (*gsi_p
), x
;
9181 tree block
= make_node (BLOCK
);
9183 bind
= gimple_build_bind (NULL
, NULL
, block
);
9184 gsi_replace (gsi_p
, bind
, true);
9185 gimple_bind_add_stmt (bind
, stmt
);
9187 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
9189 gimple_bind_add_stmt (bind
, x
);
9191 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9192 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9193 gimple_omp_set_body (stmt
, NULL
);
9195 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9197 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9198 BLOCK_VARS (block
) = ctx
->block_vars
;
9202 /* Expand code for an OpenMP ordered directive. */
9205 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9208 gimple stmt
= gsi_stmt (*gsi_p
), x
;
9211 push_gimplify_context ();
9213 block
= make_node (BLOCK
);
9214 bind
= gimple_build_bind (NULL
, NULL
, block
);
9215 gsi_replace (gsi_p
, bind
, true);
9216 gimple_bind_add_stmt (bind
, stmt
);
9218 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
9220 gimple_bind_add_stmt (bind
, x
);
9222 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9223 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9224 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9225 gimple_omp_set_body (stmt
, NULL
);
9227 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
9228 gimple_bind_add_stmt (bind
, x
);
9230 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9232 pop_gimplify_context (bind
);
9234 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9235 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9239 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
9240 substitution of a couple of function calls. But in the NAMED case,
9241 requires that languages coordinate a symbol name. It is therefore
9242 best put here in common code. */
9244 static GTY((param1_is (tree
), param2_is (tree
)))
9245 splay_tree critical_name_mutexes
;
9248 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9251 tree name
, lock
, unlock
;
9252 gimple stmt
= gsi_stmt (*gsi_p
);
9254 location_t loc
= gimple_location (stmt
);
9257 name
= gimple_omp_critical_name (stmt
);
9263 if (!critical_name_mutexes
)
9264 critical_name_mutexes
9265 = splay_tree_new_ggc (splay_tree_compare_pointers
,
9266 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
9267 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
9269 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
9274 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
9276 new_str
= ACONCAT ((".gomp_critical_user_",
9277 IDENTIFIER_POINTER (name
), NULL
));
9278 DECL_NAME (decl
) = get_identifier (new_str
);
9279 TREE_PUBLIC (decl
) = 1;
9280 TREE_STATIC (decl
) = 1;
9281 DECL_COMMON (decl
) = 1;
9282 DECL_ARTIFICIAL (decl
) = 1;
9283 DECL_IGNORED_P (decl
) = 1;
9284 varpool_node::finalize_decl (decl
);
9286 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
9287 (splay_tree_value
) decl
);
9290 decl
= (tree
) n
->value
;
9292 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
9293 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
9295 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
9296 unlock
= build_call_expr_loc (loc
, unlock
, 1,
9297 build_fold_addr_expr_loc (loc
, decl
));
9301 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9302 lock
= build_call_expr_loc (loc
, lock
, 0);
9304 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9305 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9308 push_gimplify_context ();
9310 block
= make_node (BLOCK
);
9311 bind
= gimple_build_bind (NULL
, NULL
, block
);
9312 gsi_replace (gsi_p
, bind
, true);
9313 gimple_bind_add_stmt (bind
, stmt
);
9315 tbody
= gimple_bind_body (bind
);
9316 gimplify_and_add (lock
, &tbody
);
9317 gimple_bind_set_body (bind
, tbody
);
9319 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9320 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9321 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9322 gimple_omp_set_body (stmt
, NULL
);
9324 tbody
= gimple_bind_body (bind
);
9325 gimplify_and_add (unlock
, &tbody
);
9326 gimple_bind_set_body (bind
, tbody
);
9328 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9330 pop_gimplify_context (bind
);
9331 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9332 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9336 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9337 for a lastprivate clause. Given a loop control predicate of (V
9338 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9339 is appended to *DLIST, iterator initialization is appended to
9343 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9344 gimple_seq
*dlist
, struct omp_context
*ctx
)
9346 tree clauses
, cond
, vinit
;
9347 enum tree_code cond_code
;
9350 cond_code
= fd
->loop
.cond_code
;
9351 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9353 /* When possible, use a strict equality expression. This can let VRP
9354 type optimizations deduce the value and remove a copy. */
9355 if (tree_fits_shwi_p (fd
->loop
.step
))
9357 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9358 if (step
== 1 || step
== -1)
9359 cond_code
= EQ_EXPR
;
9362 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
9364 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9366 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
9367 if (!gimple_seq_empty_p (stmts
))
9369 gimple_seq_add_seq (&stmts
, *dlist
);
9372 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9373 vinit
= fd
->loop
.n1
;
9374 if (cond_code
== EQ_EXPR
9375 && tree_fits_shwi_p (fd
->loop
.n2
)
9376 && ! integer_zerop (fd
->loop
.n2
))
9377 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9379 vinit
= unshare_expr (vinit
);
9381 /* Initialize the iterator variable, so that threads that don't execute
9382 any iterations don't execute the lastprivate clauses by accident. */
9383 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9388 /* Lower code for an OpenMP loop directive. */
9391 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9394 struct omp_for_data fd
, *fdp
= NULL
;
9395 gimple stmt
= gsi_stmt (*gsi_p
);
9396 gimple_bind new_stmt
;
9397 gimple_seq omp_for_body
, body
, dlist
;
9400 push_gimplify_context ();
9402 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9404 block
= make_node (BLOCK
);
9405 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9406 /* Replace at gsi right away, so that 'stmt' is no member
9407 of a sequence anymore as we're going to add to to a different
9409 gsi_replace (gsi_p
, new_stmt
, true);
9411 /* Move declaration of temporaries in the loop body before we make
9413 omp_for_body
= gimple_omp_body (stmt
);
9414 if (!gimple_seq_empty_p (omp_for_body
)
9415 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9417 gimple_bind inner_bind
=
9418 as_a
<gimple_bind
> (gimple_seq_first_stmt (omp_for_body
));
9419 tree vars
= gimple_bind_vars (inner_bind
);
9420 gimple_bind_append_vars (new_stmt
, vars
);
9421 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9422 keep them on the inner_bind and it's block. */
9423 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9424 if (gimple_bind_block (inner_bind
))
9425 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9428 if (gimple_omp_for_combined_into_p (stmt
))
9430 extract_omp_for_data (stmt
, &fd
, NULL
);
9433 /* We need two temporaries with fd.loop.v type (istart/iend)
9434 and then (fd.collapse - 1) temporaries with the same
9435 type for count2 ... countN-1 vars if not constant. */
9437 tree type
= fd
.iter_type
;
9439 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9440 count
+= fd
.collapse
- 1;
9441 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9442 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9446 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9447 OMP_CLAUSE__LOOPTEMP_
);
9448 for (i
= 0; i
< count
; i
++)
9453 gcc_assert (outerc
);
9454 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9455 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9456 OMP_CLAUSE__LOOPTEMP_
);
9460 temp
= create_tmp_var (type
, NULL
);
9461 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9463 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9464 OMP_CLAUSE_DECL (*pc
) = temp
;
9465 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9470 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9473 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9475 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9477 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9479 /* Lower the header expressions. At this point, we can assume that
9480 the header is of the form:
9482 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9484 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9485 using the .omp_data_s mapping, if needed. */
9486 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9488 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9489 if (!is_gimple_min_invariant (*rhs_p
))
9490 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9492 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9493 if (!is_gimple_min_invariant (*rhs_p
))
9494 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9496 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9497 if (!is_gimple_min_invariant (*rhs_p
))
9498 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9501 /* Once lowered, extract the bounds and clauses. */
9502 extract_omp_for_data (stmt
, &fd
, NULL
);
9504 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9506 gimple_seq_add_stmt (&body
, stmt
);
9507 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9509 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9512 /* After the loop, add exit clauses. */
9513 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9515 if (ctx
->cancellable
)
9516 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9518 gimple_seq_add_seq (&body
, dlist
);
9520 body
= maybe_catch_exception (body
);
9522 /* Region exit marker goes at the end of the loop body. */
9523 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9524 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9525 pop_gimplify_context (new_stmt
);
9527 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9528 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9529 if (BLOCK_VARS (block
))
9530 TREE_USED (block
) = 1;
9532 gimple_bind_set_body (new_stmt
, body
);
9533 gimple_omp_set_body (stmt
, NULL
);
9534 gimple_omp_for_set_pre_body (stmt
, NULL
);
9537 /* Callback for walk_stmts. Check if the current statement only contains
9538 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9541 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9542 bool *handled_ops_p
,
9543 struct walk_stmt_info
*wi
)
9545 int *info
= (int *) wi
->info
;
9546 gimple stmt
= gsi_stmt (*gsi_p
);
9548 *handled_ops_p
= true;
9549 switch (gimple_code (stmt
))
9553 case GIMPLE_OMP_FOR
:
9554 case GIMPLE_OMP_SECTIONS
:
9555 *info
= *info
== 0 ? 1 : -1;
9564 struct omp_taskcopy_context
9566 /* This field must be at the beginning, as we do "inheritance": Some
9567 callback functions for tree-inline.c (e.g., omp_copy_decl)
9568 receive a copy_body_data pointer that is up-casted to an
9569 omp_context pointer. */
9575 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9577 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9579 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9580 return create_tmp_var (TREE_TYPE (var
), NULL
);
9586 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9588 tree name
, new_fields
= NULL
, type
, f
;
9590 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9591 name
= DECL_NAME (TYPE_NAME (orig_type
));
9592 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9593 TYPE_DECL
, name
, type
);
9594 TYPE_NAME (type
) = name
;
9596 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9598 tree new_f
= copy_node (f
);
9599 DECL_CONTEXT (new_f
) = type
;
9600 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9601 TREE_CHAIN (new_f
) = new_fields
;
9602 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9603 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9604 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9607 tcctx
->cb
.decl_map
->put (f
, new_f
);
9609 TYPE_FIELDS (type
) = nreverse (new_fields
);
9614 /* Create task copyfn. */
9617 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9619 struct function
*child_cfun
;
9620 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9621 tree record_type
, srecord_type
, bind
, list
;
9622 bool record_needs_remap
= false, srecord_needs_remap
= false;
9624 struct omp_taskcopy_context tcctx
;
9625 location_t loc
= gimple_location (task_stmt
);
9627 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9628 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9629 gcc_assert (child_cfun
->cfg
== NULL
);
9630 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9632 /* Reset DECL_CONTEXT on function arguments. */
9633 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9634 DECL_CONTEXT (t
) = child_fn
;
9636 /* Populate the function. */
9637 push_gimplify_context ();
9638 push_cfun (child_cfun
);
9640 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9641 TREE_SIDE_EFFECTS (bind
) = 1;
9643 DECL_SAVED_TREE (child_fn
) = bind
;
9644 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9646 /* Remap src and dst argument types if needed. */
9647 record_type
= ctx
->record_type
;
9648 srecord_type
= ctx
->srecord_type
;
9649 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9650 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9652 record_needs_remap
= true;
9655 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9656 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9658 srecord_needs_remap
= true;
9662 if (record_needs_remap
|| srecord_needs_remap
)
9664 memset (&tcctx
, '\0', sizeof (tcctx
));
9665 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9666 tcctx
.cb
.dst_fn
= child_fn
;
9667 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
9668 gcc_checking_assert (tcctx
.cb
.src_node
);
9669 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9670 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9671 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9672 tcctx
.cb
.eh_lp_nr
= 0;
9673 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9674 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
9677 if (record_needs_remap
)
9678 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9679 if (srecord_needs_remap
)
9680 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9683 tcctx
.cb
.decl_map
= NULL
;
9685 arg
= DECL_ARGUMENTS (child_fn
);
9686 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9687 sarg
= DECL_CHAIN (arg
);
9688 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9690 /* First pass: initialize temporaries used in record_type and srecord_type
9691 sizes and field offsets. */
9692 if (tcctx
.cb
.decl_map
)
9693 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9694 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9698 decl
= OMP_CLAUSE_DECL (c
);
9699 p
= tcctx
.cb
.decl_map
->get (decl
);
9702 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9703 sf
= (tree
) n
->value
;
9704 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9705 src
= build_simple_mem_ref_loc (loc
, sarg
);
9706 src
= omp_build_component_ref (src
, sf
);
9707 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9708 append_to_statement_list (t
, &list
);
9711 /* Second pass: copy shared var pointers and copy construct non-VLA
9712 firstprivate vars. */
9713 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9714 switch (OMP_CLAUSE_CODE (c
))
9716 case OMP_CLAUSE_SHARED
:
9717 decl
= OMP_CLAUSE_DECL (c
);
9718 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9721 f
= (tree
) n
->value
;
9722 if (tcctx
.cb
.decl_map
)
9723 f
= *tcctx
.cb
.decl_map
->get (f
);
9724 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9725 sf
= (tree
) n
->value
;
9726 if (tcctx
.cb
.decl_map
)
9727 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9728 src
= build_simple_mem_ref_loc (loc
, sarg
);
9729 src
= omp_build_component_ref (src
, sf
);
9730 dst
= build_simple_mem_ref_loc (loc
, arg
);
9731 dst
= omp_build_component_ref (dst
, f
);
9732 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9733 append_to_statement_list (t
, &list
);
9735 case OMP_CLAUSE_FIRSTPRIVATE
:
9736 decl
= OMP_CLAUSE_DECL (c
);
9737 if (is_variable_sized (decl
))
9739 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9742 f
= (tree
) n
->value
;
9743 if (tcctx
.cb
.decl_map
)
9744 f
= *tcctx
.cb
.decl_map
->get (f
);
9745 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9748 sf
= (tree
) n
->value
;
9749 if (tcctx
.cb
.decl_map
)
9750 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9751 src
= build_simple_mem_ref_loc (loc
, sarg
);
9752 src
= omp_build_component_ref (src
, sf
);
9753 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9754 src
= build_simple_mem_ref_loc (loc
, src
);
9758 dst
= build_simple_mem_ref_loc (loc
, arg
);
9759 dst
= omp_build_component_ref (dst
, f
);
9760 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9761 append_to_statement_list (t
, &list
);
9763 case OMP_CLAUSE_PRIVATE
:
9764 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9766 decl
= OMP_CLAUSE_DECL (c
);
9767 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9768 f
= (tree
) n
->value
;
9769 if (tcctx
.cb
.decl_map
)
9770 f
= *tcctx
.cb
.decl_map
->get (f
);
9771 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9774 sf
= (tree
) n
->value
;
9775 if (tcctx
.cb
.decl_map
)
9776 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9777 src
= build_simple_mem_ref_loc (loc
, sarg
);
9778 src
= omp_build_component_ref (src
, sf
);
9779 if (use_pointer_for_field (decl
, NULL
))
9780 src
= build_simple_mem_ref_loc (loc
, src
);
9784 dst
= build_simple_mem_ref_loc (loc
, arg
);
9785 dst
= omp_build_component_ref (dst
, f
);
9786 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9787 append_to_statement_list (t
, &list
);
9793 /* Last pass: handle VLA firstprivates. */
9794 if (tcctx
.cb
.decl_map
)
9795 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9796 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9800 decl
= OMP_CLAUSE_DECL (c
);
9801 if (!is_variable_sized (decl
))
9803 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9806 f
= (tree
) n
->value
;
9807 f
= *tcctx
.cb
.decl_map
->get (f
);
9808 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9809 ind
= DECL_VALUE_EXPR (decl
);
9810 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9811 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9812 n
= splay_tree_lookup (ctx
->sfield_map
,
9813 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9814 sf
= (tree
) n
->value
;
9815 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9816 src
= build_simple_mem_ref_loc (loc
, sarg
);
9817 src
= omp_build_component_ref (src
, sf
);
9818 src
= build_simple_mem_ref_loc (loc
, src
);
9819 dst
= build_simple_mem_ref_loc (loc
, arg
);
9820 dst
= omp_build_component_ref (dst
, f
);
9821 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9822 append_to_statement_list (t
, &list
);
9823 n
= splay_tree_lookup (ctx
->field_map
,
9824 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9825 df
= (tree
) n
->value
;
9826 df
= *tcctx
.cb
.decl_map
->get (df
);
9827 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9828 ptr
= omp_build_component_ref (ptr
, df
);
9829 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9830 build_fold_addr_expr_loc (loc
, dst
));
9831 append_to_statement_list (t
, &list
);
9834 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9835 append_to_statement_list (t
, &list
);
9837 if (tcctx
.cb
.decl_map
)
9838 delete tcctx
.cb
.decl_map
;
9839 pop_gimplify_context (NULL
);
9840 BIND_EXPR_BODY (bind
) = list
;
9845 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9849 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9851 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9853 gcc_assert (clauses
);
9854 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9855 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9856 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9858 case OMP_CLAUSE_DEPEND_IN
:
9861 case OMP_CLAUSE_DEPEND_OUT
:
9862 case OMP_CLAUSE_DEPEND_INOUT
:
9868 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9869 tree array
= create_tmp_var (type
, NULL
);
9870 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9872 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9873 gimple_seq_add_stmt (iseq
, g
);
9874 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9876 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9877 gimple_seq_add_stmt (iseq
, g
);
9878 for (i
= 0; i
< 2; i
++)
9880 if ((i
? n_in
: n_out
) == 0)
9882 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9883 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9884 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9886 tree t
= OMP_CLAUSE_DECL (c
);
9887 t
= fold_convert (ptr_type_node
, t
);
9888 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9889 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9890 NULL_TREE
, NULL_TREE
);
9891 g
= gimple_build_assign (r
, t
);
9892 gimple_seq_add_stmt (iseq
, g
);
9895 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9896 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9897 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9898 OMP_CLAUSE_CHAIN (c
) = *p
;
9900 tree clobber
= build_constructor (type
, NULL
);
9901 TREE_THIS_VOLATILE (clobber
) = 1;
9902 g
= gimple_build_assign (array
, clobber
);
9903 gimple_seq_add_stmt (oseq
, g
);
9906 /* Lower the OpenMP parallel or task directive in the current statement
9907 in GSI_P. CTX holds context information for the directive. */
9910 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9914 gimple stmt
= gsi_stmt (*gsi_p
);
9915 gimple_bind par_bind
, bind
, dep_bind
= NULL
;
9916 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9917 location_t loc
= gimple_location (stmt
);
9919 clauses
= gimple_omp_taskreg_clauses (stmt
);
9921 as_a
<gimple_bind
> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
9922 par_body
= gimple_bind_body (par_bind
);
9923 child_fn
= ctx
->cb
.dst_fn
;
9924 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9925 && !gimple_omp_parallel_combined_p (stmt
))
9927 struct walk_stmt_info wi
;
9930 memset (&wi
, 0, sizeof (wi
));
9933 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9935 gimple_omp_parallel_set_combined_p (stmt
, true);
9937 gimple_seq dep_ilist
= NULL
;
9938 gimple_seq dep_olist
= NULL
;
9939 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9940 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9942 push_gimplify_context ();
9943 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9944 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9947 if (ctx
->srecord_type
)
9948 create_task_copyfn (stmt
, ctx
);
9950 push_gimplify_context ();
9955 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9956 lower_omp (&par_body
, ctx
);
9957 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9958 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9960 /* Declare all the variables created by mapping and the variables
9961 declared in the scope of the parallel body. */
9962 record_vars_into (ctx
->block_vars
, child_fn
);
9963 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9965 if (ctx
->record_type
)
9968 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9969 : ctx
->record_type
, ".omp_data_o");
9970 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9971 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9972 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9977 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9978 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9980 if (ctx
->record_type
)
9982 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9983 TREE_THIS_VOLATILE (clobber
) = 1;
9984 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9988 /* Once all the expansions are done, sequence all the different
9989 fragments inside gimple_omp_body. */
9993 if (ctx
->record_type
)
9995 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9996 /* fixup_child_record_type might have changed receiver_decl's type. */
9997 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9998 gimple_seq_add_stmt (&new_body
,
9999 gimple_build_assign (ctx
->receiver_decl
, t
));
10002 gimple_seq_add_seq (&new_body
, par_ilist
);
10003 gimple_seq_add_seq (&new_body
, par_body
);
10004 gimple_seq_add_seq (&new_body
, par_rlist
);
10005 if (ctx
->cancellable
)
10006 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10007 gimple_seq_add_seq (&new_body
, par_olist
);
10008 new_body
= maybe_catch_exception (new_body
);
10009 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10010 gimple_omp_set_body (stmt
, new_body
);
10012 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
10013 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
10014 gimple_bind_add_seq (bind
, ilist
);
10015 gimple_bind_add_stmt (bind
, stmt
);
10016 gimple_bind_add_seq (bind
, olist
);
10018 pop_gimplify_context (NULL
);
10022 gimple_bind_add_seq (dep_bind
, dep_ilist
);
10023 gimple_bind_add_stmt (dep_bind
, bind
);
10024 gimple_bind_add_seq (dep_bind
, dep_olist
);
10025 pop_gimplify_context (dep_bind
);
10029 /* Lower the OpenMP target directive in the current statement
10030 in GSI_P. CTX holds context information for the directive. */
10033 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10036 tree child_fn
, t
, c
;
10037 gimple stmt
= gsi_stmt (*gsi_p
);
10038 gimple_bind tgt_bind
= NULL
, bind
;
10039 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
10040 location_t loc
= gimple_location (stmt
);
10041 int kind
= gimple_omp_target_kind (stmt
);
10042 unsigned int map_cnt
= 0;
10044 clauses
= gimple_omp_target_clauses (stmt
);
10045 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10047 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
10048 tgt_body
= gimple_bind_body (tgt_bind
);
10050 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10051 tgt_body
= gimple_omp_body (stmt
);
10052 child_fn
= ctx
->cb
.dst_fn
;
10054 push_gimplify_context ();
10056 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10057 switch (OMP_CLAUSE_CODE (c
))
10063 case OMP_CLAUSE_MAP
:
10064 case OMP_CLAUSE_TO
:
10065 case OMP_CLAUSE_FROM
:
10066 var
= OMP_CLAUSE_DECL (c
);
10069 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
10070 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10075 if (DECL_SIZE (var
)
10076 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
10078 tree var2
= DECL_VALUE_EXPR (var
);
10079 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
10080 var2
= TREE_OPERAND (var2
, 0);
10081 gcc_assert (DECL_P (var2
));
10085 if (!maybe_lookup_field (var
, ctx
))
10088 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10090 x
= build_receiver_ref (var
, true, ctx
);
10091 tree new_var
= lookup_decl (var
, ctx
);
10092 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10093 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10094 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10095 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
10096 x
= build_simple_mem_ref (x
);
10097 SET_DECL_VALUE_EXPR (new_var
, x
);
10098 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
10103 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10105 target_nesting_level
++;
10106 lower_omp (&tgt_body
, ctx
);
10107 target_nesting_level
--;
10109 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10110 lower_omp (&tgt_body
, ctx
);
10112 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10114 /* Declare all the variables created by mapping and the variables
10115 declared in the scope of the target body. */
10116 record_vars_into (ctx
->block_vars
, child_fn
);
10117 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
10122 if (ctx
->record_type
)
10125 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
10126 DECL_NAMELESS (ctx
->sender_decl
) = 1;
10127 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
10128 t
= make_tree_vec (3);
10129 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
10130 TREE_VEC_ELT (t
, 1)
10131 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
10132 ".omp_data_sizes");
10133 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
10134 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
10135 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
10136 TREE_VEC_ELT (t
, 2)
10137 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
10139 ".omp_data_kinds");
10140 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
10141 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
10142 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
10143 gimple_omp_target_set_data_arg (stmt
, t
);
10145 vec
<constructor_elt
, va_gc
> *vsize
;
10146 vec
<constructor_elt
, va_gc
> *vkind
;
10147 vec_alloc (vsize
, map_cnt
);
10148 vec_alloc (vkind
, map_cnt
);
10149 unsigned int map_idx
= 0;
10151 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10152 switch (OMP_CLAUSE_CODE (c
))
10158 case OMP_CLAUSE_MAP
:
10159 case OMP_CLAUSE_TO
:
10160 case OMP_CLAUSE_FROM
:
10162 ovar
= OMP_CLAUSE_DECL (c
);
10163 if (!DECL_P (ovar
))
10165 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10166 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10168 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
10169 == get_base_address (ovar
));
10170 nc
= OMP_CLAUSE_CHAIN (c
);
10171 ovar
= OMP_CLAUSE_DECL (nc
);
10175 tree x
= build_sender_ref (ovar
, ctx
);
10177 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
10178 gimplify_assign (x
, v
, &ilist
);
10184 if (DECL_SIZE (ovar
)
10185 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
10187 tree ovar2
= DECL_VALUE_EXPR (ovar
);
10188 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
10189 ovar2
= TREE_OPERAND (ovar2
, 0);
10190 gcc_assert (DECL_P (ovar2
));
10193 if (!maybe_lookup_field (ovar
, ctx
))
10197 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
10198 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
10199 talign
= DECL_ALIGN_UNIT (ovar
);
10202 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
10203 tree x
= build_sender_ref (ovar
, ctx
);
10204 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10205 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10206 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10207 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
10209 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10211 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
10212 mark_addressable (avar
);
10213 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
10214 talign
= DECL_ALIGN_UNIT (avar
);
10215 avar
= build_fold_addr_expr (avar
);
10216 gimplify_assign (x
, avar
, &ilist
);
10218 else if (is_gimple_reg (var
))
10220 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10221 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
10222 mark_addressable (avar
);
10223 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
10224 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
10225 gimplify_assign (avar
, var
, &ilist
);
10226 avar
= build_fold_addr_expr (avar
);
10227 gimplify_assign (x
, avar
, &ilist
);
10228 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
10229 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
10230 && !TYPE_READONLY (TREE_TYPE (var
)))
10232 x
= build_sender_ref (ovar
, ctx
);
10233 x
= build_simple_mem_ref (x
);
10234 gimplify_assign (var
, x
, &olist
);
10239 var
= build_fold_addr_expr (var
);
10240 gimplify_assign (x
, var
, &ilist
);
10243 tree s
= OMP_CLAUSE_SIZE (c
);
10244 if (s
== NULL_TREE
)
10245 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
10246 s
= fold_convert (size_type_node
, s
);
10247 tree purpose
= size_int (map_idx
++);
10248 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
10249 if (TREE_CODE (s
) != INTEGER_CST
)
10250 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
10252 unsigned char tkind
= 0;
10253 switch (OMP_CLAUSE_CODE (c
))
10255 case OMP_CLAUSE_MAP
:
10256 tkind
= OMP_CLAUSE_MAP_KIND (c
);
10258 case OMP_CLAUSE_TO
:
10259 tkind
= OMP_CLAUSE_MAP_TO
;
10261 case OMP_CLAUSE_FROM
:
10262 tkind
= OMP_CLAUSE_MAP_FROM
;
10265 gcc_unreachable ();
10267 talign
= ceil_log2 (talign
);
10268 tkind
|= talign
<< 3;
10269 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
10270 build_int_cst (unsigned_char_type_node
,
10276 gcc_assert (map_idx
== map_cnt
);
10278 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
10279 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
10280 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
10281 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
10282 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
10284 gimple_seq initlist
= NULL
;
10285 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
10286 TREE_VEC_ELT (t
, 1)),
10287 &initlist
, true, NULL_TREE
);
10288 gimple_seq_add_seq (&ilist
, initlist
);
10290 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
10292 TREE_THIS_VOLATILE (clobber
) = 1;
10293 gimple_seq_add_stmt (&olist
,
10294 gimple_build_assign (TREE_VEC_ELT (t
, 1),
10298 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10299 TREE_THIS_VOLATILE (clobber
) = 1;
10300 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
10304 /* Once all the expansions are done, sequence all the different
10305 fragments inside gimple_omp_body. */
10309 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
10311 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10312 /* fixup_child_record_type might have changed receiver_decl's type. */
10313 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10314 gimple_seq_add_stmt (&new_body
,
10315 gimple_build_assign (ctx
->receiver_decl
, t
));
10318 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10320 gimple_seq_add_seq (&new_body
, tgt_body
);
10321 new_body
= maybe_catch_exception (new_body
);
10323 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10324 new_body
= tgt_body
;
10325 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
10327 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10328 gimple_omp_set_body (stmt
, new_body
);
10331 bind
= gimple_build_bind (NULL
, NULL
,
10332 tgt_bind
? gimple_bind_block (tgt_bind
)
10334 gsi_replace (gsi_p
, bind
, true);
10335 gimple_bind_add_seq (bind
, ilist
);
10336 gimple_bind_add_stmt (bind
, stmt
);
10337 gimple_bind_add_seq (bind
, olist
);
10339 pop_gimplify_context (NULL
);
10342 /* Expand code for an OpenMP teams directive. */
10345 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10347 gimple teams_stmt
= gsi_stmt (*gsi_p
);
10348 push_gimplify_context ();
10350 tree block
= make_node (BLOCK
);
10351 gimple_bind bind
= gimple_build_bind (NULL
, NULL
, block
);
10352 gsi_replace (gsi_p
, bind
, true);
10353 gimple_seq bind_body
= NULL
;
10354 gimple_seq dlist
= NULL
;
10355 gimple_seq olist
= NULL
;
10357 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10358 OMP_CLAUSE_NUM_TEAMS
);
10359 if (num_teams
== NULL_TREE
)
10360 num_teams
= build_int_cst (unsigned_type_node
, 0);
10363 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
10364 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
10365 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
10367 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10368 OMP_CLAUSE_THREAD_LIMIT
);
10369 if (thread_limit
== NULL_TREE
)
10370 thread_limit
= build_int_cst (unsigned_type_node
, 0);
10373 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
10374 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
10375 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
10379 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
10380 &bind_body
, &dlist
, ctx
, NULL
);
10381 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
10382 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
10383 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
10385 location_t loc
= gimple_location (teams_stmt
);
10386 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
10387 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
10388 gimple_set_location (call
, loc
);
10389 gimple_seq_add_stmt (&bind_body
, call
);
10391 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10392 gimple_omp_set_body (teams_stmt
, NULL
);
10393 gimple_seq_add_seq (&bind_body
, olist
);
10394 gimple_seq_add_seq (&bind_body
, dlist
);
10395 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10396 gimple_bind_set_body (bind
, bind_body
);
10398 pop_gimplify_context (bind
);
10400 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10401 BLOCK_VARS (block
) = ctx
->block_vars
;
10402 if (BLOCK_VARS (block
))
10403 TREE_USED (block
) = 1;
10407 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10408 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10409 of OpenMP context, but with task_shared_vars set. */
10412 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10417 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10418 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10421 if (task_shared_vars
10423 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10426 /* If a global variable has been privatized, TREE_CONSTANT on
10427 ADDR_EXPR might be wrong. */
10428 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10429 recompute_tree_invariant_for_addr_expr (t
);
10431 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10436 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10438 gimple stmt
= gsi_stmt (*gsi_p
);
10439 struct walk_stmt_info wi
;
10441 if (gimple_has_location (stmt
))
10442 input_location
= gimple_location (stmt
);
10444 if (task_shared_vars
)
10445 memset (&wi
, '\0', sizeof (wi
));
10447 /* If we have issued syntax errors, avoid doing any heavy lifting.
10448 Just replace the OpenMP directives with a NOP to avoid
10449 confusing RTL expansion. */
10450 if (seen_error () && is_gimple_omp (stmt
))
10452 gsi_replace (gsi_p
, gimple_build_nop (), true);
10456 switch (gimple_code (stmt
))
10459 if ((ctx
|| task_shared_vars
)
10460 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10461 ctx
? NULL
: &wi
, NULL
)
10462 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10463 ctx
? NULL
: &wi
, NULL
)))
10464 gimple_regimplify_operands (stmt
, gsi_p
);
10467 lower_omp (gimple_catch_handler_ptr (as_a
<gimple_catch
> (stmt
)), ctx
);
10469 case GIMPLE_EH_FILTER
:
10470 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10473 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10474 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10476 case GIMPLE_TRANSACTION
:
10477 lower_omp (gimple_transaction_body_ptr (
10478 as_a
<gimple_transaction
> (stmt
)),
10482 lower_omp (gimple_bind_body_ptr (as_a
<gimple_bind
> (stmt
)), ctx
);
10484 case GIMPLE_OMP_PARALLEL
:
10485 case GIMPLE_OMP_TASK
:
10486 ctx
= maybe_lookup_ctx (stmt
);
10488 if (ctx
->cancellable
)
10489 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10490 lower_omp_taskreg (gsi_p
, ctx
);
10492 case GIMPLE_OMP_FOR
:
10493 ctx
= maybe_lookup_ctx (stmt
);
10495 if (ctx
->cancellable
)
10496 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10497 lower_omp_for (gsi_p
, ctx
);
10499 case GIMPLE_OMP_SECTIONS
:
10500 ctx
= maybe_lookup_ctx (stmt
);
10502 if (ctx
->cancellable
)
10503 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10504 lower_omp_sections (gsi_p
, ctx
);
10506 case GIMPLE_OMP_SINGLE
:
10507 ctx
= maybe_lookup_ctx (stmt
);
10509 lower_omp_single (gsi_p
, ctx
);
10511 case GIMPLE_OMP_MASTER
:
10512 ctx
= maybe_lookup_ctx (stmt
);
10514 lower_omp_master (gsi_p
, ctx
);
10516 case GIMPLE_OMP_TASKGROUP
:
10517 ctx
= maybe_lookup_ctx (stmt
);
10519 lower_omp_taskgroup (gsi_p
, ctx
);
10521 case GIMPLE_OMP_ORDERED
:
10522 ctx
= maybe_lookup_ctx (stmt
);
10524 lower_omp_ordered (gsi_p
, ctx
);
10526 case GIMPLE_OMP_CRITICAL
:
10527 ctx
= maybe_lookup_ctx (stmt
);
10529 lower_omp_critical (gsi_p
, ctx
);
10531 case GIMPLE_OMP_ATOMIC_LOAD
:
10532 if ((ctx
|| task_shared_vars
)
10533 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
10534 as_a
<gimple_omp_atomic_load
> (stmt
)),
10535 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10536 gimple_regimplify_operands (stmt
, gsi_p
);
10538 case GIMPLE_OMP_TARGET
:
10539 ctx
= maybe_lookup_ctx (stmt
);
10541 lower_omp_target (gsi_p
, ctx
);
10543 case GIMPLE_OMP_TEAMS
:
10544 ctx
= maybe_lookup_ctx (stmt
);
10546 lower_omp_teams (gsi_p
, ctx
);
10550 fndecl
= gimple_call_fndecl (stmt
);
10552 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10553 switch (DECL_FUNCTION_CODE (fndecl
))
10555 case BUILT_IN_GOMP_BARRIER
:
10559 case BUILT_IN_GOMP_CANCEL
:
10560 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10563 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10564 cctx
= cctx
->outer
;
10565 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10566 if (!cctx
->cancellable
)
10568 if (DECL_FUNCTION_CODE (fndecl
)
10569 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10571 stmt
= gimple_build_nop ();
10572 gsi_replace (gsi_p
, stmt
, false);
10576 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10578 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10579 gimple_call_set_fndecl (stmt
, fndecl
);
10580 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10583 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10584 gimple_call_set_lhs (stmt
, lhs
);
10585 tree fallthru_label
;
10586 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10588 g
= gimple_build_label (fallthru_label
);
10589 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10590 g
= gimple_build_cond (NE_EXPR
, lhs
,
10591 fold_convert (TREE_TYPE (lhs
),
10592 boolean_false_node
),
10593 cctx
->cancel_label
, fallthru_label
);
10594 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10601 if ((ctx
|| task_shared_vars
)
10602 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10605 /* Just remove clobbers, this should happen only if we have
10606 "privatized" local addressable variables in SIMD regions,
10607 the clobber isn't needed in that case and gimplifying address
10608 of the ARRAY_REF into a pointer and creating MEM_REF based
10609 clobber would create worse code than we get with the clobber
10611 if (gimple_clobber_p (stmt
))
10613 gsi_replace (gsi_p
, gimple_build_nop (), true);
10616 gimple_regimplify_operands (stmt
, gsi_p
);
10623 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10625 location_t saved_location
= input_location
;
10626 gimple_stmt_iterator gsi
;
10627 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10628 lower_omp_1 (&gsi
, ctx
);
10629 /* During gimplification, we have not always invoked fold_stmt
10630 (gimplify.c:maybe_fold_stmt); call it now. */
10631 if (target_nesting_level
)
10632 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10634 input_location
= saved_location
;
10637 /* Main entry point. */
10639 static unsigned int
10640 execute_lower_omp (void)
10646 /* This pass always runs, to provide PROP_gimple_lomp.
10647 But there is nothing to do unless -fopenmp is given. */
10648 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10651 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10652 delete_omp_context
);
10654 body
= gimple_body (current_function_decl
);
10655 scan_omp (&body
, NULL
);
10656 gcc_assert (taskreg_nesting_level
== 0);
10657 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
10658 finish_taskreg_scan (ctx
);
10659 taskreg_contexts
.release ();
10661 if (all_contexts
->root
)
10663 if (task_shared_vars
)
10664 push_gimplify_context ();
10665 lower_omp (&body
, NULL
);
10666 if (task_shared_vars
)
10667 pop_gimplify_context (NULL
);
10672 splay_tree_delete (all_contexts
);
10673 all_contexts
= NULL
;
10675 BITMAP_FREE (task_shared_vars
);
10681 const pass_data pass_data_lower_omp
=
10683 GIMPLE_PASS
, /* type */
10684 "omplower", /* name */
10685 OPTGROUP_NONE
, /* optinfo_flags */
10686 TV_NONE
, /* tv_id */
10687 PROP_gimple_any
, /* properties_required */
10688 PROP_gimple_lomp
, /* properties_provided */
10689 0, /* properties_destroyed */
10690 0, /* todo_flags_start */
10691 0, /* todo_flags_finish */
10694 class pass_lower_omp
: public gimple_opt_pass
10697 pass_lower_omp (gcc::context
*ctxt
)
10698 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10701 /* opt_pass methods: */
10702 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10704 }; // class pass_lower_omp
10706 } // anon namespace
10709 make_pass_lower_omp (gcc::context
*ctxt
)
10711 return new pass_lower_omp (ctxt
);
10714 /* The following is a utility to diagnose OpenMP structured block violations.
10715 It is not part of the "omplower" pass, as that's invoked too late. It
10716 should be invoked by the respective front ends after gimplification. */
10718 static splay_tree all_labels
;
10720 /* Check for mismatched contexts and generate an error if needed. Return
10721 true if an error is detected. */
10724 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10725 gimple branch_ctx
, gimple label_ctx
)
10727 if (label_ctx
== branch_ctx
)
10732 Previously we kept track of the label's entire context in diagnose_sb_[12]
10733 so we could traverse it and issue a correct "exit" or "enter" error
10734 message upon a structured block violation.
10736 We built the context by building a list with tree_cons'ing, but there is
10737 no easy counterpart in gimple tuples. It seems like far too much work
10738 for issuing exit/enter error messages. If someone really misses the
10739 distinct error message... patches welcome.
10743 /* Try to avoid confusing the user by producing and error message
10744 with correct "exit" or "enter" verbiage. We prefer "exit"
10745 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10746 if (branch_ctx
== NULL
)
10752 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10757 label_ctx
= TREE_CHAIN (label_ctx
);
10762 error ("invalid exit from OpenMP structured block");
10764 error ("invalid entry to OpenMP structured block");
10767 bool cilkplus_block
= false;
10771 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10772 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10774 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10775 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10776 cilkplus_block
= true;
10779 /* If it's obvious we have an invalid entry, be specific about the error. */
10780 if (branch_ctx
== NULL
)
10782 if (cilkplus_block
)
10783 error ("invalid entry to Cilk Plus structured block");
10785 error ("invalid entry to OpenMP structured block");
10789 /* Otherwise, be vague and lazy, but efficient. */
10790 if (cilkplus_block
)
10791 error ("invalid branch to/from a Cilk Plus structured block");
10793 error ("invalid branch to/from an OpenMP structured block");
10796 gsi_replace (gsi_p
, gimple_build_nop (), false);
10800 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10801 where each label is found. */
10804 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10805 struct walk_stmt_info
*wi
)
10807 gimple context
= (gimple
) wi
->info
;
10808 gimple inner_context
;
10809 gimple stmt
= gsi_stmt (*gsi_p
);
10811 *handled_ops_p
= true;
10813 switch (gimple_code (stmt
))
10817 case GIMPLE_OMP_PARALLEL
:
10818 case GIMPLE_OMP_TASK
:
10819 case GIMPLE_OMP_SECTIONS
:
10820 case GIMPLE_OMP_SINGLE
:
10821 case GIMPLE_OMP_SECTION
:
10822 case GIMPLE_OMP_MASTER
:
10823 case GIMPLE_OMP_ORDERED
:
10824 case GIMPLE_OMP_CRITICAL
:
10825 case GIMPLE_OMP_TARGET
:
10826 case GIMPLE_OMP_TEAMS
:
10827 case GIMPLE_OMP_TASKGROUP
:
10828 /* The minimal context here is just the current OMP construct. */
10829 inner_context
= stmt
;
10830 wi
->info
= inner_context
;
10831 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10832 wi
->info
= context
;
10835 case GIMPLE_OMP_FOR
:
10836 inner_context
= stmt
;
10837 wi
->info
= inner_context
;
10838 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10840 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10841 diagnose_sb_1
, NULL
, wi
);
10842 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10843 wi
->info
= context
;
10847 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10848 (splay_tree_value
) context
);
10858 /* Pass 2: Check each branch and see if its context differs from that of
10859 the destination label's context. */
10862 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10863 struct walk_stmt_info
*wi
)
10865 gimple context
= (gimple
) wi
->info
;
10867 gimple stmt
= gsi_stmt (*gsi_p
);
10869 *handled_ops_p
= true;
10871 switch (gimple_code (stmt
))
10875 case GIMPLE_OMP_PARALLEL
:
10876 case GIMPLE_OMP_TASK
:
10877 case GIMPLE_OMP_SECTIONS
:
10878 case GIMPLE_OMP_SINGLE
:
10879 case GIMPLE_OMP_SECTION
:
10880 case GIMPLE_OMP_MASTER
:
10881 case GIMPLE_OMP_ORDERED
:
10882 case GIMPLE_OMP_CRITICAL
:
10883 case GIMPLE_OMP_TARGET
:
10884 case GIMPLE_OMP_TEAMS
:
10885 case GIMPLE_OMP_TASKGROUP
:
10887 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10888 wi
->info
= context
;
10891 case GIMPLE_OMP_FOR
:
10893 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10895 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10896 diagnose_sb_2
, NULL
, wi
);
10897 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10898 wi
->info
= context
;
10903 tree lab
= gimple_cond_true_label (stmt
);
10906 n
= splay_tree_lookup (all_labels
,
10907 (splay_tree_key
) lab
);
10908 diagnose_sb_0 (gsi_p
, context
,
10909 n
? (gimple
) n
->value
: NULL
);
10911 lab
= gimple_cond_false_label (stmt
);
10914 n
= splay_tree_lookup (all_labels
,
10915 (splay_tree_key
) lab
);
10916 diagnose_sb_0 (gsi_p
, context
,
10917 n
? (gimple
) n
->value
: NULL
);
10924 tree lab
= gimple_goto_dest (stmt
);
10925 if (TREE_CODE (lab
) != LABEL_DECL
)
10928 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10929 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10933 case GIMPLE_SWITCH
:
10935 gimple_switch switch_stmt
= as_a
<gimple_switch
> (stmt
);
10937 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
10939 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
10940 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10941 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10947 case GIMPLE_RETURN
:
10948 diagnose_sb_0 (gsi_p
, context
, NULL
);
10958 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10961 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10964 gimple last
= last_stmt (bb
);
10965 enum gimple_code code
= gimple_code (last
);
10966 struct omp_region
*cur_region
= *region
;
10967 bool fallthru
= false;
10971 case GIMPLE_OMP_PARALLEL
:
10972 case GIMPLE_OMP_TASK
:
10973 case GIMPLE_OMP_FOR
:
10974 case GIMPLE_OMP_SINGLE
:
10975 case GIMPLE_OMP_TEAMS
:
10976 case GIMPLE_OMP_MASTER
:
10977 case GIMPLE_OMP_TASKGROUP
:
10978 case GIMPLE_OMP_ORDERED
:
10979 case GIMPLE_OMP_CRITICAL
:
10980 case GIMPLE_OMP_SECTION
:
10981 cur_region
= new_omp_region (bb
, code
, cur_region
);
10985 case GIMPLE_OMP_TARGET
:
10986 cur_region
= new_omp_region (bb
, code
, cur_region
);
10988 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10989 cur_region
= cur_region
->outer
;
10992 case GIMPLE_OMP_SECTIONS
:
10993 cur_region
= new_omp_region (bb
, code
, cur_region
);
10997 case GIMPLE_OMP_SECTIONS_SWITCH
:
11001 case GIMPLE_OMP_ATOMIC_LOAD
:
11002 case GIMPLE_OMP_ATOMIC_STORE
:
11006 case GIMPLE_OMP_RETURN
:
11007 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
11008 somewhere other than the next block. This will be
11010 cur_region
->exit
= bb
;
11011 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
11012 cur_region
= cur_region
->outer
;
11015 case GIMPLE_OMP_CONTINUE
:
11016 cur_region
->cont
= bb
;
11017 switch (cur_region
->type
)
11019 case GIMPLE_OMP_FOR
:
11020 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
11021 succs edges as abnormal to prevent splitting
11023 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
11024 /* Make the loopback edge. */
11025 make_edge (bb
, single_succ (cur_region
->entry
),
11028 /* Create an edge from GIMPLE_OMP_FOR to exit, which
11029 corresponds to the case that the body of the loop
11030 is not executed at all. */
11031 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
11032 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
11036 case GIMPLE_OMP_SECTIONS
:
11037 /* Wire up the edges into and out of the nested sections. */
11039 basic_block switch_bb
= single_succ (cur_region
->entry
);
11041 struct omp_region
*i
;
11042 for (i
= cur_region
->inner
; i
; i
= i
->next
)
11044 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
11045 make_edge (switch_bb
, i
->entry
, 0);
11046 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
11049 /* Make the loopback edge to the block with
11050 GIMPLE_OMP_SECTIONS_SWITCH. */
11051 make_edge (bb
, switch_bb
, 0);
11053 /* Make the edge from the switch to exit. */
11054 make_edge (switch_bb
, bb
->next_bb
, 0);
11060 gcc_unreachable ();
11065 gcc_unreachable ();
11068 if (*region
!= cur_region
)
11070 *region
= cur_region
;
11072 *region_idx
= cur_region
->entry
->index
;
11080 static unsigned int
11081 diagnose_omp_structured_block_errors (void)
11083 struct walk_stmt_info wi
;
11084 gimple_seq body
= gimple_body (current_function_decl
);
11086 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
11088 memset (&wi
, 0, sizeof (wi
));
11089 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
11091 memset (&wi
, 0, sizeof (wi
));
11092 wi
.want_locations
= true;
11093 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
11095 gimple_set_body (current_function_decl
, body
);
11097 splay_tree_delete (all_labels
);
11105 const pass_data pass_data_diagnose_omp_blocks
=
11107 GIMPLE_PASS
, /* type */
11108 "*diagnose_omp_blocks", /* name */
11109 OPTGROUP_NONE
, /* optinfo_flags */
11110 TV_NONE
, /* tv_id */
11111 PROP_gimple_any
, /* properties_required */
11112 0, /* properties_provided */
11113 0, /* properties_destroyed */
11114 0, /* todo_flags_start */
11115 0, /* todo_flags_finish */
11118 class pass_diagnose_omp_blocks
: public gimple_opt_pass
11121 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11122 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
11125 /* opt_pass methods: */
11126 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
11127 virtual unsigned int execute (function
*)
11129 return diagnose_omp_structured_block_errors ();
11132 }; // class pass_diagnose_omp_blocks
11134 } // anon namespace
11137 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11139 return new pass_diagnose_omp_blocks (ctxt
);
11142 /* SIMD clone supporting code. */
11144 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
11145 of arguments to reserve space for. */
11147 static struct cgraph_simd_clone
*
11148 simd_clone_struct_alloc (int nargs
)
11150 struct cgraph_simd_clone
*clone_info
;
11151 size_t len
= (sizeof (struct cgraph_simd_clone
)
11152 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
11153 clone_info
= (struct cgraph_simd_clone
*)
11154 ggc_internal_cleared_alloc (len
);
11158 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
11161 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
11162 struct cgraph_simd_clone
*from
)
11164 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
11165 + ((from
->nargs
- from
->inbranch
)
11166 * sizeof (struct cgraph_simd_clone_arg
))));
11169 /* Return vector of parameter types of function FNDECL. This uses
11170 TYPE_ARG_TYPES if available, otherwise falls back to types of
11171 DECL_ARGUMENTS types. */
11174 simd_clone_vector_of_formal_parm_types (tree fndecl
)
11176 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
11177 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
11178 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
11181 FOR_EACH_VEC_ELT (args
, i
, arg
)
11182 args
[i
] = TREE_TYPE (args
[i
]);
11186 /* Given a simd function in NODE, extract the simd specific
11187 information from the OMP clauses passed in CLAUSES, and return
11188 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
11189 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
11190 otherwise set to FALSE. */
11192 static struct cgraph_simd_clone
*
11193 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
11194 bool *inbranch_specified
)
11196 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11199 *inbranch_specified
= false;
11201 n
= args
.length ();
11202 if (n
> 0 && args
.last () == void_type_node
)
11205 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
11206 be cloned have a distinctive artificial label in addition to "omp
11210 && lookup_attribute ("cilk simd function",
11211 DECL_ATTRIBUTES (node
->decl
)));
11213 /* Allocate one more than needed just in case this is an in-branch
11214 clone which will require a mask argument. */
11215 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
11216 clone_info
->nargs
= n
;
11217 clone_info
->cilk_elemental
= cilk_clone
;
11224 clauses
= TREE_VALUE (clauses
);
11225 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
11228 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
11230 switch (OMP_CLAUSE_CODE (t
))
11232 case OMP_CLAUSE_INBRANCH
:
11233 clone_info
->inbranch
= 1;
11234 *inbranch_specified
= true;
11236 case OMP_CLAUSE_NOTINBRANCH
:
11237 clone_info
->inbranch
= 0;
11238 *inbranch_specified
= true;
11240 case OMP_CLAUSE_SIMDLEN
:
11241 clone_info
->simdlen
11242 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
11244 case OMP_CLAUSE_LINEAR
:
11246 tree decl
= OMP_CLAUSE_DECL (t
);
11247 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
11248 int argno
= TREE_INT_CST_LOW (decl
);
11249 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
11251 clone_info
->args
[argno
].arg_type
11252 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
11253 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11254 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
11255 && clone_info
->args
[argno
].linear_step
< n
);
11259 if (POINTER_TYPE_P (args
[argno
]))
11260 step
= fold_convert (ssizetype
, step
);
11261 if (!tree_fits_shwi_p (step
))
11263 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11264 "ignoring large linear step");
11268 else if (integer_zerop (step
))
11270 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11271 "ignoring zero linear step");
11277 clone_info
->args
[argno
].arg_type
11278 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
11279 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11284 case OMP_CLAUSE_UNIFORM
:
11286 tree decl
= OMP_CLAUSE_DECL (t
);
11287 int argno
= tree_to_uhwi (decl
);
11288 clone_info
->args
[argno
].arg_type
11289 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
11292 case OMP_CLAUSE_ALIGNED
:
11294 tree decl
= OMP_CLAUSE_DECL (t
);
11295 int argno
= tree_to_uhwi (decl
);
11296 clone_info
->args
[argno
].alignment
11297 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
11308 /* Given a SIMD clone in NODE, calculate the characteristic data
11309 type and return the coresponding type. The characteristic data
11310 type is computed as described in the Intel Vector ABI. */
11313 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
11314 struct cgraph_simd_clone
*clone_info
)
11316 tree type
= integer_type_node
;
11317 tree fndecl
= node
->decl
;
11319 /* a) For non-void function, the characteristic data type is the
11321 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
11322 type
= TREE_TYPE (TREE_TYPE (fndecl
));
11324 /* b) If the function has any non-uniform, non-linear parameters,
11325 then the characteristic data type is the type of the first
11329 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
11330 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
11331 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
11339 /* c) If the characteristic data type determined by a) or b) above
11340 is struct, union, or class type which is pass-by-value (except
11341 for the type that maps to the built-in complex data type), the
11342 characteristic data type is int. */
11343 if (RECORD_OR_UNION_TYPE_P (type
)
11344 && !aggregate_value_p (type
, NULL
)
11345 && TREE_CODE (type
) != COMPLEX_TYPE
)
11346 return integer_type_node
;
11348 /* d) If none of the above three classes is applicable, the
11349 characteristic data type is int. */
11353 /* e) For Intel Xeon Phi native and offload compilation, if the
11354 resulting characteristic data type is 8-bit or 16-bit integer
11355 data type, the characteristic data type is int. */
11356 /* Well, we don't handle Xeon Phi yet. */
11360 simd_clone_mangle (struct cgraph_node
*node
,
11361 struct cgraph_simd_clone
*clone_info
)
11363 char vecsize_mangle
= clone_info
->vecsize_mangle
;
11364 char mask
= clone_info
->inbranch
? 'M' : 'N';
11365 unsigned int simdlen
= clone_info
->simdlen
;
11369 gcc_assert (vecsize_mangle
&& simdlen
);
11371 pp_string (&pp
, "_ZGV");
11372 pp_character (&pp
, vecsize_mangle
);
11373 pp_character (&pp
, mask
);
11374 pp_decimal_int (&pp
, simdlen
);
11376 for (n
= 0; n
< clone_info
->nargs
; ++n
)
11378 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
11380 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
11381 pp_character (&pp
, 'u');
11382 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11384 gcc_assert (arg
.linear_step
!= 0);
11385 pp_character (&pp
, 'l');
11386 if (arg
.linear_step
> 1)
11387 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11388 else if (arg
.linear_step
< 0)
11390 pp_character (&pp
, 'n');
11391 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
11395 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11397 pp_character (&pp
, 's');
11398 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11401 pp_character (&pp
, 'v');
11404 pp_character (&pp
, 'a');
11405 pp_decimal_int (&pp
, arg
.alignment
);
11409 pp_underscore (&pp
);
11411 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11412 const char *str
= pp_formatted_text (&pp
);
11414 /* If there already is a SIMD clone with the same mangled name, don't
11415 add another one. This can happen e.g. for
11416 #pragma omp declare simd
11417 #pragma omp declare simd simdlen(8)
11418 int foo (int, int);
11419 if the simdlen is assumed to be 8 for the first one, etc. */
11420 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11421 clone
= clone
->simdclone
->next_clone
)
11422 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11426 return get_identifier (str
);
11429 /* Create a simd clone of OLD_NODE and return it. */
11431 static struct cgraph_node
*
11432 simd_clone_create (struct cgraph_node
*old_node
)
11434 struct cgraph_node
*new_node
;
11435 if (old_node
->definition
)
11437 if (!old_node
->has_gimple_body_p ())
11439 old_node
->get_body ();
11440 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
11446 tree old_decl
= old_node
->decl
;
11447 tree new_decl
= copy_node (old_node
->decl
);
11448 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11449 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11450 SET_DECL_RTL (new_decl
, NULL
);
11451 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11452 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11453 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
11454 symtab
->call_cgraph_insertion_hooks (new_node
);
11456 if (new_node
== NULL
)
11459 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11461 /* The function cgraph_function_versioning () will force the new
11462 symbol local. Undo this, and inherit external visability from
11464 new_node
->local
.local
= old_node
->local
.local
;
11465 new_node
->externally_visible
= old_node
->externally_visible
;
11470 /* Adjust the return type of the given function to its appropriate
11471 vector counterpart. Returns a simd array to be used throughout the
11472 function as a return value. */
11475 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11477 tree fndecl
= node
->decl
;
11478 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11479 unsigned int veclen
;
11482 /* Adjust the function return type. */
11483 if (orig_rettype
== void_type_node
)
11485 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11486 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
11487 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
11488 veclen
= node
->simdclone
->vecsize_int
;
11490 veclen
= node
->simdclone
->vecsize_float
;
11491 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
11492 if (veclen
> node
->simdclone
->simdlen
)
11493 veclen
= node
->simdclone
->simdlen
;
11494 if (veclen
== node
->simdclone
->simdlen
)
11495 TREE_TYPE (TREE_TYPE (fndecl
))
11496 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
11497 node
->simdclone
->simdlen
);
11500 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
11501 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11502 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11504 if (!node
->definition
)
11507 t
= DECL_RESULT (fndecl
);
11508 /* Adjust the DECL_RESULT. */
11509 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11510 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11513 tree atype
= build_array_type_nelts (orig_rettype
,
11514 node
->simdclone
->simdlen
);
11515 if (veclen
!= node
->simdclone
->simdlen
)
11516 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11518 /* Set up a SIMD array to use as the return value. */
11519 tree retval
= create_tmp_var_raw (atype
, "retval");
11520 gimple_add_tmp_var (retval
);
11524 /* Each vector argument has a corresponding array to be used locally
11525 as part of the eventual loop. Create such temporary array and
11528 PREFIX is the prefix to be used for the temporary.
11530 TYPE is the inner element type.
11532 SIMDLEN is the number of elements. */
11535 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11537 tree atype
= build_array_type_nelts (type
, simdlen
);
11538 tree avar
= create_tmp_var_raw (atype
, prefix
);
11539 gimple_add_tmp_var (avar
);
11543 /* Modify the function argument types to their corresponding vector
11544 counterparts if appropriate. Also, create one array for each simd
11545 argument to be used locally when using the function arguments as
11548 NODE is the function whose arguments are to be adjusted.
11550 Returns an adjustment vector that will be filled describing how the
11551 argument types will be adjusted. */
11553 static ipa_parm_adjustment_vec
11554 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11557 ipa_parm_adjustment_vec adjustments
;
11559 if (node
->definition
)
11560 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11562 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11563 adjustments
.create (args
.length ());
11564 unsigned i
, j
, veclen
;
11565 struct ipa_parm_adjustment adj
;
11566 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11568 memset (&adj
, 0, sizeof (adj
));
11569 tree parm
= args
[i
];
11570 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11571 adj
.base_index
= i
;
11574 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11575 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11577 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11579 /* No adjustment necessary for scalar arguments. */
11580 adj
.op
= IPA_PARM_OP_COPY
;
11584 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11585 veclen
= node
->simdclone
->vecsize_int
;
11587 veclen
= node
->simdclone
->vecsize_float
;
11588 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11589 if (veclen
> node
->simdclone
->simdlen
)
11590 veclen
= node
->simdclone
->simdlen
;
11591 adj
.arg_prefix
= "simd";
11592 adj
.type
= build_vector_type (parm_type
, veclen
);
11593 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11594 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11596 adjustments
.safe_push (adj
);
11599 memset (&adj
, 0, sizeof (adj
));
11600 adj
.op
= IPA_PARM_OP_NEW
;
11601 adj
.arg_prefix
= "simd";
11602 adj
.base_index
= i
;
11603 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11607 if (node
->definition
)
11608 node
->simdclone
->args
[i
].simd_array
11609 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11610 parm_type
, node
->simdclone
->simdlen
);
11612 adjustments
.safe_push (adj
);
11615 if (node
->simdclone
->inbranch
)
11618 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11621 memset (&adj
, 0, sizeof (adj
));
11622 adj
.op
= IPA_PARM_OP_NEW
;
11623 adj
.arg_prefix
= "mask";
11625 adj
.base_index
= i
;
11626 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11627 veclen
= node
->simdclone
->vecsize_int
;
11629 veclen
= node
->simdclone
->vecsize_float
;
11630 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11631 if (veclen
> node
->simdclone
->simdlen
)
11632 veclen
= node
->simdclone
->simdlen
;
11633 adj
.type
= build_vector_type (base_type
, veclen
);
11634 adjustments
.safe_push (adj
);
11636 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11637 adjustments
.safe_push (adj
);
11639 /* We have previously allocated one extra entry for the mask. Use
11641 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11643 if (node
->definition
)
11645 sc
->args
[i
].orig_arg
11646 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11647 sc
->args
[i
].simd_array
11648 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11650 sc
->args
[i
].orig_type
= base_type
;
11651 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11654 if (node
->definition
)
11655 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11658 tree new_arg_types
= NULL_TREE
, new_reversed
;
11659 bool last_parm_void
= false;
11660 if (args
.length () > 0 && args
.last () == void_type_node
)
11661 last_parm_void
= true;
11663 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11664 j
= adjustments
.length ();
11665 for (i
= 0; i
< j
; i
++)
11667 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11669 if (adj
->op
== IPA_PARM_OP_COPY
)
11670 ptype
= args
[adj
->base_index
];
11673 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11675 new_reversed
= nreverse (new_arg_types
);
11676 if (last_parm_void
)
11679 TREE_CHAIN (new_arg_types
) = void_list_node
;
11681 new_reversed
= void_list_node
;
11684 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11685 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11686 TREE_TYPE (node
->decl
) = new_type
;
11688 adjustments
.release ();
11691 return adjustments
;
11694 /* Initialize and copy the function arguments in NODE to their
11695 corresponding local simd arrays. Returns a fresh gimple_seq with
11696 the instruction sequence generated. */
11699 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11700 ipa_parm_adjustment_vec adjustments
)
11702 gimple_seq seq
= NULL
;
11703 unsigned i
= 0, j
= 0, k
;
11705 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11707 arg
= DECL_CHAIN (arg
), i
++, j
++)
11709 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11712 node
->simdclone
->args
[i
].vector_arg
= arg
;
11714 tree array
= node
->simdclone
->args
[i
].simd_array
;
11715 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11717 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11718 tree ptr
= build_fold_addr_expr (array
);
11719 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11720 build_int_cst (ptype
, 0));
11721 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11722 gimplify_and_add (t
, &seq
);
11726 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11727 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11728 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11730 tree ptr
= build_fold_addr_expr (array
);
11734 arg
= DECL_CHAIN (arg
);
11738 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11739 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11740 build_int_cst (ptype
, k
* elemsize
));
11741 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11742 gimplify_and_add (t
, &seq
);
11749 /* Callback info for ipa_simd_modify_stmt_ops below. */
11751 struct modify_stmt_info
{
11752 ipa_parm_adjustment_vec adjustments
;
11754 /* True if the parent statement was modified by
11755 ipa_simd_modify_stmt_ops. */
11759 /* Callback for walk_gimple_op.
11761 Adjust operands from a given statement as specified in the
11762 adjustments vector in the callback data. */
11765 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11767 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11768 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11769 tree
*orig_tp
= tp
;
11770 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11771 tp
= &TREE_OPERAND (*tp
, 0);
11772 struct ipa_parm_adjustment
*cand
= NULL
;
11773 if (TREE_CODE (*tp
) == PARM_DECL
)
11774 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11778 *walk_subtrees
= 0;
11781 tree repl
= NULL_TREE
;
11783 repl
= unshare_expr (cand
->new_decl
);
11788 *walk_subtrees
= 0;
11789 bool modified
= info
->modified
;
11790 info
->modified
= false;
11791 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11792 if (!info
->modified
)
11794 info
->modified
= modified
;
11797 info
->modified
= modified
;
11806 repl
= build_fold_addr_expr (repl
);
11808 if (is_gimple_debug (info
->stmt
))
11810 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
11811 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
11812 DECL_ARTIFICIAL (vexpr
) = 1;
11813 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
11814 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
11819 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
11821 repl
= gimple_assign_lhs (stmt
);
11823 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11824 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11827 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11829 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11835 info
->modified
= true;
11839 /* Traverse the function body and perform all modifications as
11840 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11841 modified such that the replacement/reduction value will now be an
11842 offset into the corresponding simd_array.
11844 This function will replace all function argument uses with their
11845 corresponding simd array elements, and ajust the return values
11849 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11850 ipa_parm_adjustment_vec adjustments
,
11851 tree retval_array
, tree iter
)
11854 unsigned int i
, j
, l
;
11856 /* Re-use the adjustments array, but this time use it to replace
11857 every function argument use to an offset into the corresponding
11859 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11861 if (!node
->simdclone
->args
[i
].vector_arg
)
11864 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11865 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11866 adjustments
[j
].new_decl
11867 = build4 (ARRAY_REF
,
11869 node
->simdclone
->args
[i
].simd_array
,
11871 NULL_TREE
, NULL_TREE
);
11872 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11873 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11874 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11877 l
= adjustments
.length ();
11878 for (i
= 1; i
< num_ssa_names
; i
++)
11880 tree name
= ssa_name (i
);
11882 && SSA_NAME_VAR (name
)
11883 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11885 for (j
= 0; j
< l
; j
++)
11886 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11887 && adjustments
[j
].new_decl
)
11890 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11893 = copy_var_decl (adjustments
[j
].base
,
11894 DECL_NAME (adjustments
[j
].base
),
11895 TREE_TYPE (adjustments
[j
].base
));
11896 adjustments
[j
].new_ssa_base
= base_var
;
11899 base_var
= adjustments
[j
].new_ssa_base
;
11900 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11902 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11903 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11904 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11905 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11906 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11907 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11908 gimple stmt
= gimple_build_assign (name
, new_decl
);
11909 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11912 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11917 struct modify_stmt_info info
;
11918 info
.adjustments
= adjustments
;
11920 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11922 gimple_stmt_iterator gsi
;
11924 gsi
= gsi_start_bb (bb
);
11925 while (!gsi_end_p (gsi
))
11927 gimple stmt
= gsi_stmt (gsi
);
11929 struct walk_stmt_info wi
;
11931 memset (&wi
, 0, sizeof (wi
));
11932 info
.modified
= false;
11934 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11936 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11938 tree retval
= gimple_return_retval (stmt
);
11941 gsi_remove (&gsi
, true);
11945 /* Replace `return foo' with `retval_array[iter] = foo'. */
11946 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11947 retval_array
, iter
, NULL
, NULL
);
11948 stmt
= gimple_build_assign (ref
, retval
);
11949 gsi_replace (&gsi
, stmt
, true);
11950 info
.modified
= true;
11955 update_stmt (stmt
);
11956 if (maybe_clean_eh_stmt (stmt
))
11957 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11964 /* Adjust the argument types in NODE to their appropriate vector
11968 simd_clone_adjust (struct cgraph_node
*node
)
11970 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11972 targetm
.simd_clone
.adjust (node
);
11974 tree retval
= simd_clone_adjust_return_type (node
);
11975 ipa_parm_adjustment_vec adjustments
11976 = simd_clone_adjust_argument_types (node
);
11978 push_gimplify_context ();
11980 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11982 /* Adjust all uses of vector arguments accordingly. Adjust all
11983 return values accordingly. */
11984 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11985 tree iter1
= make_ssa_name (iter
, NULL
);
11986 tree iter2
= make_ssa_name (iter
, NULL
);
11987 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11989 /* Initialize the iteration variable. */
11990 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11991 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11992 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11993 /* Insert the SIMD array and iv initialization at function
11995 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11997 pop_gimplify_context (NULL
);
11999 /* Create a new BB right before the original exit BB, to hold the
12000 iteration increment and the condition/branch. */
12001 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
12002 basic_block incr_bb
= create_empty_bb (orig_exit
);
12003 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
12004 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
12005 flag. Set it now to be a FALLTHRU_EDGE. */
12006 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
12007 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
12008 for (unsigned i
= 0;
12009 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
12011 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
12012 redirect_edge_succ (e
, incr_bb
);
12014 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
12015 e
->probability
= REG_BR_PROB_BASE
;
12016 gsi
= gsi_last_bb (incr_bb
);
12017 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
12018 build_int_cst (unsigned_type_node
,
12020 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12022 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
12023 struct loop
*loop
= alloc_loop ();
12024 cfun
->has_force_vectorize_loops
= true;
12025 loop
->safelen
= node
->simdclone
->simdlen
;
12026 loop
->force_vectorize
= true;
12027 loop
->header
= body_bb
;
12029 /* Branch around the body if the mask applies. */
12030 if (node
->simdclone
->inbranch
)
12032 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
12034 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
12035 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
12036 tree aref
= build4 (ARRAY_REF
,
12037 TREE_TYPE (TREE_TYPE (mask_array
)),
12040 g
= gimple_build_assign (mask
, aref
);
12041 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12042 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
12043 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
12045 aref
= build1 (VIEW_CONVERT_EXPR
,
12046 build_nonstandard_integer_type (bitsize
, 0), mask
);
12047 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
12048 g
= gimple_build_assign (mask
, aref
);
12049 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12052 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
12054 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12055 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
12056 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
12059 /* Generate the condition. */
12060 g
= gimple_build_cond (LT_EXPR
,
12062 build_int_cst (unsigned_type_node
,
12063 node
->simdclone
->simdlen
),
12065 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12066 e
= split_block (incr_bb
, gsi_stmt (gsi
));
12067 basic_block latch_bb
= e
->dest
;
12068 basic_block new_exit_bb
;
12069 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
12070 loop
->latch
= latch_bb
;
12072 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
12074 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
12075 /* The successor of incr_bb is already pointing to latch_bb; just
12077 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
12078 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
12080 gimple_phi phi
= create_phi_node (iter1
, body_bb
);
12081 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
12082 edge latch_edge
= single_succ_edge (latch_bb
);
12083 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
12085 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12087 /* Generate the new return. */
12088 gsi
= gsi_last_bb (new_exit_bb
);
12090 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
12091 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
12092 retval
= TREE_OPERAND (retval
, 0);
12095 retval
= build1 (VIEW_CONVERT_EXPR
,
12096 TREE_TYPE (TREE_TYPE (node
->decl
)),
12098 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
12099 false, GSI_CONTINUE_LINKING
);
12101 g
= gimple_build_return (retval
);
12102 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12104 /* Handle aligned clauses by replacing default defs of the aligned
12105 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
12106 lhs. Handle linear by adding PHIs. */
12107 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
12108 if (node
->simdclone
->args
[i
].alignment
12109 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
12110 && (node
->simdclone
->args
[i
].alignment
12111 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
12112 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
12115 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
12116 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12117 tree def
= ssa_default_def (cfun
, orig_arg
);
12118 if (def
&& !has_zero_uses (def
))
12120 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
12121 gimple_seq seq
= NULL
;
12122 bool need_cvt
= false;
12124 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
12126 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
12129 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
12130 gimple_call_set_lhs (g
, t
);
12131 gimple_seq_add_stmt_without_update (&seq
, g
);
12134 t
= make_ssa_name (orig_arg
, NULL
);
12135 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
12136 gimple_call_lhs (g
),
12138 gimple_seq_add_stmt_without_update (&seq
, g
);
12140 gsi_insert_seq_on_edge_immediate
12141 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
12143 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
12144 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
12146 node
->create_edge (cgraph_node::get_create (fn
),
12147 call
, entry_bb
->count
, freq
);
12149 imm_use_iterator iter
;
12150 use_operand_p use_p
;
12152 tree repl
= gimple_get_lhs (g
);
12153 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12154 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
12157 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12158 SET_USE (use_p
, repl
);
12161 else if (node
->simdclone
->args
[i
].arg_type
12162 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12164 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12165 tree def
= ssa_default_def (cfun
, orig_arg
);
12166 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12167 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
12168 if (def
&& !has_zero_uses (def
))
12170 iter1
= make_ssa_name (orig_arg
, NULL
);
12171 iter2
= make_ssa_name (orig_arg
, NULL
);
12172 phi
= create_phi_node (iter1
, body_bb
);
12173 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
12174 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12175 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12176 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
12177 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12178 ? TREE_TYPE (orig_arg
) : sizetype
;
12180 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
12181 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
12182 gsi
= gsi_last_bb (incr_bb
);
12183 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12185 imm_use_iterator iter
;
12186 use_operand_p use_p
;
12188 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12189 if (use_stmt
== phi
)
12192 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12193 SET_USE (use_p
, iter1
);
12197 calculate_dominance_info (CDI_DOMINATORS
);
12198 add_loop (loop
, loop
->header
->loop_father
);
12199 update_ssa (TODO_update_ssa
);
12204 /* If the function in NODE is tagged as an elemental SIMD function,
12205 create the appropriate SIMD clones. */
12208 expand_simd_clones (struct cgraph_node
*node
)
12210 tree attr
= lookup_attribute ("omp declare simd",
12211 DECL_ATTRIBUTES (node
->decl
));
12212 if (attr
== NULL_TREE
12213 || node
->global
.inlined_to
12214 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
12218 #pragma omp declare simd
12220 in C, there we don't know the argument types at all. */
12221 if (!node
->definition
12222 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
12227 /* Start with parsing the "omp declare simd" attribute(s). */
12228 bool inbranch_clause_specified
;
12229 struct cgraph_simd_clone
*clone_info
12230 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
12231 &inbranch_clause_specified
);
12232 if (clone_info
== NULL
)
12235 int orig_simdlen
= clone_info
->simdlen
;
12236 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
12237 /* The target can return 0 (no simd clones should be created),
12238 1 (just one ISA of simd clones should be created) or higher
12239 count of ISA variants. In that case, clone_info is initialized
12240 for the first ISA variant. */
12242 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
12247 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
12248 also create one inbranch and one !inbranch clone of it. */
12249 for (int i
= 0; i
< count
* 2; i
++)
12251 struct cgraph_simd_clone
*clone
= clone_info
;
12252 if (inbranch_clause_specified
&& (i
& 1) != 0)
12257 clone
= simd_clone_struct_alloc (clone_info
->nargs
12259 simd_clone_struct_copy (clone
, clone_info
);
12260 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
12261 and simd_clone_adjust_argument_types did to the first
12263 clone
->nargs
-= clone_info
->inbranch
;
12264 clone
->simdlen
= orig_simdlen
;
12265 /* And call the target hook again to get the right ISA. */
12266 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
12270 clone
->inbranch
= 1;
12273 /* simd_clone_mangle might fail if such a clone has been created
12275 tree id
= simd_clone_mangle (node
, clone
);
12276 if (id
== NULL_TREE
)
12279 /* Only when we are sure we want to create the clone actually
12280 clone the function (or definitions) or create another
12281 extern FUNCTION_DECL (for prototypes without definitions). */
12282 struct cgraph_node
*n
= simd_clone_create (node
);
12286 n
->simdclone
= clone
;
12287 clone
->origin
= node
;
12288 clone
->next_clone
= NULL
;
12289 if (node
->simd_clones
== NULL
)
12291 clone
->prev_clone
= n
;
12292 node
->simd_clones
= n
;
12296 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
12297 clone
->prev_clone
->simdclone
->next_clone
= n
;
12298 node
->simd_clones
->simdclone
->prev_clone
= n
;
12300 symtab
->change_decl_assembler_name (n
->decl
, id
);
12301 /* And finally adjust the return type, parameters and for
12302 definitions also function body. */
12303 if (node
->definition
)
12304 simd_clone_adjust (n
);
12307 simd_clone_adjust_return_type (n
);
12308 simd_clone_adjust_argument_types (n
);
12312 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
12315 /* Entry point for IPA simd clone creation pass. */
12317 static unsigned int
12318 ipa_omp_simd_clone (void)
12320 struct cgraph_node
*node
;
12321 FOR_EACH_FUNCTION (node
)
12322 expand_simd_clones (node
);
12328 const pass_data pass_data_omp_simd_clone
=
12330 SIMPLE_IPA_PASS
, /* type */
12331 "simdclone", /* name */
12332 OPTGROUP_NONE
, /* optinfo_flags */
12333 TV_NONE
, /* tv_id */
12334 ( PROP_ssa
| PROP_cfg
), /* properties_required */
12335 0, /* properties_provided */
12336 0, /* properties_destroyed */
12337 0, /* todo_flags_start */
12338 0, /* todo_flags_finish */
12341 class pass_omp_simd_clone
: public simple_ipa_opt_pass
12344 pass_omp_simd_clone(gcc::context
*ctxt
)
12345 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
12348 /* opt_pass methods: */
12349 virtual bool gate (function
*);
12350 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
12354 pass_omp_simd_clone::gate (function
*)
12356 return ((flag_openmp
|| flag_openmp_simd
12358 || (in_lto_p
&& !flag_wpa
))
12359 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
12362 } // anon namespace
12364 simple_ipa_opt_pass
*
12365 make_pass_omp_simd_clone (gcc::context
*ctxt
)
12367 return new pass_omp_simd_clone (ctxt
);
12370 #include "gt-omp-low.h"