1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
207 static vec
<omp_context
*> taskreg_contexts
;
209 static void scan_omp (gimple_seq
*, omp_context
*);
210 static tree
scan_omp_1_op (tree
*, int *, void *);
212 #define WALK_SUBSTMTS \
216 case GIMPLE_EH_FILTER: \
217 case GIMPLE_TRANSACTION: \
218 /* The sub-statements for these should be walked. */ \
219 *handled_ops_p = false; \
222 /* Convenience function for calling scan_omp_1_op on tree operands. */
225 scan_omp_op (tree
*tp
, omp_context
*ctx
)
227 struct walk_stmt_info wi
;
229 memset (&wi
, 0, sizeof (wi
));
231 wi
.want_locations
= true;
233 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
236 static void lower_omp (gimple_seq
*, omp_context
*);
237 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
238 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
240 /* Find an OpenMP clause of type KIND within CLAUSES. */
243 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
245 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
246 if (OMP_CLAUSE_CODE (clauses
) == kind
)
252 /* Return true if CTX is for an omp parallel. */
255 is_parallel_ctx (omp_context
*ctx
)
257 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
261 /* Return true if CTX is for an omp task. */
264 is_task_ctx (omp_context
*ctx
)
266 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
270 /* Return true if CTX is for an omp parallel or omp task. */
273 is_taskreg_ctx (omp_context
*ctx
)
275 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
276 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
280 /* Return true if REGION is a combined parallel+workshare region. */
283 is_combined_parallel (struct omp_region
*region
)
285 return region
->is_combined_parallel
;
289 /* Extract the header elements of parallel loop FOR_STMT and store
293 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
294 struct omp_for_data_loop
*loops
)
296 tree t
, var
, *collapse_iter
, *collapse_count
;
297 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
298 struct omp_for_data_loop
*loop
;
300 struct omp_for_data_loop dummy_loop
;
301 location_t loc
= gimple_location (for_stmt
);
302 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_KIND_SIMD
;
303 bool distribute
= gimple_omp_for_kind (for_stmt
)
304 == GF_OMP_FOR_KIND_DISTRIBUTE
;
306 fd
->for_stmt
= for_stmt
;
308 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
309 if (fd
->collapse
> 1)
312 fd
->loops
= &fd
->loop
;
314 fd
->have_nowait
= distribute
|| simd
;
315 fd
->have_ordered
= false;
316 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
317 fd
->chunk_size
= NULL_TREE
;
318 collapse_iter
= NULL
;
319 collapse_count
= NULL
;
321 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
322 switch (OMP_CLAUSE_CODE (t
))
324 case OMP_CLAUSE_NOWAIT
:
325 fd
->have_nowait
= true;
327 case OMP_CLAUSE_ORDERED
:
328 fd
->have_ordered
= true;
330 case OMP_CLAUSE_SCHEDULE
:
331 gcc_assert (!distribute
);
332 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
333 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
335 case OMP_CLAUSE_DIST_SCHEDULE
:
336 gcc_assert (distribute
);
337 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
339 case OMP_CLAUSE_COLLAPSE
:
340 if (fd
->collapse
> 1)
342 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
343 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
349 /* FIXME: for now map schedule(auto) to schedule(static).
350 There should be analysis to determine whether all iterations
351 are approximately the same amount of work (then schedule(static)
352 is best) or if it varies (then schedule(dynamic,N) is better). */
353 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
355 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
356 gcc_assert (fd
->chunk_size
== NULL
);
358 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
359 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
360 gcc_assert (fd
->chunk_size
== NULL
);
361 else if (fd
->chunk_size
== NULL
)
363 /* We only need to compute a default chunk size for ordered
364 static loops and dynamic loops. */
365 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
367 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
368 ? integer_zero_node
: integer_one_node
;
371 for (i
= 0; i
< fd
->collapse
; i
++)
373 if (fd
->collapse
== 1)
375 else if (loops
!= NULL
)
380 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
381 gcc_assert (SSA_VAR_P (loop
->v
));
382 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
383 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
384 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
385 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
387 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
388 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
389 switch (loop
->cond_code
)
395 gcc_assert (gimple_omp_for_kind (for_stmt
)
396 == GF_OMP_FOR_KIND_CILKSIMD
);
399 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
400 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
402 loop
->n2
= fold_build2_loc (loc
,
403 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
404 build_int_cst (TREE_TYPE (loop
->n2
), 1));
405 loop
->cond_code
= LT_EXPR
;
408 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
409 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
411 loop
->n2
= fold_build2_loc (loc
,
412 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
413 build_int_cst (TREE_TYPE (loop
->n2
), 1));
414 loop
->cond_code
= GT_EXPR
;
420 t
= gimple_omp_for_incr (for_stmt
, i
);
421 gcc_assert (TREE_OPERAND (t
, 0) == var
);
422 switch (TREE_CODE (t
))
425 loop
->step
= TREE_OPERAND (t
, 1);
427 case POINTER_PLUS_EXPR
:
428 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
431 loop
->step
= TREE_OPERAND (t
, 1);
432 loop
->step
= fold_build1_loc (loc
,
433 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
441 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
442 && !fd
->have_ordered
))
444 if (fd
->collapse
== 1)
445 iter_type
= TREE_TYPE (loop
->v
);
447 || TYPE_PRECISION (iter_type
)
448 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
450 = build_nonstandard_integer_type
451 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
453 else if (iter_type
!= long_long_unsigned_type_node
)
455 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
456 iter_type
= long_long_unsigned_type_node
;
457 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
458 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
459 >= TYPE_PRECISION (iter_type
))
463 if (loop
->cond_code
== LT_EXPR
)
464 n
= fold_build2_loc (loc
,
465 PLUS_EXPR
, TREE_TYPE (loop
->v
),
466 loop
->n2
, loop
->step
);
469 if (TREE_CODE (n
) != INTEGER_CST
470 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
471 iter_type
= long_long_unsigned_type_node
;
473 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
474 > TYPE_PRECISION (iter_type
))
478 if (loop
->cond_code
== LT_EXPR
)
481 n2
= fold_build2_loc (loc
,
482 PLUS_EXPR
, TREE_TYPE (loop
->v
),
483 loop
->n2
, loop
->step
);
487 n1
= fold_build2_loc (loc
,
488 MINUS_EXPR
, TREE_TYPE (loop
->v
),
489 loop
->n2
, loop
->step
);
492 if (TREE_CODE (n1
) != INTEGER_CST
493 || TREE_CODE (n2
) != INTEGER_CST
494 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
495 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
496 iter_type
= long_long_unsigned_type_node
;
500 if (collapse_count
&& *collapse_count
== NULL
)
502 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
503 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
504 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
505 if (t
&& integer_zerop (t
))
506 count
= build_zero_cst (long_long_unsigned_type_node
);
507 else if ((i
== 0 || count
!= NULL_TREE
)
508 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
509 && TREE_CONSTANT (loop
->n1
)
510 && TREE_CONSTANT (loop
->n2
)
511 && TREE_CODE (loop
->step
) == INTEGER_CST
)
513 tree itype
= TREE_TYPE (loop
->v
);
515 if (POINTER_TYPE_P (itype
))
516 itype
= signed_type_for (itype
);
517 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
518 t
= fold_build2_loc (loc
,
520 fold_convert_loc (loc
, itype
, loop
->step
), t
);
521 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
522 fold_convert_loc (loc
, itype
, loop
->n2
));
523 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
524 fold_convert_loc (loc
, itype
, loop
->n1
));
525 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
526 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
527 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
528 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
529 fold_convert_loc (loc
, itype
,
532 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
533 fold_convert_loc (loc
, itype
, loop
->step
));
534 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
535 if (count
!= NULL_TREE
)
536 count
= fold_build2_loc (loc
,
537 MULT_EXPR
, long_long_unsigned_type_node
,
541 if (TREE_CODE (count
) != INTEGER_CST
)
544 else if (count
&& !integer_zerop (count
))
551 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
552 || fd
->have_ordered
))
554 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
555 iter_type
= long_long_unsigned_type_node
;
557 iter_type
= long_integer_type_node
;
559 else if (collapse_iter
&& *collapse_iter
!= NULL
)
560 iter_type
= TREE_TYPE (*collapse_iter
);
561 fd
->iter_type
= iter_type
;
562 if (collapse_iter
&& *collapse_iter
== NULL
)
563 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
564 if (collapse_count
&& *collapse_count
== NULL
)
567 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
569 *collapse_count
= create_tmp_var (iter_type
, ".count");
572 if (fd
->collapse
> 1)
574 fd
->loop
.v
= *collapse_iter
;
575 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
576 fd
->loop
.n2
= *collapse_count
;
577 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
578 fd
->loop
.cond_code
= LT_EXPR
;
583 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
584 is the immediate dominator of PAR_ENTRY_BB, return true if there
585 are no data dependencies that would prevent expanding the parallel
586 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
588 When expanding a combined parallel+workshare region, the call to
589 the child function may need additional arguments in the case of
590 GIMPLE_OMP_FOR regions. In some cases, these arguments are
591 computed out of variables passed in from the parent to the child
592 via 'struct .omp_data_s'. For instance:
594 #pragma omp parallel for schedule (guided, i * 4)
599 # BLOCK 2 (PAR_ENTRY_BB)
601 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
603 # BLOCK 3 (WS_ENTRY_BB)
604 .omp_data_i = &.omp_data_o;
605 D.1667 = .omp_data_i->i;
607 #pragma omp for schedule (guided, D.1598)
609 When we outline the parallel region, the call to the child function
610 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
611 that value is computed *after* the call site. So, in principle we
612 cannot do the transformation.
614 To see whether the code in WS_ENTRY_BB blocks the combined
615 parallel+workshare call, we collect all the variables used in the
616 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
617 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
620 FIXME. If we had the SSA form built at this point, we could merely
621 hoist the code in block 3 into block 2 and be done with it. But at
622 this point we don't have dataflow information and though we could
623 hack something up here, it is really not worth the aggravation. */
626 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
628 struct omp_for_data fd
;
629 gimple ws_stmt
= last_stmt (ws_entry_bb
);
631 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
634 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
636 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
638 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
640 if (fd
.iter_type
!= long_integer_type_node
)
643 /* FIXME. We give up too easily here. If any of these arguments
644 are not constants, they will likely involve variables that have
645 been mapped into fields of .omp_data_s for sharing with the child
646 function. With appropriate data flow, it would be possible to
648 if (!is_gimple_min_invariant (fd
.loop
.n1
)
649 || !is_gimple_min_invariant (fd
.loop
.n2
)
650 || !is_gimple_min_invariant (fd
.loop
.step
)
651 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
658 /* Collect additional arguments needed to emit a combined
659 parallel+workshare call. WS_STMT is the workshare directive being
662 static vec
<tree
, va_gc
> *
663 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
666 location_t loc
= gimple_location (ws_stmt
);
667 vec
<tree
, va_gc
> *ws_args
;
669 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
671 struct omp_for_data fd
;
674 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
678 if (gimple_omp_for_combined_into_p (ws_stmt
))
681 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
682 OMP_CLAUSE__LOOPTEMP_
);
684 n1
= OMP_CLAUSE_DECL (innerc
);
685 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
686 OMP_CLAUSE__LOOPTEMP_
);
688 n2
= OMP_CLAUSE_DECL (innerc
);
691 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
693 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
694 ws_args
->quick_push (t
);
696 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
697 ws_args
->quick_push (t
);
699 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
700 ws_args
->quick_push (t
);
704 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
705 ws_args
->quick_push (t
);
710 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
712 /* Number of sections is equal to the number of edges from the
713 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
714 the exit of the sections region. */
715 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
716 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
717 vec_alloc (ws_args
, 1);
718 ws_args
->quick_push (t
);
726 /* Discover whether REGION is a combined parallel+workshare region. */
729 determine_parallel_type (struct omp_region
*region
)
731 basic_block par_entry_bb
, par_exit_bb
;
732 basic_block ws_entry_bb
, ws_exit_bb
;
734 if (region
== NULL
|| region
->inner
== NULL
735 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
736 || region
->inner
->cont
== NULL
)
739 /* We only support parallel+for and parallel+sections. */
740 if (region
->type
!= GIMPLE_OMP_PARALLEL
741 || (region
->inner
->type
!= GIMPLE_OMP_FOR
742 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
745 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
746 WS_EXIT_BB -> PAR_EXIT_BB. */
747 par_entry_bb
= region
->entry
;
748 par_exit_bb
= region
->exit
;
749 ws_entry_bb
= region
->inner
->entry
;
750 ws_exit_bb
= region
->inner
->exit
;
752 if (single_succ (par_entry_bb
) == ws_entry_bb
753 && single_succ (ws_exit_bb
) == par_exit_bb
754 && workshare_safe_to_combine_p (ws_entry_bb
)
755 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
756 || (last_and_only_stmt (ws_entry_bb
)
757 && last_and_only_stmt (par_exit_bb
))))
759 gimple par_stmt
= last_stmt (par_entry_bb
);
760 gimple ws_stmt
= last_stmt (ws_entry_bb
);
762 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
764 /* If this is a combined parallel loop, we need to determine
765 whether or not to use the combined library calls. There
766 are two cases where we do not apply the transformation:
767 static loops and any kind of ordered loop. In the first
768 case, we already open code the loop so there is no need
769 to do anything else. In the latter case, the combined
770 parallel loop call would still need extra synchronization
771 to implement ordered semantics, so there would not be any
772 gain in using the combined call. */
773 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
774 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
776 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
777 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
779 region
->is_combined_parallel
= false;
780 region
->inner
->is_combined_parallel
= false;
785 region
->is_combined_parallel
= true;
786 region
->inner
->is_combined_parallel
= true;
787 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
792 /* Return true if EXPR is variable sized. */
795 is_variable_sized (const_tree expr
)
797 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
800 /* Return true if DECL is a reference type. */
803 is_reference (tree decl
)
805 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
808 /* Lookup variables in the decl or field splay trees. The "maybe" form
809 allows for the variable form to not have been entered, otherwise we
810 assert that the variable must have been entered. */
813 lookup_decl (tree var
, omp_context
*ctx
)
816 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
821 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
824 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
825 return n
? *n
: NULL_TREE
;
829 lookup_field (tree var
, omp_context
*ctx
)
832 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
833 return (tree
) n
->value
;
837 lookup_sfield (tree var
, omp_context
*ctx
)
840 n
= splay_tree_lookup (ctx
->sfield_map
841 ? ctx
->sfield_map
: ctx
->field_map
,
842 (splay_tree_key
) var
);
843 return (tree
) n
->value
;
847 maybe_lookup_field (tree var
, omp_context
*ctx
)
850 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
851 return n
? (tree
) n
->value
: NULL_TREE
;
854 /* Return true if DECL should be copied by pointer. SHARED_CTX is
855 the parallel context if DECL is to be shared. */
858 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
860 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
863 /* We can only use copy-in/copy-out semantics for shared variables
864 when we know the value is not accessible from an outer scope. */
867 /* ??? Trivially accessible from anywhere. But why would we even
868 be passing an address in this case? Should we simply assert
869 this to be false, or should we have a cleanup pass that removes
870 these from the list of mappings? */
871 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
874 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
875 without analyzing the expression whether or not its location
876 is accessible to anyone else. In the case of nested parallel
877 regions it certainly may be. */
878 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
881 /* Do not use copy-in/copy-out for variables that have their
883 if (TREE_ADDRESSABLE (decl
))
886 /* lower_send_shared_vars only uses copy-in, but not copy-out
888 if (TREE_READONLY (decl
)
889 || ((TREE_CODE (decl
) == RESULT_DECL
890 || TREE_CODE (decl
) == PARM_DECL
)
891 && DECL_BY_REFERENCE (decl
)))
894 /* Disallow copy-in/out in nested parallel if
895 decl is shared in outer parallel, otherwise
896 each thread could store the shared variable
897 in its own copy-in location, making the
898 variable no longer really shared. */
899 if (shared_ctx
->is_nested
)
903 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
904 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
911 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
912 c
; c
= OMP_CLAUSE_CHAIN (c
))
913 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
914 && OMP_CLAUSE_DECL (c
) == decl
)
918 goto maybe_mark_addressable_and_ret
;
922 /* For tasks avoid using copy-in/out. As tasks can be
923 deferred or executed in different thread, when GOMP_task
924 returns, the task hasn't necessarily terminated. */
925 if (is_task_ctx (shared_ctx
))
928 maybe_mark_addressable_and_ret
:
929 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
930 if (is_gimple_reg (outer
))
932 /* Taking address of OUTER in lower_send_shared_vars
933 might need regimplification of everything that uses the
935 if (!task_shared_vars
)
936 task_shared_vars
= BITMAP_ALLOC (NULL
);
937 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
938 TREE_ADDRESSABLE (outer
) = 1;
947 /* Construct a new automatic decl similar to VAR. */
950 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
952 tree copy
= copy_var_decl (var
, name
, type
);
954 DECL_CONTEXT (copy
) = current_function_decl
;
955 DECL_CHAIN (copy
) = ctx
->block_vars
;
956 ctx
->block_vars
= copy
;
962 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
964 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
967 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
970 omp_build_component_ref (tree obj
, tree field
)
972 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
973 if (TREE_THIS_VOLATILE (field
))
974 TREE_THIS_VOLATILE (ret
) |= 1;
975 if (TREE_READONLY (field
))
976 TREE_READONLY (ret
) |= 1;
980 /* Build tree nodes to access the field for VAR on the receiver side. */
983 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
985 tree x
, field
= lookup_field (var
, ctx
);
987 /* If the receiver record type was remapped in the child function,
988 remap the field into the new record type. */
989 x
= maybe_lookup_field (field
, ctx
);
993 x
= build_simple_mem_ref (ctx
->receiver_decl
);
994 x
= omp_build_component_ref (x
, field
);
996 x
= build_simple_mem_ref (x
);
1001 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1002 of a parallel, this is a component reference; for workshare constructs
1003 this is some variable. */
1006 build_outer_var_ref (tree var
, omp_context
*ctx
)
1010 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1012 else if (is_variable_sized (var
))
1014 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1015 x
= build_outer_var_ref (x
, ctx
);
1016 x
= build_simple_mem_ref (x
);
1018 else if (is_taskreg_ctx (ctx
))
1020 bool by_ref
= use_pointer_for_field (var
, NULL
);
1021 x
= build_receiver_ref (var
, by_ref
, ctx
);
1023 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1024 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
1026 /* #pragma omp simd isn't a worksharing construct, and can reference even
1027 private vars in its linear etc. clauses. */
1029 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1030 x
= lookup_decl (var
, ctx
->outer
);
1031 else if (ctx
->outer
)
1032 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1036 else if (ctx
->outer
)
1037 x
= lookup_decl (var
, ctx
->outer
);
1038 else if (is_reference (var
))
1039 /* This can happen with orphaned constructs. If var is reference, it is
1040 possible it is shared and as such valid. */
1045 if (is_reference (var
))
1046 x
= build_simple_mem_ref (x
);
1051 /* Build tree nodes to access the field for VAR on the sender side. */
1054 build_sender_ref (tree var
, omp_context
*ctx
)
1056 tree field
= lookup_sfield (var
, ctx
);
1057 return omp_build_component_ref (ctx
->sender_decl
, field
);
1060 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1063 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1065 tree field
, type
, sfield
= NULL_TREE
;
1067 gcc_assert ((mask
& 1) == 0
1068 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1069 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1070 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1072 type
= TREE_TYPE (var
);
1075 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1076 type
= build_pointer_type (build_pointer_type (type
));
1079 type
= build_pointer_type (type
);
1080 else if ((mask
& 3) == 1 && is_reference (var
))
1081 type
= TREE_TYPE (type
);
1083 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1084 FIELD_DECL
, DECL_NAME (var
), type
);
1086 /* Remember what variable this field was created for. This does have a
1087 side effect of making dwarf2out ignore this member, so for helpful
1088 debugging we clear it later in delete_omp_context. */
1089 DECL_ABSTRACT_ORIGIN (field
) = var
;
1090 if (type
== TREE_TYPE (var
))
1092 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1093 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1094 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1097 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1099 if ((mask
& 3) == 3)
1101 insert_field_into_struct (ctx
->record_type
, field
);
1102 if (ctx
->srecord_type
)
1104 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1105 FIELD_DECL
, DECL_NAME (var
), type
);
1106 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1107 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1108 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1109 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1110 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1115 if (ctx
->srecord_type
== NULL_TREE
)
1119 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1120 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1121 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1123 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1124 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1125 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1126 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1127 splay_tree_insert (ctx
->sfield_map
,
1128 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1129 (splay_tree_value
) sfield
);
1133 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1134 : ctx
->srecord_type
, field
);
1138 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1139 (splay_tree_value
) field
);
1140 if ((mask
& 2) && ctx
->sfield_map
)
1141 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1142 (splay_tree_value
) sfield
);
1146 install_var_local (tree var
, omp_context
*ctx
)
1148 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1149 insert_decl_map (&ctx
->cb
, var
, new_var
);
1153 /* Adjust the replacement for DECL in CTX for the new context. This means
1154 copying the DECL_VALUE_EXPR, and fixing up the type. */
1157 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1159 tree new_decl
, size
;
1161 new_decl
= lookup_decl (decl
, ctx
);
1163 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1165 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1166 && DECL_HAS_VALUE_EXPR_P (decl
))
1168 tree ve
= DECL_VALUE_EXPR (decl
);
1169 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1170 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1171 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1174 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1176 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1177 if (size
== error_mark_node
)
1178 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1179 DECL_SIZE (new_decl
) = size
;
1181 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1182 if (size
== error_mark_node
)
1183 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1184 DECL_SIZE_UNIT (new_decl
) = size
;
1188 /* The callback for remap_decl. Search all containing contexts for a
1189 mapping of the variable; this avoids having to duplicate the splay
1190 tree ahead of time. We know a mapping doesn't already exist in the
1191 given context. Create new mappings to implement default semantics. */
1194 omp_copy_decl (tree var
, copy_body_data
*cb
)
1196 omp_context
*ctx
= (omp_context
*) cb
;
1199 if (TREE_CODE (var
) == LABEL_DECL
)
1201 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1202 DECL_CONTEXT (new_var
) = current_function_decl
;
1203 insert_decl_map (&ctx
->cb
, var
, new_var
);
1207 while (!is_taskreg_ctx (ctx
))
1212 new_var
= maybe_lookup_decl (var
, ctx
);
1217 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1220 return error_mark_node
;
1224 /* Debugging dumps for parallel regions. */
1225 void dump_omp_region (FILE *, struct omp_region
*, int);
1226 void debug_omp_region (struct omp_region
*);
1227 void debug_all_omp_regions (void);
1229 /* Dump the parallel region tree rooted at REGION. */
1232 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1234 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1235 gimple_code_name
[region
->type
]);
1238 dump_omp_region (file
, region
->inner
, indent
+ 4);
1242 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1243 region
->cont
->index
);
1247 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1248 region
->exit
->index
);
1250 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1253 dump_omp_region (file
, region
->next
, indent
);
1257 debug_omp_region (struct omp_region
*region
)
1259 dump_omp_region (stderr
, region
, 0);
1263 debug_all_omp_regions (void)
1265 dump_omp_region (stderr
, root_omp_region
, 0);
1269 /* Create a new parallel region starting at STMT inside region PARENT. */
1271 static struct omp_region
*
1272 new_omp_region (basic_block bb
, enum gimple_code type
,
1273 struct omp_region
*parent
)
1275 struct omp_region
*region
= XCNEW (struct omp_region
);
1277 region
->outer
= parent
;
1279 region
->type
= type
;
1283 /* This is a nested region. Add it to the list of inner
1284 regions in PARENT. */
1285 region
->next
= parent
->inner
;
1286 parent
->inner
= region
;
1290 /* This is a toplevel region. Add it to the list of toplevel
1291 regions in ROOT_OMP_REGION. */
1292 region
->next
= root_omp_region
;
1293 root_omp_region
= region
;
1299 /* Release the memory associated with the region tree rooted at REGION. */
1302 free_omp_region_1 (struct omp_region
*region
)
1304 struct omp_region
*i
, *n
;
1306 for (i
= region
->inner
; i
; i
= n
)
1309 free_omp_region_1 (i
);
1315 /* Release the memory for the entire omp region tree. */
1318 free_omp_regions (void)
1320 struct omp_region
*r
, *n
;
1321 for (r
= root_omp_region
; r
; r
= n
)
1324 free_omp_region_1 (r
);
1326 root_omp_region
= NULL
;
1330 /* Create a new context, with OUTER_CTX being the surrounding context. */
1332 static omp_context
*
1333 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1335 omp_context
*ctx
= XCNEW (omp_context
);
1337 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1338 (splay_tree_value
) ctx
);
1343 ctx
->outer
= outer_ctx
;
1344 ctx
->cb
= outer_ctx
->cb
;
1345 ctx
->cb
.block
= NULL
;
1346 ctx
->depth
= outer_ctx
->depth
+ 1;
1350 ctx
->cb
.src_fn
= current_function_decl
;
1351 ctx
->cb
.dst_fn
= current_function_decl
;
1352 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1353 gcc_checking_assert (ctx
->cb
.src_node
);
1354 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1355 ctx
->cb
.src_cfun
= cfun
;
1356 ctx
->cb
.copy_decl
= omp_copy_decl
;
1357 ctx
->cb
.eh_lp_nr
= 0;
1358 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1362 ctx
->cb
.decl_map
= pointer_map_create ();
1367 static gimple_seq
maybe_catch_exception (gimple_seq
);
1369 /* Finalize task copyfn. */
1372 finalize_task_copyfn (gimple task_stmt
)
1374 struct function
*child_cfun
;
1376 gimple_seq seq
= NULL
, new_seq
;
1379 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1380 if (child_fn
== NULL_TREE
)
1383 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1384 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1386 push_cfun (child_cfun
);
1387 bind
= gimplify_body (child_fn
, false);
1388 gimple_seq_add_stmt (&seq
, bind
);
1389 new_seq
= maybe_catch_exception (seq
);
1392 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1394 gimple_seq_add_stmt (&seq
, bind
);
1396 gimple_set_body (child_fn
, seq
);
1399 /* Inform the callgraph about the new function. */
1400 cgraph_add_new_function (child_fn
, false);
1403 /* Destroy a omp_context data structures. Called through the splay tree
1404 value delete callback. */
1407 delete_omp_context (splay_tree_value value
)
1409 omp_context
*ctx
= (omp_context
*) value
;
1411 pointer_map_destroy (ctx
->cb
.decl_map
);
1414 splay_tree_delete (ctx
->field_map
);
1415 if (ctx
->sfield_map
)
1416 splay_tree_delete (ctx
->sfield_map
);
1418 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1419 it produces corrupt debug information. */
1420 if (ctx
->record_type
)
1423 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1424 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1426 if (ctx
->srecord_type
)
1429 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1430 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1433 if (is_task_ctx (ctx
))
1434 finalize_task_copyfn (ctx
->stmt
);
1439 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1443 fixup_child_record_type (omp_context
*ctx
)
1445 tree f
, type
= ctx
->record_type
;
1447 /* ??? It isn't sufficient to just call remap_type here, because
1448 variably_modified_type_p doesn't work the way we expect for
1449 record types. Testing each field for whether it needs remapping
1450 and creating a new record by hand works, however. */
1451 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1452 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1456 tree name
, new_fields
= NULL
;
1458 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1459 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1460 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1461 TYPE_DECL
, name
, type
);
1462 TYPE_NAME (type
) = name
;
1464 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1466 tree new_f
= copy_node (f
);
1467 DECL_CONTEXT (new_f
) = type
;
1468 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1469 DECL_CHAIN (new_f
) = new_fields
;
1470 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1471 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1473 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1477 /* Arrange to be able to look up the receiver field
1478 given the sender field. */
1479 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1480 (splay_tree_value
) new_f
);
1482 TYPE_FIELDS (type
) = nreverse (new_fields
);
1486 TREE_TYPE (ctx
->receiver_decl
)
1487 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1490 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1491 specified by CLAUSES. */
1494 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1497 bool scan_array_reductions
= false;
1499 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1503 switch (OMP_CLAUSE_CODE (c
))
1505 case OMP_CLAUSE_PRIVATE
:
1506 decl
= OMP_CLAUSE_DECL (c
);
1507 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1509 else if (!is_variable_sized (decl
))
1510 install_var_local (decl
, ctx
);
1513 case OMP_CLAUSE_SHARED
:
1514 decl
= OMP_CLAUSE_DECL (c
);
1515 /* Ignore shared directives in teams construct. */
1516 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1518 /* Global variables don't need to be copied,
1519 the receiver side will use them directly. */
1520 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1521 if (is_global_var (odecl
))
1523 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1526 gcc_assert (is_taskreg_ctx (ctx
));
1527 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1528 || !is_variable_sized (decl
));
1529 /* Global variables don't need to be copied,
1530 the receiver side will use them directly. */
1531 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1533 by_ref
= use_pointer_for_field (decl
, ctx
);
1534 if (! TREE_READONLY (decl
)
1535 || TREE_ADDRESSABLE (decl
)
1537 || is_reference (decl
))
1539 install_var_field (decl
, by_ref
, 3, ctx
);
1540 install_var_local (decl
, ctx
);
1543 /* We don't need to copy const scalar vars back. */
1544 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1547 case OMP_CLAUSE_LASTPRIVATE
:
1548 /* Let the corresponding firstprivate clause create
1550 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1554 case OMP_CLAUSE_FIRSTPRIVATE
:
1555 case OMP_CLAUSE_REDUCTION
:
1556 case OMP_CLAUSE_LINEAR
:
1557 decl
= OMP_CLAUSE_DECL (c
);
1559 if (is_variable_sized (decl
))
1561 if (is_task_ctx (ctx
))
1562 install_var_field (decl
, false, 1, ctx
);
1565 else if (is_taskreg_ctx (ctx
))
1568 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1569 by_ref
= use_pointer_for_field (decl
, NULL
);
1571 if (is_task_ctx (ctx
)
1572 && (global
|| by_ref
|| is_reference (decl
)))
1574 install_var_field (decl
, false, 1, ctx
);
1576 install_var_field (decl
, by_ref
, 2, ctx
);
1579 install_var_field (decl
, by_ref
, 3, ctx
);
1581 install_var_local (decl
, ctx
);
1584 case OMP_CLAUSE__LOOPTEMP_
:
1585 gcc_assert (is_parallel_ctx (ctx
));
1586 decl
= OMP_CLAUSE_DECL (c
);
1587 install_var_field (decl
, false, 3, ctx
);
1588 install_var_local (decl
, ctx
);
1591 case OMP_CLAUSE_COPYPRIVATE
:
1592 case OMP_CLAUSE_COPYIN
:
1593 decl
= OMP_CLAUSE_DECL (c
);
1594 by_ref
= use_pointer_for_field (decl
, NULL
);
1595 install_var_field (decl
, by_ref
, 3, ctx
);
1598 case OMP_CLAUSE_DEFAULT
:
1599 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1602 case OMP_CLAUSE_FINAL
:
1604 case OMP_CLAUSE_NUM_THREADS
:
1605 case OMP_CLAUSE_NUM_TEAMS
:
1606 case OMP_CLAUSE_THREAD_LIMIT
:
1607 case OMP_CLAUSE_DEVICE
:
1608 case OMP_CLAUSE_SCHEDULE
:
1609 case OMP_CLAUSE_DIST_SCHEDULE
:
1610 case OMP_CLAUSE_DEPEND
:
1612 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1616 case OMP_CLAUSE_FROM
:
1617 case OMP_CLAUSE_MAP
:
1619 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1620 decl
= OMP_CLAUSE_DECL (c
);
1621 /* Global variables with "omp declare target" attribute
1622 don't need to be copied, the receiver side will use them
1624 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1626 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1627 && lookup_attribute ("omp declare target",
1628 DECL_ATTRIBUTES (decl
)))
1630 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1631 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1633 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1634 #pragma omp target data, there is nothing to map for
1636 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1637 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1638 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1643 if (DECL_SIZE (decl
)
1644 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1646 tree decl2
= DECL_VALUE_EXPR (decl
);
1647 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1648 decl2
= TREE_OPERAND (decl2
, 0);
1649 gcc_assert (DECL_P (decl2
));
1650 install_var_field (decl2
, true, 3, ctx
);
1651 install_var_local (decl2
, ctx
);
1652 install_var_local (decl
, ctx
);
1656 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1657 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1658 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1659 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1660 install_var_field (decl
, true, 7, ctx
);
1662 install_var_field (decl
, true, 3, ctx
);
1663 if (gimple_omp_target_kind (ctx
->stmt
)
1664 == GF_OMP_TARGET_KIND_REGION
)
1665 install_var_local (decl
, ctx
);
1670 tree base
= get_base_address (decl
);
1671 tree nc
= OMP_CLAUSE_CHAIN (c
);
1674 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1675 && OMP_CLAUSE_DECL (nc
) == base
1676 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1677 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1679 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1680 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1686 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1687 decl
= OMP_CLAUSE_DECL (c
);
1689 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1690 (splay_tree_key
) decl
));
1692 = build_decl (OMP_CLAUSE_LOCATION (c
),
1693 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1694 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1695 insert_field_into_struct (ctx
->record_type
, field
);
1696 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1697 (splay_tree_value
) field
);
1702 case OMP_CLAUSE_NOWAIT
:
1703 case OMP_CLAUSE_ORDERED
:
1704 case OMP_CLAUSE_COLLAPSE
:
1705 case OMP_CLAUSE_UNTIED
:
1706 case OMP_CLAUSE_MERGEABLE
:
1707 case OMP_CLAUSE_PROC_BIND
:
1708 case OMP_CLAUSE_SAFELEN
:
1711 case OMP_CLAUSE_ALIGNED
:
1712 decl
= OMP_CLAUSE_DECL (c
);
1713 if (is_global_var (decl
)
1714 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1715 install_var_local (decl
, ctx
);
1723 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1725 switch (OMP_CLAUSE_CODE (c
))
1727 case OMP_CLAUSE_LASTPRIVATE
:
1728 /* Let the corresponding firstprivate clause create
1730 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1731 scan_array_reductions
= true;
1732 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1736 case OMP_CLAUSE_PRIVATE
:
1737 case OMP_CLAUSE_FIRSTPRIVATE
:
1738 case OMP_CLAUSE_REDUCTION
:
1739 case OMP_CLAUSE_LINEAR
:
1740 decl
= OMP_CLAUSE_DECL (c
);
1741 if (is_variable_sized (decl
))
1742 install_var_local (decl
, ctx
);
1743 fixup_remapped_decl (decl
, ctx
,
1744 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1745 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1746 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1747 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1748 scan_array_reductions
= true;
1749 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1750 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1751 scan_array_reductions
= true;
1754 case OMP_CLAUSE_SHARED
:
1755 /* Ignore shared directives in teams construct. */
1756 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1758 decl
= OMP_CLAUSE_DECL (c
);
1759 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1760 fixup_remapped_decl (decl
, ctx
, false);
1763 case OMP_CLAUSE_MAP
:
1764 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1766 decl
= OMP_CLAUSE_DECL (c
);
1768 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1769 && lookup_attribute ("omp declare target",
1770 DECL_ATTRIBUTES (decl
)))
1774 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1775 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1776 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1778 tree new_decl
= lookup_decl (decl
, ctx
);
1779 TREE_TYPE (new_decl
)
1780 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1782 else if (DECL_SIZE (decl
)
1783 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1785 tree decl2
= DECL_VALUE_EXPR (decl
);
1786 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1787 decl2
= TREE_OPERAND (decl2
, 0);
1788 gcc_assert (DECL_P (decl2
));
1789 fixup_remapped_decl (decl2
, ctx
, false);
1790 fixup_remapped_decl (decl
, ctx
, true);
1793 fixup_remapped_decl (decl
, ctx
, false);
1797 case OMP_CLAUSE_COPYPRIVATE
:
1798 case OMP_CLAUSE_COPYIN
:
1799 case OMP_CLAUSE_DEFAULT
:
1801 case OMP_CLAUSE_NUM_THREADS
:
1802 case OMP_CLAUSE_NUM_TEAMS
:
1803 case OMP_CLAUSE_THREAD_LIMIT
:
1804 case OMP_CLAUSE_DEVICE
:
1805 case OMP_CLAUSE_SCHEDULE
:
1806 case OMP_CLAUSE_DIST_SCHEDULE
:
1807 case OMP_CLAUSE_NOWAIT
:
1808 case OMP_CLAUSE_ORDERED
:
1809 case OMP_CLAUSE_COLLAPSE
:
1810 case OMP_CLAUSE_UNTIED
:
1811 case OMP_CLAUSE_FINAL
:
1812 case OMP_CLAUSE_MERGEABLE
:
1813 case OMP_CLAUSE_PROC_BIND
:
1814 case OMP_CLAUSE_SAFELEN
:
1815 case OMP_CLAUSE_ALIGNED
:
1816 case OMP_CLAUSE_DEPEND
:
1817 case OMP_CLAUSE__LOOPTEMP_
:
1819 case OMP_CLAUSE_FROM
:
1827 if (scan_array_reductions
)
1828 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1829 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1830 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1832 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1833 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1835 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1836 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1837 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1838 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1839 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1840 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1843 /* Create a new name for omp child function. Returns an identifier. */
1846 create_omp_child_function_name (bool task_copy
)
1848 return (clone_function_name (current_function_decl
,
1849 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1852 /* Build a decl for the omp child function. It'll not contain a body
1853 yet, just the bare decl. */
1856 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1858 tree decl
, type
, name
, t
;
1860 name
= create_omp_child_function_name (task_copy
);
1862 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1863 ptr_type_node
, NULL_TREE
);
1865 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1867 decl
= build_decl (gimple_location (ctx
->stmt
),
1868 FUNCTION_DECL
, name
, type
);
1871 ctx
->cb
.dst_fn
= decl
;
1873 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1875 TREE_STATIC (decl
) = 1;
1876 TREE_USED (decl
) = 1;
1877 DECL_ARTIFICIAL (decl
) = 1;
1878 DECL_IGNORED_P (decl
) = 0;
1879 TREE_PUBLIC (decl
) = 0;
1880 DECL_UNINLINABLE (decl
) = 1;
1881 DECL_EXTERNAL (decl
) = 0;
1882 DECL_CONTEXT (decl
) = NULL_TREE
;
1883 DECL_INITIAL (decl
) = make_node (BLOCK
);
1884 bool target_p
= false;
1885 if (lookup_attribute ("omp declare target",
1886 DECL_ATTRIBUTES (current_function_decl
)))
1891 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1892 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1893 && gimple_omp_target_kind (octx
->stmt
)
1894 == GF_OMP_TARGET_KIND_REGION
)
1901 DECL_ATTRIBUTES (decl
)
1902 = tree_cons (get_identifier ("omp declare target"),
1903 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1905 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1906 RESULT_DECL
, NULL_TREE
, void_type_node
);
1907 DECL_ARTIFICIAL (t
) = 1;
1908 DECL_IGNORED_P (t
) = 1;
1909 DECL_CONTEXT (t
) = decl
;
1910 DECL_RESULT (decl
) = t
;
1912 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1913 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1914 DECL_ARTIFICIAL (t
) = 1;
1915 DECL_NAMELESS (t
) = 1;
1916 DECL_ARG_TYPE (t
) = ptr_type_node
;
1917 DECL_CONTEXT (t
) = current_function_decl
;
1919 DECL_ARGUMENTS (decl
) = t
;
1921 ctx
->receiver_decl
= t
;
1924 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1925 PARM_DECL
, get_identifier (".omp_data_o"),
1927 DECL_ARTIFICIAL (t
) = 1;
1928 DECL_NAMELESS (t
) = 1;
1929 DECL_ARG_TYPE (t
) = ptr_type_node
;
1930 DECL_CONTEXT (t
) = current_function_decl
;
1932 TREE_ADDRESSABLE (t
) = 1;
1933 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1934 DECL_ARGUMENTS (decl
) = t
;
1937 /* Allocate memory for the function structure. The call to
1938 allocate_struct_function clobbers CFUN, so we need to restore
1940 push_struct_function (decl
);
1941 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1945 /* Callback for walk_gimple_seq. Check if combined parallel
1946 contains gimple_omp_for_combined_into_p OMP_FOR. */
1949 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1950 bool *handled_ops_p
,
1951 struct walk_stmt_info
*wi
)
1953 gimple stmt
= gsi_stmt (*gsi_p
);
1955 *handled_ops_p
= true;
1956 switch (gimple_code (stmt
))
1960 case GIMPLE_OMP_FOR
:
1961 if (gimple_omp_for_combined_into_p (stmt
)
1962 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1965 return integer_zero_node
;
1974 /* Scan an OpenMP parallel directive. */
1977 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1981 gimple stmt
= gsi_stmt (*gsi
);
1983 /* Ignore parallel directives with empty bodies, unless there
1984 are copyin clauses. */
1986 && empty_body_p (gimple_omp_body (stmt
))
1987 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1988 OMP_CLAUSE_COPYIN
) == NULL
)
1990 gsi_replace (gsi
, gimple_build_nop (), false);
1994 if (gimple_omp_parallel_combined_p (stmt
))
1997 struct walk_stmt_info wi
;
1999 memset (&wi
, 0, sizeof (wi
));
2001 walk_gimple_seq (gimple_omp_body (stmt
),
2002 find_combined_for
, NULL
, &wi
);
2003 for_stmt
= (gimple
) wi
.info
;
2006 struct omp_for_data fd
;
2007 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2008 /* We need two temporaries with fd.loop.v type (istart/iend)
2009 and then (fd.collapse - 1) temporaries with the same
2010 type for count2 ... countN-1 vars if not constant. */
2011 size_t count
= 2, i
;
2012 tree type
= fd
.iter_type
;
2014 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2015 count
+= fd
.collapse
- 1;
2016 for (i
= 0; i
< count
; i
++)
2018 tree temp
= create_tmp_var (type
, NULL
);
2019 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2020 OMP_CLAUSE__LOOPTEMP_
);
2021 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2022 OMP_CLAUSE_DECL (c
) = temp
;
2023 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2024 gimple_omp_parallel_set_clauses (stmt
, c
);
2029 ctx
= new_omp_context (stmt
, outer_ctx
);
2030 taskreg_contexts
.safe_push (ctx
);
2031 if (taskreg_nesting_level
> 1)
2032 ctx
->is_nested
= true;
2033 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2034 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2035 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2036 name
= create_tmp_var_name (".omp_data_s");
2037 name
= build_decl (gimple_location (stmt
),
2038 TYPE_DECL
, name
, ctx
->record_type
);
2039 DECL_ARTIFICIAL (name
) = 1;
2040 DECL_NAMELESS (name
) = 1;
2041 TYPE_NAME (ctx
->record_type
) = name
;
2042 create_omp_child_function (ctx
, false);
2043 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2045 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2046 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2048 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2049 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2052 /* Scan an OpenMP task directive. */
2055 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2059 gimple stmt
= gsi_stmt (*gsi
);
2061 /* Ignore task directives with empty bodies. */
2063 && empty_body_p (gimple_omp_body (stmt
)))
2065 gsi_replace (gsi
, gimple_build_nop (), false);
2069 ctx
= new_omp_context (stmt
, outer_ctx
);
2070 taskreg_contexts
.safe_push (ctx
);
2071 if (taskreg_nesting_level
> 1)
2072 ctx
->is_nested
= true;
2073 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2074 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2075 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2076 name
= create_tmp_var_name (".omp_data_s");
2077 name
= build_decl (gimple_location (stmt
),
2078 TYPE_DECL
, name
, ctx
->record_type
);
2079 DECL_ARTIFICIAL (name
) = 1;
2080 DECL_NAMELESS (name
) = 1;
2081 TYPE_NAME (ctx
->record_type
) = name
;
2082 create_omp_child_function (ctx
, false);
2083 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2085 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2087 if (ctx
->srecord_type
)
2089 name
= create_tmp_var_name (".omp_data_a");
2090 name
= build_decl (gimple_location (stmt
),
2091 TYPE_DECL
, name
, ctx
->srecord_type
);
2092 DECL_ARTIFICIAL (name
) = 1;
2093 DECL_NAMELESS (name
) = 1;
2094 TYPE_NAME (ctx
->srecord_type
) = name
;
2095 create_omp_child_function (ctx
, true);
2098 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2100 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2102 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2103 t
= build_int_cst (long_integer_type_node
, 0);
2104 gimple_omp_task_set_arg_size (stmt
, t
);
2105 t
= build_int_cst (long_integer_type_node
, 1);
2106 gimple_omp_task_set_arg_align (stmt
, t
);
2111 /* If any decls have been made addressable during scan_omp,
2112 adjust their fields if needed, and layout record types
2113 of parallel/task constructs. */
2116 finish_taskreg_scan (omp_context
*ctx
)
2118 if (ctx
->record_type
== NULL_TREE
)
2121 /* If any task_shared_vars were needed, verify all
2122 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2123 statements if use_pointer_for_field hasn't changed
2124 because of that. If it did, update field types now. */
2125 if (task_shared_vars
)
2129 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2130 c
; c
= OMP_CLAUSE_CHAIN (c
))
2131 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2133 tree decl
= OMP_CLAUSE_DECL (c
);
2135 /* Global variables don't need to be copied,
2136 the receiver side will use them directly. */
2137 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2139 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2140 || !use_pointer_for_field (decl
, ctx
))
2142 tree field
= lookup_field (decl
, ctx
);
2143 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2144 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2146 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2147 TREE_THIS_VOLATILE (field
) = 0;
2148 DECL_USER_ALIGN (field
) = 0;
2149 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2150 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2151 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2152 if (ctx
->srecord_type
)
2154 tree sfield
= lookup_sfield (decl
, ctx
);
2155 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2156 TREE_THIS_VOLATILE (sfield
) = 0;
2157 DECL_USER_ALIGN (sfield
) = 0;
2158 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2159 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2160 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2165 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2167 layout_type (ctx
->record_type
);
2168 fixup_child_record_type (ctx
);
2172 location_t loc
= gimple_location (ctx
->stmt
);
2173 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2174 /* Move VLA fields to the end. */
2175 p
= &TYPE_FIELDS (ctx
->record_type
);
2177 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2178 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2181 *p
= TREE_CHAIN (*p
);
2182 TREE_CHAIN (*q
) = NULL_TREE
;
2183 q
= &TREE_CHAIN (*q
);
2186 p
= &DECL_CHAIN (*p
);
2188 layout_type (ctx
->record_type
);
2189 fixup_child_record_type (ctx
);
2190 if (ctx
->srecord_type
)
2191 layout_type (ctx
->srecord_type
);
2192 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2193 TYPE_SIZE_UNIT (ctx
->record_type
));
2194 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2195 t
= build_int_cst (long_integer_type_node
,
2196 TYPE_ALIGN_UNIT (ctx
->record_type
));
2197 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2202 /* Scan an OpenMP loop directive. */
2205 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2210 ctx
= new_omp_context (stmt
, outer_ctx
);
2212 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2214 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2215 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2217 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2218 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2219 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2220 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2222 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2225 /* Scan an OpenMP sections directive. */
2228 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2232 ctx
= new_omp_context (stmt
, outer_ctx
);
2233 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2234 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2237 /* Scan an OpenMP single directive. */
2240 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2245 ctx
= new_omp_context (stmt
, outer_ctx
);
2246 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2247 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2248 name
= create_tmp_var_name (".omp_copy_s");
2249 name
= build_decl (gimple_location (stmt
),
2250 TYPE_DECL
, name
, ctx
->record_type
);
2251 TYPE_NAME (ctx
->record_type
) = name
;
2253 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2254 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2256 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2257 ctx
->record_type
= NULL
;
2259 layout_type (ctx
->record_type
);
2262 /* Scan an OpenMP target{, data, update} directive. */
2265 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2269 int kind
= gimple_omp_target_kind (stmt
);
2271 ctx
= new_omp_context (stmt
, outer_ctx
);
2272 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2273 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2274 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2275 name
= create_tmp_var_name (".omp_data_t");
2276 name
= build_decl (gimple_location (stmt
),
2277 TYPE_DECL
, name
, ctx
->record_type
);
2278 DECL_ARTIFICIAL (name
) = 1;
2279 DECL_NAMELESS (name
) = 1;
2280 TYPE_NAME (ctx
->record_type
) = name
;
2281 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2283 create_omp_child_function (ctx
, false);
2284 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2287 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2288 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2290 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2291 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2294 TYPE_FIELDS (ctx
->record_type
)
2295 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2296 #ifdef ENABLE_CHECKING
2298 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2299 for (field
= TYPE_FIELDS (ctx
->record_type
);
2301 field
= DECL_CHAIN (field
))
2302 gcc_assert (DECL_ALIGN (field
) == align
);
2304 layout_type (ctx
->record_type
);
2305 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2306 fixup_child_record_type (ctx
);
2310 /* Scan an OpenMP teams directive. */
2313 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2315 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2316 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2317 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2320 /* Check OpenMP nesting restrictions. */
2322 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2326 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2327 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2329 error_at (gimple_location (stmt
),
2330 "OpenMP constructs may not be nested inside simd region");
2333 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2335 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2336 || (gimple_omp_for_kind (stmt
)
2337 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2338 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2340 error_at (gimple_location (stmt
),
2341 "only distribute or parallel constructs are allowed to "
2342 "be closely nested inside teams construct");
2347 switch (gimple_code (stmt
))
2349 case GIMPLE_OMP_FOR
:
2350 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_KIND_SIMD
)
2352 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2354 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2356 error_at (gimple_location (stmt
),
2357 "distribute construct must be closely nested inside "
2365 if (is_gimple_call (stmt
)
2366 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2367 == BUILT_IN_GOMP_CANCEL
2368 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2369 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2371 const char *bad
= NULL
;
2372 const char *kind
= NULL
;
2375 error_at (gimple_location (stmt
), "orphaned %qs construct",
2376 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2377 == BUILT_IN_GOMP_CANCEL
2378 ? "#pragma omp cancel"
2379 : "#pragma omp cancellation point");
2382 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2383 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2387 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2388 bad
= "#pragma omp parallel";
2389 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2390 == BUILT_IN_GOMP_CANCEL
2391 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2392 ctx
->cancellable
= true;
2396 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2397 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2398 bad
= "#pragma omp for";
2399 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2400 == BUILT_IN_GOMP_CANCEL
2401 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2403 ctx
->cancellable
= true;
2404 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2406 warning_at (gimple_location (stmt
), 0,
2407 "%<#pragma omp cancel for%> inside "
2408 "%<nowait%> for construct");
2409 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2410 OMP_CLAUSE_ORDERED
))
2411 warning_at (gimple_location (stmt
), 0,
2412 "%<#pragma omp cancel for%> inside "
2413 "%<ordered%> for construct");
2418 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2419 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2420 bad
= "#pragma omp sections";
2421 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2422 == BUILT_IN_GOMP_CANCEL
2423 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2425 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2427 ctx
->cancellable
= true;
2428 if (find_omp_clause (gimple_omp_sections_clauses
2431 warning_at (gimple_location (stmt
), 0,
2432 "%<#pragma omp cancel sections%> inside "
2433 "%<nowait%> sections construct");
2437 gcc_assert (ctx
->outer
2438 && gimple_code (ctx
->outer
->stmt
)
2439 == GIMPLE_OMP_SECTIONS
);
2440 ctx
->outer
->cancellable
= true;
2441 if (find_omp_clause (gimple_omp_sections_clauses
2444 warning_at (gimple_location (stmt
), 0,
2445 "%<#pragma omp cancel sections%> inside "
2446 "%<nowait%> sections construct");
2452 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2453 bad
= "#pragma omp task";
2455 ctx
->cancellable
= true;
2459 error_at (gimple_location (stmt
), "invalid arguments");
2464 error_at (gimple_location (stmt
),
2465 "%<%s %s%> construct not closely nested inside of %qs",
2466 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2467 == BUILT_IN_GOMP_CANCEL
2468 ? "#pragma omp cancel"
2469 : "#pragma omp cancellation point", kind
, bad
);
2474 case GIMPLE_OMP_SECTIONS
:
2475 case GIMPLE_OMP_SINGLE
:
2476 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2477 switch (gimple_code (ctx
->stmt
))
2479 case GIMPLE_OMP_FOR
:
2480 case GIMPLE_OMP_SECTIONS
:
2481 case GIMPLE_OMP_SINGLE
:
2482 case GIMPLE_OMP_ORDERED
:
2483 case GIMPLE_OMP_MASTER
:
2484 case GIMPLE_OMP_TASK
:
2485 case GIMPLE_OMP_CRITICAL
:
2486 if (is_gimple_call (stmt
))
2488 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2489 != BUILT_IN_GOMP_BARRIER
)
2491 error_at (gimple_location (stmt
),
2492 "barrier region may not be closely nested inside "
2493 "of work-sharing, critical, ordered, master or "
2494 "explicit task region");
2497 error_at (gimple_location (stmt
),
2498 "work-sharing region may not be closely nested inside "
2499 "of work-sharing, critical, ordered, master or explicit "
2502 case GIMPLE_OMP_PARALLEL
:
2508 case GIMPLE_OMP_MASTER
:
2509 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2510 switch (gimple_code (ctx
->stmt
))
2512 case GIMPLE_OMP_FOR
:
2513 case GIMPLE_OMP_SECTIONS
:
2514 case GIMPLE_OMP_SINGLE
:
2515 case GIMPLE_OMP_TASK
:
2516 error_at (gimple_location (stmt
),
2517 "master region may not be closely nested inside "
2518 "of work-sharing or explicit task region");
2520 case GIMPLE_OMP_PARALLEL
:
2526 case GIMPLE_OMP_ORDERED
:
2527 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2528 switch (gimple_code (ctx
->stmt
))
2530 case GIMPLE_OMP_CRITICAL
:
2531 case GIMPLE_OMP_TASK
:
2532 error_at (gimple_location (stmt
),
2533 "ordered region may not be closely nested inside "
2534 "of critical or explicit task region");
2536 case GIMPLE_OMP_FOR
:
2537 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2538 OMP_CLAUSE_ORDERED
) == NULL
)
2540 error_at (gimple_location (stmt
),
2541 "ordered region must be closely nested inside "
2542 "a loop region with an ordered clause");
2546 case GIMPLE_OMP_PARALLEL
:
2547 error_at (gimple_location (stmt
),
2548 "ordered region must be closely nested inside "
2549 "a loop region with an ordered clause");
2555 case GIMPLE_OMP_CRITICAL
:
2556 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2557 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2558 && (gimple_omp_critical_name (stmt
)
2559 == gimple_omp_critical_name (ctx
->stmt
)))
2561 error_at (gimple_location (stmt
),
2562 "critical region may not be nested inside a critical "
2563 "region with the same name");
2567 case GIMPLE_OMP_TEAMS
:
2569 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2570 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2572 error_at (gimple_location (stmt
),
2573 "teams construct not closely nested inside of target "
2578 case GIMPLE_OMP_TARGET
:
2579 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2580 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2581 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2584 switch (gimple_omp_target_kind (stmt
))
2586 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2587 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2588 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2589 default: gcc_unreachable ();
2591 warning_at (gimple_location (stmt
), 0,
2592 "%s construct inside of target region", name
);
2602 /* Helper function scan_omp.
2604 Callback for walk_tree or operators in walk_gimple_stmt used to
2605 scan for OpenMP directives in TP. */
2608 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2610 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2611 omp_context
*ctx
= (omp_context
*) wi
->info
;
2614 switch (TREE_CODE (t
))
2621 *tp
= remap_decl (t
, &ctx
->cb
);
2625 if (ctx
&& TYPE_P (t
))
2626 *tp
= remap_type (t
, &ctx
->cb
);
2627 else if (!DECL_P (t
))
2632 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2633 if (tem
!= TREE_TYPE (t
))
2635 if (TREE_CODE (t
) == INTEGER_CST
)
2636 *tp
= build_int_cst_wide (tem
,
2637 TREE_INT_CST_LOW (t
),
2638 TREE_INT_CST_HIGH (t
));
2640 TREE_TYPE (t
) = tem
;
2650 /* Return true if FNDECL is a setjmp or a longjmp. */
2653 setjmp_or_longjmp_p (const_tree fndecl
)
2655 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2656 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2657 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2660 tree declname
= DECL_NAME (fndecl
);
2663 const char *name
= IDENTIFIER_POINTER (declname
);
2664 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2668 /* Helper function for scan_omp.
2670 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2671 the current statement in GSI. */
2674 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2675 struct walk_stmt_info
*wi
)
2677 gimple stmt
= gsi_stmt (*gsi
);
2678 omp_context
*ctx
= (omp_context
*) wi
->info
;
2680 if (gimple_has_location (stmt
))
2681 input_location
= gimple_location (stmt
);
2683 /* Check the OpenMP nesting restrictions. */
2684 bool remove
= false;
2685 if (is_gimple_omp (stmt
))
2686 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2687 else if (is_gimple_call (stmt
))
2689 tree fndecl
= gimple_call_fndecl (stmt
);
2692 if (setjmp_or_longjmp_p (fndecl
)
2694 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2695 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2698 error_at (gimple_location (stmt
),
2699 "setjmp/longjmp inside simd construct");
2701 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2702 switch (DECL_FUNCTION_CODE (fndecl
))
2704 case BUILT_IN_GOMP_BARRIER
:
2705 case BUILT_IN_GOMP_CANCEL
:
2706 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2707 case BUILT_IN_GOMP_TASKYIELD
:
2708 case BUILT_IN_GOMP_TASKWAIT
:
2709 case BUILT_IN_GOMP_TASKGROUP_START
:
2710 case BUILT_IN_GOMP_TASKGROUP_END
:
2711 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2720 stmt
= gimple_build_nop ();
2721 gsi_replace (gsi
, stmt
, false);
2724 *handled_ops_p
= true;
2726 switch (gimple_code (stmt
))
2728 case GIMPLE_OMP_PARALLEL
:
2729 taskreg_nesting_level
++;
2730 scan_omp_parallel (gsi
, ctx
);
2731 taskreg_nesting_level
--;
2734 case GIMPLE_OMP_TASK
:
2735 taskreg_nesting_level
++;
2736 scan_omp_task (gsi
, ctx
);
2737 taskreg_nesting_level
--;
2740 case GIMPLE_OMP_FOR
:
2741 scan_omp_for (stmt
, ctx
);
2744 case GIMPLE_OMP_SECTIONS
:
2745 scan_omp_sections (stmt
, ctx
);
2748 case GIMPLE_OMP_SINGLE
:
2749 scan_omp_single (stmt
, ctx
);
2752 case GIMPLE_OMP_SECTION
:
2753 case GIMPLE_OMP_MASTER
:
2754 case GIMPLE_OMP_TASKGROUP
:
2755 case GIMPLE_OMP_ORDERED
:
2756 case GIMPLE_OMP_CRITICAL
:
2757 ctx
= new_omp_context (stmt
, ctx
);
2758 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2761 case GIMPLE_OMP_TARGET
:
2762 scan_omp_target (stmt
, ctx
);
2765 case GIMPLE_OMP_TEAMS
:
2766 scan_omp_teams (stmt
, ctx
);
2773 *handled_ops_p
= false;
2775 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2776 insert_decl_map (&ctx
->cb
, var
, var
);
2780 *handled_ops_p
= false;
2788 /* Scan all the statements starting at the current statement. CTX
2789 contains context information about the OpenMP directives and
2790 clauses found during the scan. */
2793 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2795 location_t saved_location
;
2796 struct walk_stmt_info wi
;
2798 memset (&wi
, 0, sizeof (wi
));
2800 wi
.want_locations
= true;
2802 saved_location
= input_location
;
2803 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2804 input_location
= saved_location
;
2807 /* Re-gimplification and code generation routines. */
2809 /* Build a call to GOMP_barrier. */
2812 build_omp_barrier (tree lhs
)
2814 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2815 : BUILT_IN_GOMP_BARRIER
);
2816 gimple g
= gimple_build_call (fndecl
, 0);
2818 gimple_call_set_lhs (g
, lhs
);
2822 /* If a context was created for STMT when it was scanned, return it. */
2824 static omp_context
*
2825 maybe_lookup_ctx (gimple stmt
)
2828 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2829 return n
? (omp_context
*) n
->value
: NULL
;
2833 /* Find the mapping for DECL in CTX or the immediately enclosing
2834 context that has a mapping for DECL.
2836 If CTX is a nested parallel directive, we may have to use the decl
2837 mappings created in CTX's parent context. Suppose that we have the
2838 following parallel nesting (variable UIDs showed for clarity):
2841 #omp parallel shared(iD.1562) -> outer parallel
2842 iD.1562 = iD.1562 + 1;
2844 #omp parallel shared (iD.1562) -> inner parallel
2845 iD.1562 = iD.1562 - 1;
2847 Each parallel structure will create a distinct .omp_data_s structure
2848 for copying iD.1562 in/out of the directive:
2850 outer parallel .omp_data_s.1.i -> iD.1562
2851 inner parallel .omp_data_s.2.i -> iD.1562
2853 A shared variable mapping will produce a copy-out operation before
2854 the parallel directive and a copy-in operation after it. So, in
2855 this case we would have:
2858 .omp_data_o.1.i = iD.1562;
2859 #omp parallel shared(iD.1562) -> outer parallel
2860 .omp_data_i.1 = &.omp_data_o.1
2861 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2863 .omp_data_o.2.i = iD.1562; -> **
2864 #omp parallel shared(iD.1562) -> inner parallel
2865 .omp_data_i.2 = &.omp_data_o.2
2866 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2869 ** This is a problem. The symbol iD.1562 cannot be referenced
2870 inside the body of the outer parallel region. But since we are
2871 emitting this copy operation while expanding the inner parallel
2872 directive, we need to access the CTX structure of the outer
2873 parallel directive to get the correct mapping:
2875 .omp_data_o.2.i = .omp_data_i.1->i
2877 Since there may be other workshare or parallel directives enclosing
2878 the parallel directive, it may be necessary to walk up the context
2879 parent chain. This is not a problem in general because nested
2880 parallelism happens only rarely. */
2883 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2888 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2889 t
= maybe_lookup_decl (decl
, up
);
2891 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2893 return t
? t
: decl
;
2897 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2898 in outer contexts. */
2901 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2906 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2907 t
= maybe_lookup_decl (decl
, up
);
2909 return t
? t
: decl
;
2913 /* Construct the initialization value for reduction CLAUSE. */
2916 omp_reduction_init (tree clause
, tree type
)
2918 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2919 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2926 case TRUTH_ORIF_EXPR
:
2927 case TRUTH_XOR_EXPR
:
2929 return build_zero_cst (type
);
2932 case TRUTH_AND_EXPR
:
2933 case TRUTH_ANDIF_EXPR
:
2935 return fold_convert_loc (loc
, type
, integer_one_node
);
2938 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2941 if (SCALAR_FLOAT_TYPE_P (type
))
2943 REAL_VALUE_TYPE max
, min
;
2944 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2947 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2950 real_maxval (&min
, 1, TYPE_MODE (type
));
2951 return build_real (type
, min
);
2955 gcc_assert (INTEGRAL_TYPE_P (type
));
2956 return TYPE_MIN_VALUE (type
);
2960 if (SCALAR_FLOAT_TYPE_P (type
))
2962 REAL_VALUE_TYPE max
;
2963 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2966 real_maxval (&max
, 0, TYPE_MODE (type
));
2967 return build_real (type
, max
);
2971 gcc_assert (INTEGRAL_TYPE_P (type
));
2972 return TYPE_MAX_VALUE (type
);
2980 /* Return alignment to be assumed for var in CLAUSE, which should be
2981 OMP_CLAUSE_ALIGNED. */
2984 omp_clause_aligned_alignment (tree clause
)
2986 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2987 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2989 /* Otherwise return implementation defined alignment. */
2990 unsigned int al
= 1;
2991 enum machine_mode mode
, vmode
;
2992 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2994 vs
= 1 << floor_log2 (vs
);
2995 static enum mode_class classes
[]
2996 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2997 for (int i
= 0; i
< 4; i
+= 2)
2998 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3000 mode
= GET_MODE_WIDER_MODE (mode
))
3002 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3003 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3006 && GET_MODE_SIZE (vmode
) < vs
3007 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3008 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3010 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3011 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3013 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3014 / GET_MODE_SIZE (mode
));
3015 if (TYPE_MODE (type
) != vmode
)
3017 if (TYPE_ALIGN_UNIT (type
) > al
)
3018 al
= TYPE_ALIGN_UNIT (type
);
3020 return build_int_cst (integer_type_node
, al
);
3023 /* Return maximum possible vectorization factor for the target. */
3030 || !flag_tree_loop_optimize
3031 || (!flag_tree_loop_vectorize
3032 && (global_options_set
.x_flag_tree_loop_vectorize
3033 || global_options_set
.x_flag_tree_vectorize
)))
3036 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3039 vs
= 1 << floor_log2 (vs
);
3042 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3043 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3044 return GET_MODE_NUNITS (vqimode
);
3048 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3052 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3053 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3057 max_vf
= omp_max_vf ();
3060 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3061 OMP_CLAUSE_SAFELEN
);
3062 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3064 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3066 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3070 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3071 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3077 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3078 tree avar
= create_tmp_var_raw (atype
, NULL
);
3079 if (TREE_ADDRESSABLE (new_var
))
3080 TREE_ADDRESSABLE (avar
) = 1;
3081 DECL_ATTRIBUTES (avar
)
3082 = tree_cons (get_identifier ("omp simd array"), NULL
,
3083 DECL_ATTRIBUTES (avar
));
3084 gimple_add_tmp_var (avar
);
3085 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3086 NULL_TREE
, NULL_TREE
);
3087 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3088 NULL_TREE
, NULL_TREE
);
3089 if (DECL_P (new_var
))
3091 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3092 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3097 /* Helper function of lower_rec_input_clauses. For a reference
3098 in simd reduction, add an underlying variable it will reference. */
3101 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3103 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3104 if (TREE_CONSTANT (z
))
3106 const char *name
= NULL
;
3107 if (DECL_NAME (new_vard
))
3108 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3110 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3111 gimple_add_tmp_var (z
);
3112 TREE_ADDRESSABLE (z
) = 1;
3113 z
= build_fold_addr_expr_loc (loc
, z
);
3114 gimplify_assign (new_vard
, z
, ilist
);
3118 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3119 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3120 private variables. Initialization statements go in ILIST, while calls
3121 to destructors go in DLIST. */
3124 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3125 omp_context
*ctx
, struct omp_for_data
*fd
)
3127 tree c
, dtor
, copyin_seq
, x
, ptr
;
3128 bool copyin_by_ref
= false;
3129 bool lastprivate_firstprivate
= false;
3130 bool reduction_omp_orig_ref
= false;
3132 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3133 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
);
3135 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3136 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3137 gimple_seq llist
[2] = { NULL
, NULL
};
3141 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3142 with data sharing clauses referencing variable sized vars. That
3143 is unnecessarily hard to support and very unlikely to result in
3144 vectorized code anyway. */
3146 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3147 switch (OMP_CLAUSE_CODE (c
))
3149 case OMP_CLAUSE_LINEAR
:
3150 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3153 case OMP_CLAUSE_REDUCTION
:
3154 case OMP_CLAUSE_PRIVATE
:
3155 case OMP_CLAUSE_FIRSTPRIVATE
:
3156 case OMP_CLAUSE_LASTPRIVATE
:
3157 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3164 /* Do all the fixed sized types in the first pass, and the variable sized
3165 types in the second pass. This makes sure that the scalar arguments to
3166 the variable sized types are processed before we use them in the
3167 variable sized operations. */
3168 for (pass
= 0; pass
< 2; ++pass
)
3170 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3172 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3175 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3179 case OMP_CLAUSE_PRIVATE
:
3180 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3183 case OMP_CLAUSE_SHARED
:
3184 /* Ignore shared directives in teams construct. */
3185 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3187 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3189 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3192 case OMP_CLAUSE_FIRSTPRIVATE
:
3193 case OMP_CLAUSE_COPYIN
:
3194 case OMP_CLAUSE_LINEAR
:
3196 case OMP_CLAUSE_REDUCTION
:
3197 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3198 reduction_omp_orig_ref
= true;
3200 case OMP_CLAUSE__LOOPTEMP_
:
3201 /* Handle _looptemp_ clauses only on parallel. */
3205 case OMP_CLAUSE_LASTPRIVATE
:
3206 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3208 lastprivate_firstprivate
= true;
3212 /* Even without corresponding firstprivate, if
3213 decl is Fortran allocatable, it needs outer var
3216 && lang_hooks
.decls
.omp_private_outer_ref
3217 (OMP_CLAUSE_DECL (c
)))
3218 lastprivate_firstprivate
= true;
3220 case OMP_CLAUSE_ALIGNED
:
3223 var
= OMP_CLAUSE_DECL (c
);
3224 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3225 && !is_global_var (var
))
3227 new_var
= maybe_lookup_decl (var
, ctx
);
3228 if (new_var
== NULL_TREE
)
3229 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3230 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3231 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3232 omp_clause_aligned_alignment (c
));
3233 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3234 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3235 gimplify_and_add (x
, ilist
);
3237 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3238 && is_global_var (var
))
3240 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3241 new_var
= lookup_decl (var
, ctx
);
3242 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3243 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3244 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3245 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3246 omp_clause_aligned_alignment (c
));
3247 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3248 x
= create_tmp_var (ptype
, NULL
);
3249 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3250 gimplify_and_add (t
, ilist
);
3251 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3252 SET_DECL_VALUE_EXPR (new_var
, t
);
3253 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3260 new_var
= var
= OMP_CLAUSE_DECL (c
);
3261 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3262 new_var
= lookup_decl (var
, ctx
);
3264 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3269 else if (is_variable_sized (var
))
3271 /* For variable sized types, we need to allocate the
3272 actual storage here. Call alloca and store the
3273 result in the pointer decl that we created elsewhere. */
3277 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3282 ptr
= DECL_VALUE_EXPR (new_var
);
3283 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3284 ptr
= TREE_OPERAND (ptr
, 0);
3285 gcc_assert (DECL_P (ptr
));
3286 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3288 /* void *tmp = __builtin_alloca */
3289 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3290 stmt
= gimple_build_call (atmp
, 1, x
);
3291 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3292 gimple_add_tmp_var (tmp
);
3293 gimple_call_set_lhs (stmt
, tmp
);
3295 gimple_seq_add_stmt (ilist
, stmt
);
3297 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3298 gimplify_assign (ptr
, x
, ilist
);
3301 else if (is_reference (var
))
3303 /* For references that are being privatized for Fortran,
3304 allocate new backing storage for the new pointer
3305 variable. This allows us to avoid changing all the
3306 code that expects a pointer to something that expects
3307 a direct variable. */
3311 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3312 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3314 x
= build_receiver_ref (var
, false, ctx
);
3315 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3317 else if (TREE_CONSTANT (x
))
3319 /* For reduction in SIMD loop, defer adding the
3320 initialization of the reference, because if we decide
3321 to use SIMD array for it, the initilization could cause
3323 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3327 const char *name
= NULL
;
3328 if (DECL_NAME (var
))
3329 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3331 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3333 gimple_add_tmp_var (x
);
3334 TREE_ADDRESSABLE (x
) = 1;
3335 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3340 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3341 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3346 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3347 gimplify_assign (new_var
, x
, ilist
);
3350 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3352 else if (c_kind
== OMP_CLAUSE_REDUCTION
3353 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3361 switch (OMP_CLAUSE_CODE (c
))
3363 case OMP_CLAUSE_SHARED
:
3364 /* Ignore shared directives in teams construct. */
3365 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3367 /* Shared global vars are just accessed directly. */
3368 if (is_global_var (new_var
))
3370 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3371 needs to be delayed until after fixup_child_record_type so
3372 that we get the correct type during the dereference. */
3373 by_ref
= use_pointer_for_field (var
, ctx
);
3374 x
= build_receiver_ref (var
, by_ref
, ctx
);
3375 SET_DECL_VALUE_EXPR (new_var
, x
);
3376 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3378 /* ??? If VAR is not passed by reference, and the variable
3379 hasn't been initialized yet, then we'll get a warning for
3380 the store into the omp_data_s structure. Ideally, we'd be
3381 able to notice this and not store anything at all, but
3382 we're generating code too early. Suppress the warning. */
3384 TREE_NO_WARNING (var
) = 1;
3387 case OMP_CLAUSE_LASTPRIVATE
:
3388 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3392 case OMP_CLAUSE_PRIVATE
:
3393 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3394 x
= build_outer_var_ref (var
, ctx
);
3395 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3397 if (is_task_ctx (ctx
))
3398 x
= build_receiver_ref (var
, false, ctx
);
3400 x
= build_outer_var_ref (var
, ctx
);
3406 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3409 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3410 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3411 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3412 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3413 idx
, lane
, ivar
, lvar
))
3416 x
= lang_hooks
.decls
.omp_clause_default_ctor
3417 (c
, unshare_expr (ivar
), x
);
3419 gimplify_and_add (x
, &llist
[0]);
3422 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3425 gimple_seq tseq
= NULL
;
3428 gimplify_stmt (&dtor
, &tseq
);
3429 gimple_seq_add_seq (&llist
[1], tseq
);
3436 gimplify_and_add (nx
, ilist
);
3440 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3443 gimple_seq tseq
= NULL
;
3446 gimplify_stmt (&dtor
, &tseq
);
3447 gimple_seq_add_seq (dlist
, tseq
);
3451 case OMP_CLAUSE_LINEAR
:
3452 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3453 goto do_firstprivate
;
3454 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3457 x
= build_outer_var_ref (var
, ctx
);
3460 case OMP_CLAUSE_FIRSTPRIVATE
:
3461 if (is_task_ctx (ctx
))
3463 if (is_reference (var
) || is_variable_sized (var
))
3465 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3467 || use_pointer_for_field (var
, NULL
))
3469 x
= build_receiver_ref (var
, false, ctx
);
3470 SET_DECL_VALUE_EXPR (new_var
, x
);
3471 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3476 x
= build_outer_var_ref (var
, ctx
);
3479 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3480 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3482 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3483 tree stept
= TREE_TYPE (t
);
3484 tree ct
= find_omp_clause (clauses
,
3485 OMP_CLAUSE__LOOPTEMP_
);
3487 tree l
= OMP_CLAUSE_DECL (ct
);
3488 tree n1
= fd
->loop
.n1
;
3489 tree step
= fd
->loop
.step
;
3490 tree itype
= TREE_TYPE (l
);
3491 if (POINTER_TYPE_P (itype
))
3492 itype
= signed_type_for (itype
);
3493 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3494 if (TYPE_UNSIGNED (itype
)
3495 && fd
->loop
.cond_code
== GT_EXPR
)
3496 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3497 fold_build1 (NEGATE_EXPR
, itype
, l
),
3498 fold_build1 (NEGATE_EXPR
,
3501 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3502 t
= fold_build2 (MULT_EXPR
, stept
,
3503 fold_convert (stept
, l
), t
);
3505 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3507 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3509 gimplify_and_add (x
, ilist
);
3513 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3514 x
= fold_build2 (POINTER_PLUS_EXPR
,
3515 TREE_TYPE (x
), x
, t
);
3517 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3520 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3521 || TREE_ADDRESSABLE (new_var
))
3522 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3523 idx
, lane
, ivar
, lvar
))
3525 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3527 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3528 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3529 gimplify_and_add (x
, ilist
);
3530 gimple_stmt_iterator gsi
3531 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3533 = gimple_build_assign (unshare_expr (lvar
), iv
);
3534 gsi_insert_before_without_update (&gsi
, g
,
3536 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3537 enum tree_code code
= PLUS_EXPR
;
3538 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3539 code
= POINTER_PLUS_EXPR
;
3540 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3541 gsi_insert_before_without_update (&gsi
, g
,
3545 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3546 (c
, unshare_expr (ivar
), x
);
3547 gimplify_and_add (x
, &llist
[0]);
3548 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3551 gimple_seq tseq
= NULL
;
3554 gimplify_stmt (&dtor
, &tseq
);
3555 gimple_seq_add_seq (&llist
[1], tseq
);
3560 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3561 gimplify_and_add (x
, ilist
);
3564 case OMP_CLAUSE__LOOPTEMP_
:
3565 gcc_assert (is_parallel_ctx (ctx
));
3566 x
= build_outer_var_ref (var
, ctx
);
3567 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3568 gimplify_and_add (x
, ilist
);
3571 case OMP_CLAUSE_COPYIN
:
3572 by_ref
= use_pointer_for_field (var
, NULL
);
3573 x
= build_receiver_ref (var
, by_ref
, ctx
);
3574 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3575 append_to_statement_list (x
, ©in_seq
);
3576 copyin_by_ref
|= by_ref
;
3579 case OMP_CLAUSE_REDUCTION
:
3580 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3582 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3584 x
= build_outer_var_ref (var
, ctx
);
3586 if (is_reference (var
)
3587 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3589 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3590 SET_DECL_VALUE_EXPR (placeholder
, x
);
3591 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3592 tree new_vard
= new_var
;
3593 if (is_reference (var
))
3595 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3596 new_vard
= TREE_OPERAND (new_var
, 0);
3597 gcc_assert (DECL_P (new_vard
));
3600 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3601 idx
, lane
, ivar
, lvar
))
3603 if (new_vard
== new_var
)
3605 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3606 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3610 SET_DECL_VALUE_EXPR (new_vard
,
3611 build_fold_addr_expr (ivar
));
3612 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3614 x
= lang_hooks
.decls
.omp_clause_default_ctor
3615 (c
, unshare_expr (ivar
),
3616 build_outer_var_ref (var
, ctx
));
3618 gimplify_and_add (x
, &llist
[0]);
3619 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3621 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3622 lower_omp (&tseq
, ctx
);
3623 gimple_seq_add_seq (&llist
[0], tseq
);
3625 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3626 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3627 lower_omp (&tseq
, ctx
);
3628 gimple_seq_add_seq (&llist
[1], tseq
);
3629 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3630 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3631 if (new_vard
== new_var
)
3632 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3634 SET_DECL_VALUE_EXPR (new_vard
,
3635 build_fold_addr_expr (lvar
));
3636 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3641 gimplify_stmt (&dtor
, &tseq
);
3642 gimple_seq_add_seq (&llist
[1], tseq
);
3646 /* If this is a reference to constant size reduction var
3647 with placeholder, we haven't emitted the initializer
3648 for it because it is undesirable if SIMD arrays are used.
3649 But if they aren't used, we need to emit the deferred
3650 initialization now. */
3651 else if (is_reference (var
) && is_simd
)
3652 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3653 x
= lang_hooks
.decls
.omp_clause_default_ctor
3654 (c
, unshare_expr (new_var
),
3655 build_outer_var_ref (var
, ctx
));
3657 gimplify_and_add (x
, ilist
);
3658 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3660 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3661 lower_omp (&tseq
, ctx
);
3662 gimple_seq_add_seq (ilist
, tseq
);
3664 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3667 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3668 lower_omp (&tseq
, ctx
);
3669 gimple_seq_add_seq (dlist
, tseq
);
3670 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3672 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3677 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3678 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3679 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3681 /* reduction(-:var) sums up the partial results, so it
3682 acts identically to reduction(+:var). */
3683 if (code
== MINUS_EXPR
)
3686 tree new_vard
= new_var
;
3687 if (is_simd
&& is_reference (var
))
3689 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3690 new_vard
= TREE_OPERAND (new_var
, 0);
3691 gcc_assert (DECL_P (new_vard
));
3694 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3695 idx
, lane
, ivar
, lvar
))
3697 tree ref
= build_outer_var_ref (var
, ctx
);
3699 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3701 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3702 ref
= build_outer_var_ref (var
, ctx
);
3703 gimplify_assign (ref
, x
, &llist
[1]);
3705 if (new_vard
!= new_var
)
3707 SET_DECL_VALUE_EXPR (new_vard
,
3708 build_fold_addr_expr (lvar
));
3709 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3714 if (is_reference (var
) && is_simd
)
3715 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3716 gimplify_assign (new_var
, x
, ilist
);
3719 tree ref
= build_outer_var_ref (var
, ctx
);
3721 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3722 ref
= build_outer_var_ref (var
, ctx
);
3723 gimplify_assign (ref
, x
, dlist
);
3737 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3738 /* Don't want uninit warnings on simduid, it is always uninitialized,
3739 but we use it not for the value, but for the DECL_UID only. */
3740 TREE_NO_WARNING (uid
) = 1;
3742 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3743 gimple_call_set_lhs (g
, lane
);
3744 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3745 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3746 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3747 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3748 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3749 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3750 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3751 build_int_cst (unsigned_type_node
, 0),
3753 gimple_seq_add_stmt (ilist
, g
);
3754 for (int i
= 0; i
< 2; i
++)
3757 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3758 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3759 gimple_call_set_lhs (g
, vf
);
3760 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3761 gimple_seq_add_stmt (seq
, g
);
3762 tree t
= build_int_cst (unsigned_type_node
, 0);
3763 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3764 gimple_seq_add_stmt (seq
, g
);
3765 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3766 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3767 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3768 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3769 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3770 gimple_seq_add_seq (seq
, llist
[i
]);
3771 t
= build_int_cst (unsigned_type_node
, 1);
3772 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3773 gimple_seq_add_stmt (seq
, g
);
3774 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3775 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3776 gimple_seq_add_stmt (seq
, g
);
3777 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3781 /* The copyin sequence is not to be executed by the main thread, since
3782 that would result in self-copies. Perhaps not visible to scalars,
3783 but it certainly is to C++ operator=. */
3786 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3788 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3789 build_int_cst (TREE_TYPE (x
), 0));
3790 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3791 gimplify_and_add (x
, ilist
);
3794 /* If any copyin variable is passed by reference, we must ensure the
3795 master thread doesn't modify it before it is copied over in all
3796 threads. Similarly for variables in both firstprivate and
3797 lastprivate clauses we need to ensure the lastprivate copying
3798 happens after firstprivate copying in all threads. And similarly
3799 for UDRs if initializer expression refers to omp_orig. */
3800 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3802 /* Don't add any barrier for #pragma omp simd or
3803 #pragma omp distribute. */
3804 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3805 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3806 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3809 /* If max_vf is non-zero, then we can use only a vectorization factor
3810 up to the max_vf we chose. So stick it into the safelen clause. */
3813 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3814 OMP_CLAUSE_SAFELEN
);
3816 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3817 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3820 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3821 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3823 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3824 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3830 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3831 both parallel and workshare constructs. PREDICATE may be NULL if it's
3835 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3838 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3839 bool par_clauses
= false;
3840 tree simduid
= NULL
, lastlane
= NULL
;
3842 /* Early exit if there are no lastprivate or linear clauses. */
3843 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3844 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3845 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3846 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3848 if (clauses
== NULL
)
3850 /* If this was a workshare clause, see if it had been combined
3851 with its parallel. In that case, look for the clauses on the
3852 parallel statement itself. */
3853 if (is_parallel_ctx (ctx
))
3857 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3860 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3861 OMP_CLAUSE_LASTPRIVATE
);
3862 if (clauses
== NULL
)
3870 tree label_true
, arm1
, arm2
;
3872 label
= create_artificial_label (UNKNOWN_LOCATION
);
3873 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3874 arm1
= TREE_OPERAND (predicate
, 0);
3875 arm2
= TREE_OPERAND (predicate
, 1);
3876 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3877 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3878 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3880 gimple_seq_add_stmt (stmt_list
, stmt
);
3881 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3884 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3885 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3887 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3889 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3892 for (c
= clauses
; c
;)
3895 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3897 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3898 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3899 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3901 var
= OMP_CLAUSE_DECL (c
);
3902 new_var
= lookup_decl (var
, ctx
);
3904 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3906 tree val
= DECL_VALUE_EXPR (new_var
);
3907 if (TREE_CODE (val
) == ARRAY_REF
3908 && VAR_P (TREE_OPERAND (val
, 0))
3909 && lookup_attribute ("omp simd array",
3910 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3913 if (lastlane
== NULL
)
3915 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3917 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3919 TREE_OPERAND (val
, 1));
3920 gimple_call_set_lhs (g
, lastlane
);
3921 gimple_seq_add_stmt (stmt_list
, g
);
3923 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3924 TREE_OPERAND (val
, 0), lastlane
,
3925 NULL_TREE
, NULL_TREE
);
3929 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3930 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3932 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3933 gimple_seq_add_seq (stmt_list
,
3934 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3935 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3937 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3938 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
3940 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
3941 gimple_seq_add_seq (stmt_list
,
3942 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
3943 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
3946 x
= build_outer_var_ref (var
, ctx
);
3947 if (is_reference (var
))
3948 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3949 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3950 gimplify_and_add (x
, stmt_list
);
3952 c
= OMP_CLAUSE_CHAIN (c
);
3953 if (c
== NULL
&& !par_clauses
)
3955 /* If this was a workshare clause, see if it had been combined
3956 with its parallel. In that case, continue looking for the
3957 clauses also on the parallel statement itself. */
3958 if (is_parallel_ctx (ctx
))
3962 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3965 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3966 OMP_CLAUSE_LASTPRIVATE
);
3972 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3976 /* Generate code to implement the REDUCTION clauses. */
3979 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3981 gimple_seq sub_seq
= NULL
;
3986 /* SIMD reductions are handled in lower_rec_input_clauses. */
3987 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3988 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3991 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3992 update in that case, otherwise use a lock. */
3993 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3994 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3996 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3998 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4008 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4010 tree var
, ref
, new_var
;
4011 enum tree_code code
;
4012 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4014 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4017 var
= OMP_CLAUSE_DECL (c
);
4018 new_var
= lookup_decl (var
, ctx
);
4019 if (is_reference (var
))
4020 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4021 ref
= build_outer_var_ref (var
, ctx
);
4022 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4024 /* reduction(-:var) sums up the partial results, so it acts
4025 identically to reduction(+:var). */
4026 if (code
== MINUS_EXPR
)
4031 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4033 addr
= save_expr (addr
);
4034 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4035 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4036 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4037 gimplify_and_add (x
, stmt_seqp
);
4041 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4043 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4045 if (is_reference (var
)
4046 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4048 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4049 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4050 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4051 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4052 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4053 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4054 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4058 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4059 ref
= build_outer_var_ref (var
, ctx
);
4060 gimplify_assign (ref
, x
, &sub_seq
);
4064 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4066 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4068 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4070 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4072 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4076 /* Generate code to implement the COPYPRIVATE clauses. */
4079 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4084 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4086 tree var
, new_var
, ref
, x
;
4088 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4090 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4093 var
= OMP_CLAUSE_DECL (c
);
4094 by_ref
= use_pointer_for_field (var
, NULL
);
4096 ref
= build_sender_ref (var
, ctx
);
4097 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4100 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4101 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4103 gimplify_assign (ref
, x
, slist
);
4105 ref
= build_receiver_ref (var
, false, ctx
);
4108 ref
= fold_convert_loc (clause_loc
,
4109 build_pointer_type (TREE_TYPE (new_var
)),
4111 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4113 if (is_reference (var
))
4115 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4116 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4117 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4119 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4120 gimplify_and_add (x
, rlist
);
4125 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4126 and REDUCTION from the sender (aka parent) side. */
4129 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4134 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4136 tree val
, ref
, x
, var
;
4137 bool by_ref
, do_in
= false, do_out
= false;
4138 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4140 switch (OMP_CLAUSE_CODE (c
))
4142 case OMP_CLAUSE_PRIVATE
:
4143 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4146 case OMP_CLAUSE_FIRSTPRIVATE
:
4147 case OMP_CLAUSE_COPYIN
:
4148 case OMP_CLAUSE_LASTPRIVATE
:
4149 case OMP_CLAUSE_REDUCTION
:
4150 case OMP_CLAUSE__LOOPTEMP_
:
4156 val
= OMP_CLAUSE_DECL (c
);
4157 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4159 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4160 && is_global_var (var
))
4162 if (is_variable_sized (val
))
4164 by_ref
= use_pointer_for_field (val
, NULL
);
4166 switch (OMP_CLAUSE_CODE (c
))
4168 case OMP_CLAUSE_PRIVATE
:
4169 case OMP_CLAUSE_FIRSTPRIVATE
:
4170 case OMP_CLAUSE_COPYIN
:
4171 case OMP_CLAUSE__LOOPTEMP_
:
4175 case OMP_CLAUSE_LASTPRIVATE
:
4176 if (by_ref
|| is_reference (val
))
4178 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4185 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4190 case OMP_CLAUSE_REDUCTION
:
4192 do_out
= !(by_ref
|| is_reference (val
));
4201 ref
= build_sender_ref (val
, ctx
);
4202 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4203 gimplify_assign (ref
, x
, ilist
);
4204 if (is_task_ctx (ctx
))
4205 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4210 ref
= build_sender_ref (val
, ctx
);
4211 gimplify_assign (var
, ref
, olist
);
4216 /* Generate code to implement SHARED from the sender (aka parent)
4217 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4218 list things that got automatically shared. */
4221 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4223 tree var
, ovar
, nvar
, f
, x
, record_type
;
4225 if (ctx
->record_type
== NULL
)
4228 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4229 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4231 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4232 nvar
= maybe_lookup_decl (ovar
, ctx
);
4233 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4236 /* If CTX is a nested parallel directive. Find the immediately
4237 enclosing parallel or workshare construct that contains a
4238 mapping for OVAR. */
4239 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4241 if (use_pointer_for_field (ovar
, ctx
))
4243 x
= build_sender_ref (ovar
, ctx
);
4244 var
= build_fold_addr_expr (var
);
4245 gimplify_assign (x
, var
, ilist
);
4249 x
= build_sender_ref (ovar
, ctx
);
4250 gimplify_assign (x
, var
, ilist
);
4252 if (!TREE_READONLY (var
)
4253 /* We don't need to receive a new reference to a result
4254 or parm decl. In fact we may not store to it as we will
4255 invalidate any pending RSO and generate wrong gimple
4257 && !((TREE_CODE (var
) == RESULT_DECL
4258 || TREE_CODE (var
) == PARM_DECL
)
4259 && DECL_BY_REFERENCE (var
)))
4261 x
= build_sender_ref (ovar
, ctx
);
4262 gimplify_assign (var
, x
, olist
);
4269 /* A convenience function to build an empty GIMPLE_COND with just the
4273 gimple_build_cond_empty (tree cond
)
4275 enum tree_code pred_code
;
4278 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4279 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4283 /* Build the function calls to GOMP_parallel_start etc to actually
4284 generate the parallel operation. REGION is the parallel region
4285 being expanded. BB is the block where to insert the code. WS_ARGS
4286 will be set if this is a call to a combined parallel+workshare
4287 construct, it contains the list of additional arguments needed by
4288 the workshare construct. */
4291 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4292 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4294 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4295 gimple_stmt_iterator gsi
;
4297 enum built_in_function start_ix
;
4299 location_t clause_loc
;
4300 vec
<tree
, va_gc
> *args
;
4302 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4304 /* Determine what flavor of GOMP_parallel we will be
4306 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4307 if (is_combined_parallel (region
))
4309 switch (region
->inner
->type
)
4311 case GIMPLE_OMP_FOR
:
4312 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4313 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4314 + (region
->inner
->sched_kind
4315 == OMP_CLAUSE_SCHEDULE_RUNTIME
4316 ? 3 : region
->inner
->sched_kind
));
4317 start_ix
= (enum built_in_function
)start_ix2
;
4319 case GIMPLE_OMP_SECTIONS
:
4320 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4327 /* By default, the value of NUM_THREADS is zero (selected at run time)
4328 and there is no conditional. */
4330 val
= build_int_cst (unsigned_type_node
, 0);
4331 flags
= build_int_cst (unsigned_type_node
, 0);
4333 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4335 cond
= OMP_CLAUSE_IF_EXPR (c
);
4337 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4340 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4341 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4344 clause_loc
= gimple_location (entry_stmt
);
4346 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4348 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4350 /* Ensure 'val' is of the correct type. */
4351 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4353 /* If we found the clause 'if (cond)', build either
4354 (cond != 0) or (cond ? val : 1u). */
4357 gimple_stmt_iterator gsi
;
4359 cond
= gimple_boolify (cond
);
4361 if (integer_zerop (val
))
4362 val
= fold_build2_loc (clause_loc
,
4363 EQ_EXPR
, unsigned_type_node
, cond
,
4364 build_int_cst (TREE_TYPE (cond
), 0));
4367 basic_block cond_bb
, then_bb
, else_bb
;
4368 edge e
, e_then
, e_else
;
4369 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4371 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4372 if (gimple_in_ssa_p (cfun
))
4374 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4375 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4376 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4385 e
= split_block (bb
, NULL
);
4390 then_bb
= create_empty_bb (cond_bb
);
4391 else_bb
= create_empty_bb (then_bb
);
4392 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4393 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4395 stmt
= gimple_build_cond_empty (cond
);
4396 gsi
= gsi_start_bb (cond_bb
);
4397 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4399 gsi
= gsi_start_bb (then_bb
);
4400 stmt
= gimple_build_assign (tmp_then
, val
);
4401 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4403 gsi
= gsi_start_bb (else_bb
);
4404 stmt
= gimple_build_assign
4405 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4406 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4408 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4409 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4412 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4413 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4415 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4416 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4418 if (gimple_in_ssa_p (cfun
))
4420 gimple phi
= create_phi_node (tmp_join
, bb
);
4421 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4422 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4428 gsi
= gsi_start_bb (bb
);
4429 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4430 false, GSI_CONTINUE_LINKING
);
4433 gsi
= gsi_last_bb (bb
);
4434 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4436 t1
= null_pointer_node
;
4438 t1
= build_fold_addr_expr (t
);
4439 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4441 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4442 args
->quick_push (t2
);
4443 args
->quick_push (t1
);
4444 args
->quick_push (val
);
4446 args
->splice (*ws_args
);
4447 args
->quick_push (flags
);
4449 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4450 builtin_decl_explicit (start_ix
), args
);
4452 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4453 false, GSI_CONTINUE_LINKING
);
4457 /* Build the function call to GOMP_task to actually
4458 generate the task operation. BB is the block where to insert the code. */
4461 expand_task_call (basic_block bb
, gimple entry_stmt
)
4463 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4464 gimple_stmt_iterator gsi
;
4465 location_t loc
= gimple_location (entry_stmt
);
4467 clauses
= gimple_omp_task_clauses (entry_stmt
);
4469 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4471 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4473 cond
= boolean_true_node
;
4475 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4476 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4477 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4478 flags
= build_int_cst (unsigned_type_node
,
4479 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4481 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4484 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4485 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4486 build_int_cst (unsigned_type_node
, 2),
4487 build_int_cst (unsigned_type_node
, 0));
4488 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4491 depend
= OMP_CLAUSE_DECL (depend
);
4493 depend
= build_int_cst (ptr_type_node
, 0);
4495 gsi
= gsi_last_bb (bb
);
4496 t
= gimple_omp_task_data_arg (entry_stmt
);
4498 t2
= null_pointer_node
;
4500 t2
= build_fold_addr_expr_loc (loc
, t
);
4501 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4502 t
= gimple_omp_task_copy_fn (entry_stmt
);
4504 t3
= null_pointer_node
;
4506 t3
= build_fold_addr_expr_loc (loc
, t
);
4508 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4510 gimple_omp_task_arg_size (entry_stmt
),
4511 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4514 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4515 false, GSI_CONTINUE_LINKING
);
4519 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4520 catch handler and return it. This prevents programs from violating the
4521 structured block semantics with throws. */
4524 maybe_catch_exception (gimple_seq body
)
4529 if (!flag_exceptions
)
4532 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4533 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4535 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4537 g
= gimple_build_eh_must_not_throw (decl
);
4538 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4541 return gimple_seq_alloc_with_stmt (g
);
4544 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4547 vec2chain (vec
<tree
, va_gc
> *v
)
4549 tree chain
= NULL_TREE
, t
;
4552 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4554 DECL_CHAIN (t
) = chain
;
4562 /* Remove barriers in REGION->EXIT's block. Note that this is only
4563 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4564 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4565 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4569 remove_exit_barrier (struct omp_region
*region
)
4571 gimple_stmt_iterator gsi
;
4572 basic_block exit_bb
;
4576 int any_addressable_vars
= -1;
4578 exit_bb
= region
->exit
;
4580 /* If the parallel region doesn't return, we don't have REGION->EXIT
4585 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4586 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4587 statements that can appear in between are extremely limited -- no
4588 memory operations at all. Here, we allow nothing at all, so the
4589 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4590 gsi
= gsi_last_bb (exit_bb
);
4591 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4593 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4596 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4598 gsi
= gsi_last_bb (e
->src
);
4599 if (gsi_end_p (gsi
))
4601 stmt
= gsi_stmt (gsi
);
4602 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4603 && !gimple_omp_return_nowait_p (stmt
))
4605 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4606 in many cases. If there could be tasks queued, the barrier
4607 might be needed to let the tasks run before some local
4608 variable of the parallel that the task uses as shared
4609 runs out of scope. The task can be spawned either
4610 from within current function (this would be easy to check)
4611 or from some function it calls and gets passed an address
4612 of such a variable. */
4613 if (any_addressable_vars
< 0)
4615 gimple parallel_stmt
= last_stmt (region
->entry
);
4616 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4617 tree local_decls
, block
, decl
;
4620 any_addressable_vars
= 0;
4621 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4622 if (TREE_ADDRESSABLE (decl
))
4624 any_addressable_vars
= 1;
4627 for (block
= gimple_block (stmt
);
4628 !any_addressable_vars
4630 && TREE_CODE (block
) == BLOCK
;
4631 block
= BLOCK_SUPERCONTEXT (block
))
4633 for (local_decls
= BLOCK_VARS (block
);
4635 local_decls
= DECL_CHAIN (local_decls
))
4636 if (TREE_ADDRESSABLE (local_decls
))
4638 any_addressable_vars
= 1;
4641 if (block
== gimple_block (parallel_stmt
))
4645 if (!any_addressable_vars
)
4646 gimple_omp_return_set_nowait (stmt
);
4652 remove_exit_barriers (struct omp_region
*region
)
4654 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4655 remove_exit_barrier (region
);
4659 region
= region
->inner
;
4660 remove_exit_barriers (region
);
4661 while (region
->next
)
4663 region
= region
->next
;
4664 remove_exit_barriers (region
);
4669 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4670 calls. These can't be declared as const functions, but
4671 within one parallel body they are constant, so they can be
4672 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4673 which are declared const. Similarly for task body, except
4674 that in untied task omp_get_thread_num () can change at any task
4675 scheduling point. */
4678 optimize_omp_library_calls (gimple entry_stmt
)
4681 gimple_stmt_iterator gsi
;
4682 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4683 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4684 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4685 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4686 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4687 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4688 OMP_CLAUSE_UNTIED
) != NULL
);
4690 FOR_EACH_BB_FN (bb
, cfun
)
4691 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4693 gimple call
= gsi_stmt (gsi
);
4696 if (is_gimple_call (call
)
4697 && (decl
= gimple_call_fndecl (call
))
4698 && DECL_EXTERNAL (decl
)
4699 && TREE_PUBLIC (decl
)
4700 && DECL_INITIAL (decl
) == NULL
)
4704 if (DECL_NAME (decl
) == thr_num_id
)
4706 /* In #pragma omp task untied omp_get_thread_num () can change
4707 during the execution of the task region. */
4710 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4712 else if (DECL_NAME (decl
) == num_thr_id
)
4713 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4717 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4718 || gimple_call_num_args (call
) != 0)
4721 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4724 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4725 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4726 TREE_TYPE (TREE_TYPE (built_in
))))
4729 gimple_call_set_fndecl (call
, built_in
);
4734 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4738 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4742 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4743 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4746 if (TREE_CODE (t
) == ADDR_EXPR
)
4747 recompute_tree_invariant_for_addr_expr (t
);
4749 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4753 /* Prepend TO = FROM assignment before *GSI_P. */
4756 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4758 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4759 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4760 true, GSI_SAME_STMT
);
4761 gimple stmt
= gimple_build_assign (to
, from
);
4762 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4763 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4764 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4766 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4767 gimple_regimplify_operands (stmt
, &gsi
);
4771 /* Expand the OpenMP parallel or task directive starting at REGION. */
4774 expand_omp_taskreg (struct omp_region
*region
)
4776 basic_block entry_bb
, exit_bb
, new_bb
;
4777 struct function
*child_cfun
;
4778 tree child_fn
, block
, t
;
4779 gimple_stmt_iterator gsi
;
4780 gimple entry_stmt
, stmt
;
4782 vec
<tree
, va_gc
> *ws_args
;
4784 entry_stmt
= last_stmt (region
->entry
);
4785 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4786 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4788 entry_bb
= region
->entry
;
4789 exit_bb
= region
->exit
;
4791 if (is_combined_parallel (region
))
4792 ws_args
= region
->ws_args
;
4796 if (child_cfun
->cfg
)
4798 /* Due to inlining, it may happen that we have already outlined
4799 the region, in which case all we need to do is make the
4800 sub-graph unreachable and emit the parallel call. */
4801 edge entry_succ_e
, exit_succ_e
;
4802 gimple_stmt_iterator gsi
;
4804 entry_succ_e
= single_succ_edge (entry_bb
);
4806 gsi
= gsi_last_bb (entry_bb
);
4807 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4808 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4809 gsi_remove (&gsi
, true);
4814 exit_succ_e
= single_succ_edge (exit_bb
);
4815 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4817 remove_edge_and_dominated_blocks (entry_succ_e
);
4821 unsigned srcidx
, dstidx
, num
;
4823 /* If the parallel region needs data sent from the parent
4824 function, then the very first statement (except possible
4825 tree profile counter updates) of the parallel body
4826 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4827 &.OMP_DATA_O is passed as an argument to the child function,
4828 we need to replace it with the argument as seen by the child
4831 In most cases, this will end up being the identity assignment
4832 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4833 a function call that has been inlined, the original PARM_DECL
4834 .OMP_DATA_I may have been converted into a different local
4835 variable. In which case, we need to keep the assignment. */
4836 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4838 basic_block entry_succ_bb
= single_succ (entry_bb
);
4839 gimple_stmt_iterator gsi
;
4841 gimple parcopy_stmt
= NULL
;
4843 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4847 gcc_assert (!gsi_end_p (gsi
));
4848 stmt
= gsi_stmt (gsi
);
4849 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4852 if (gimple_num_ops (stmt
) == 2)
4854 tree arg
= gimple_assign_rhs1 (stmt
);
4856 /* We're ignore the subcode because we're
4857 effectively doing a STRIP_NOPS. */
4859 if (TREE_CODE (arg
) == ADDR_EXPR
4860 && TREE_OPERAND (arg
, 0)
4861 == gimple_omp_taskreg_data_arg (entry_stmt
))
4863 parcopy_stmt
= stmt
;
4869 gcc_assert (parcopy_stmt
!= NULL
);
4870 arg
= DECL_ARGUMENTS (child_fn
);
4872 if (!gimple_in_ssa_p (cfun
))
4874 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4875 gsi_remove (&gsi
, true);
4878 /* ?? Is setting the subcode really necessary ?? */
4879 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4880 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4885 /* If we are in ssa form, we must load the value from the default
4886 definition of the argument. That should not be defined now,
4887 since the argument is not used uninitialized. */
4888 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4889 narg
= make_ssa_name (arg
, gimple_build_nop ());
4890 set_ssa_default_def (cfun
, arg
, narg
);
4891 /* ?? Is setting the subcode really necessary ?? */
4892 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4893 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4894 update_stmt (parcopy_stmt
);
4898 /* Declare local variables needed in CHILD_CFUN. */
4899 block
= DECL_INITIAL (child_fn
);
4900 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4901 /* The gimplifier could record temporaries in parallel/task block
4902 rather than in containing function's local_decls chain,
4903 which would mean cgraph missed finalizing them. Do it now. */
4904 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4905 if (TREE_CODE (t
) == VAR_DECL
4907 && !DECL_EXTERNAL (t
))
4908 varpool_finalize_decl (t
);
4909 DECL_SAVED_TREE (child_fn
) = NULL
;
4910 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4911 gimple_set_body (child_fn
, NULL
);
4912 TREE_USED (block
) = 1;
4914 /* Reset DECL_CONTEXT on function arguments. */
4915 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4916 DECL_CONTEXT (t
) = child_fn
;
4918 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4919 so that it can be moved to the child function. */
4920 gsi
= gsi_last_bb (entry_bb
);
4921 stmt
= gsi_stmt (gsi
);
4922 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4923 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4924 gsi_remove (&gsi
, true);
4925 e
= split_block (entry_bb
, stmt
);
4927 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4929 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4932 gsi
= gsi_last_bb (exit_bb
);
4933 gcc_assert (!gsi_end_p (gsi
)
4934 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4935 stmt
= gimple_build_return (NULL
);
4936 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4937 gsi_remove (&gsi
, true);
4940 /* Move the parallel region into CHILD_CFUN. */
4942 if (gimple_in_ssa_p (cfun
))
4944 init_tree_ssa (child_cfun
);
4945 init_ssa_operands (child_cfun
);
4946 child_cfun
->gimple_df
->in_ssa_p
= true;
4950 block
= gimple_block (entry_stmt
);
4952 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4954 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4955 /* When the OMP expansion process cannot guarantee an up-to-date
4956 loop tree arrange for the child function to fixup loops. */
4957 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4958 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4960 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4961 num
= vec_safe_length (child_cfun
->local_decls
);
4962 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4964 t
= (*child_cfun
->local_decls
)[srcidx
];
4965 if (DECL_CONTEXT (t
) == cfun
->decl
)
4967 if (srcidx
!= dstidx
)
4968 (*child_cfun
->local_decls
)[dstidx
] = t
;
4972 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4974 /* Inform the callgraph about the new function. */
4975 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4976 cgraph_add_new_function (child_fn
, true);
4978 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4979 fixed in a following pass. */
4980 push_cfun (child_cfun
);
4982 optimize_omp_library_calls (entry_stmt
);
4983 rebuild_cgraph_edges ();
4985 /* Some EH regions might become dead, see PR34608. If
4986 pass_cleanup_cfg isn't the first pass to happen with the
4987 new child, these dead EH edges might cause problems.
4988 Clean them up now. */
4989 if (flag_exceptions
)
4992 bool changed
= false;
4994 FOR_EACH_BB_FN (bb
, cfun
)
4995 changed
|= gimple_purge_dead_eh_edges (bb
);
4997 cleanup_tree_cfg ();
4999 if (gimple_in_ssa_p (cfun
))
5000 update_ssa (TODO_update_ssa
);
5004 /* Emit a library call to launch the children threads. */
5005 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5006 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5008 expand_task_call (new_bb
, entry_stmt
);
5009 if (gimple_in_ssa_p (cfun
))
5010 update_ssa (TODO_update_ssa_only_virtuals
);
5014 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5015 of the combined collapse > 1 loop constructs, generate code like:
5016 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5021 count3 = (adj + N32 - N31) / STEP3;
5022 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5027 count2 = (adj + N22 - N21) / STEP2;
5028 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5033 count1 = (adj + N12 - N11) / STEP1;
5034 count = count1 * count2 * count3;
5035 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5037 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5038 of the combined loop constructs, just initialize COUNTS array
5039 from the _looptemp_ clauses. */
5041 /* NOTE: It *could* be better to moosh all of the BBs together,
5042 creating one larger BB with all the computation and the unexpected
5043 jump at the end. I.e.
5045 bool zero3, zero2, zero1, zero;
5048 count3 = (N32 - N31) /[cl] STEP3;
5050 count2 = (N22 - N21) /[cl] STEP2;
5052 count1 = (N12 - N11) /[cl] STEP1;
5053 zero = zero3 || zero2 || zero1;
5054 count = count1 * count2 * count3;
5055 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5057 After all, we expect the zero=false, and thus we expect to have to
5058 evaluate all of the comparison expressions, so short-circuiting
5059 oughtn't be a win. Since the condition isn't protecting a
5060 denominator, we're not concerned about divide-by-zero, so we can
5061 fully evaluate count even if a numerator turned out to be wrong.
5063 It seems like putting this all together would create much better
5064 scheduling opportunities, and less pressure on the chip's branch
5068 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5069 basic_block
&entry_bb
, tree
*counts
,
5070 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5071 basic_block
&l2_dom_bb
)
5073 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5078 /* Collapsed loops need work for expansion into SSA form. */
5079 gcc_assert (!gimple_in_ssa_p (cfun
));
5081 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5082 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5084 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5085 isn't supposed to be handled, as the inner loop doesn't
5087 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5088 OMP_CLAUSE__LOOPTEMP_
);
5089 gcc_assert (innerc
);
5090 for (i
= 0; i
< fd
->collapse
; i
++)
5092 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5093 OMP_CLAUSE__LOOPTEMP_
);
5094 gcc_assert (innerc
);
5096 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5098 counts
[0] = NULL_TREE
;
5103 for (i
= 0; i
< fd
->collapse
; i
++)
5105 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5107 if (SSA_VAR_P (fd
->loop
.n2
)
5108 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5109 fold_convert (itype
, fd
->loops
[i
].n1
),
5110 fold_convert (itype
, fd
->loops
[i
].n2
)))
5111 == NULL_TREE
|| !integer_onep (t
)))
5114 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5115 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5116 true, GSI_SAME_STMT
);
5117 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5118 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5119 true, GSI_SAME_STMT
);
5120 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5121 NULL_TREE
, NULL_TREE
);
5122 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5123 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5124 expand_omp_regimplify_p
, NULL
, NULL
)
5125 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5126 expand_omp_regimplify_p
, NULL
, NULL
))
5128 *gsi
= gsi_for_stmt (stmt
);
5129 gimple_regimplify_operands (stmt
, gsi
);
5131 e
= split_block (entry_bb
, stmt
);
5132 if (zero_iter_bb
== NULL
)
5134 first_zero_iter
= i
;
5135 zero_iter_bb
= create_empty_bb (entry_bb
);
5137 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5138 *gsi
= gsi_after_labels (zero_iter_bb
);
5139 stmt
= gimple_build_assign (fd
->loop
.n2
,
5140 build_zero_cst (type
));
5141 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5142 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5145 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5146 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5147 e
->flags
= EDGE_TRUE_VALUE
;
5148 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5149 if (l2_dom_bb
== NULL
)
5150 l2_dom_bb
= entry_bb
;
5152 *gsi
= gsi_last_bb (entry_bb
);
5155 if (POINTER_TYPE_P (itype
))
5156 itype
= signed_type_for (itype
);
5157 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5159 t
= fold_build2 (PLUS_EXPR
, itype
,
5160 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5161 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5162 fold_convert (itype
, fd
->loops
[i
].n2
));
5163 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5164 fold_convert (itype
, fd
->loops
[i
].n1
));
5165 /* ?? We could probably use CEIL_DIV_EXPR instead of
5166 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5167 generate the same code in the end because generically we
5168 don't know that the values involved must be negative for
5170 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5171 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5172 fold_build1 (NEGATE_EXPR
, itype
, t
),
5173 fold_build1 (NEGATE_EXPR
, itype
,
5174 fold_convert (itype
,
5175 fd
->loops
[i
].step
)));
5177 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5178 fold_convert (itype
, fd
->loops
[i
].step
));
5179 t
= fold_convert (type
, t
);
5180 if (TREE_CODE (t
) == INTEGER_CST
)
5184 counts
[i
] = create_tmp_reg (type
, ".count");
5185 expand_omp_build_assign (gsi
, counts
[i
], t
);
5187 if (SSA_VAR_P (fd
->loop
.n2
))
5192 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5193 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5199 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5201 V3 = N31 + (T % count3) * STEP3;
5203 V2 = N21 + (T % count2) * STEP2;
5205 V1 = N11 + T * STEP1;
5206 if this loop doesn't have an inner loop construct combined with it.
5207 If it does have an inner loop construct combined with it and the
5208 iteration count isn't known constant, store values from counts array
5209 into its _looptemp_ temporaries instead. */
5212 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5213 tree
*counts
, gimple inner_stmt
, tree startvar
)
5216 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5218 /* If fd->loop.n2 is constant, then no propagation of the counts
5219 is needed, they are constant. */
5220 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5223 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5224 ? gimple_omp_parallel_clauses (inner_stmt
)
5225 : gimple_omp_for_clauses (inner_stmt
);
5226 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5227 isn't supposed to be handled, as the inner loop doesn't
5229 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5230 gcc_assert (innerc
);
5231 for (i
= 0; i
< fd
->collapse
; i
++)
5233 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5234 OMP_CLAUSE__LOOPTEMP_
);
5235 gcc_assert (innerc
);
5238 tree tem
= OMP_CLAUSE_DECL (innerc
);
5239 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5240 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5241 false, GSI_CONTINUE_LINKING
);
5242 gimple stmt
= gimple_build_assign (tem
, t
);
5243 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5249 tree type
= TREE_TYPE (fd
->loop
.v
);
5250 tree tem
= create_tmp_reg (type
, ".tem");
5251 gimple stmt
= gimple_build_assign (tem
, startvar
);
5252 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5254 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5256 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5258 if (POINTER_TYPE_P (vtype
))
5259 itype
= signed_type_for (vtype
);
5261 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5264 t
= fold_convert (itype
, t
);
5265 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5266 fold_convert (itype
, fd
->loops
[i
].step
));
5267 if (POINTER_TYPE_P (vtype
))
5268 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5270 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5271 t
= force_gimple_operand_gsi (gsi
, t
,
5272 DECL_P (fd
->loops
[i
].v
)
5273 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5275 GSI_CONTINUE_LINKING
);
5276 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5277 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5280 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5281 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5282 false, GSI_CONTINUE_LINKING
);
5283 stmt
= gimple_build_assign (tem
, t
);
5284 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5290 /* Helper function for expand_omp_for_*. Generate code like:
5293 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5297 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5304 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5305 basic_block body_bb
)
5307 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5309 gimple_stmt_iterator gsi
;
5315 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5317 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5319 bb
= create_empty_bb (last_bb
);
5321 add_bb_to_loop (bb
, last_bb
->loop_father
);
5322 gsi
= gsi_start_bb (bb
);
5324 if (i
< fd
->collapse
- 1)
5326 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5327 e
->probability
= REG_BR_PROB_BASE
/ 8;
5329 t
= fd
->loops
[i
+ 1].n1
;
5330 t
= force_gimple_operand_gsi (&gsi
, t
,
5331 DECL_P (fd
->loops
[i
+ 1].v
)
5332 && TREE_ADDRESSABLE (fd
->loops
[i
5335 GSI_CONTINUE_LINKING
);
5336 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5337 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5342 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5344 if (POINTER_TYPE_P (vtype
))
5345 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5347 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5348 t
= force_gimple_operand_gsi (&gsi
, t
,
5349 DECL_P (fd
->loops
[i
].v
)
5350 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5351 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5352 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5353 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5357 t
= fd
->loops
[i
].n2
;
5358 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5359 false, GSI_CONTINUE_LINKING
);
5360 tree v
= fd
->loops
[i
].v
;
5361 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5362 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5363 false, GSI_CONTINUE_LINKING
);
5364 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5365 stmt
= gimple_build_cond_empty (t
);
5366 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5367 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5368 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5371 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5379 /* A subroutine of expand_omp_for. Generate code for a parallel
5380 loop with any schedule. Given parameters:
5382 for (V = N1; V cond N2; V += STEP) BODY;
5384 where COND is "<" or ">", we generate pseudocode
5386 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5387 if (more) goto L0; else goto L3;
5394 if (V cond iend) goto L1; else goto L2;
5396 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5399 If this is a combined omp parallel loop, instead of the call to
5400 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5401 If this is gimple_omp_for_combined_p loop, then instead of assigning
5402 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5403 inner GIMPLE_OMP_FOR and V += STEP; and
5404 if (V cond iend) goto L1; else goto L2; are removed.
5406 For collapsed loops, given parameters:
5408 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5409 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5410 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5413 we generate pseudocode
5415 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5420 count3 = (adj + N32 - N31) / STEP3;
5421 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5426 count2 = (adj + N22 - N21) / STEP2;
5427 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5432 count1 = (adj + N12 - N11) / STEP1;
5433 count = count1 * count2 * count3;
5438 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5439 if (more) goto L0; else goto L3;
5443 V3 = N31 + (T % count3) * STEP3;
5445 V2 = N21 + (T % count2) * STEP2;
5447 V1 = N11 + T * STEP1;
5452 if (V < iend) goto L10; else goto L2;
5455 if (V3 cond3 N32) goto L1; else goto L11;
5459 if (V2 cond2 N22) goto L1; else goto L12;
5465 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5471 expand_omp_for_generic (struct omp_region
*region
,
5472 struct omp_for_data
*fd
,
5473 enum built_in_function start_fn
,
5474 enum built_in_function next_fn
,
5477 tree type
, istart0
, iend0
, iend
;
5478 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5479 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5480 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5481 gimple_stmt_iterator gsi
;
5483 bool in_combined_parallel
= is_combined_parallel (region
);
5484 bool broken_loop
= region
->cont
== NULL
;
5486 tree
*counts
= NULL
;
5489 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5490 gcc_assert (fd
->iter_type
== long_integer_type_node
5491 || !in_combined_parallel
);
5493 type
= TREE_TYPE (fd
->loop
.v
);
5494 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5495 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5496 TREE_ADDRESSABLE (istart0
) = 1;
5497 TREE_ADDRESSABLE (iend0
) = 1;
5499 /* See if we need to bias by LLONG_MIN. */
5500 if (fd
->iter_type
== long_long_unsigned_type_node
5501 && TREE_CODE (type
) == INTEGER_TYPE
5502 && !TYPE_UNSIGNED (type
))
5506 if (fd
->loop
.cond_code
== LT_EXPR
)
5509 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5513 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5516 if (TREE_CODE (n1
) != INTEGER_CST
5517 || TREE_CODE (n2
) != INTEGER_CST
5518 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5519 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5522 entry_bb
= region
->entry
;
5523 cont_bb
= region
->cont
;
5525 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5526 gcc_assert (broken_loop
5527 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5528 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5529 l1_bb
= single_succ (l0_bb
);
5532 l2_bb
= create_empty_bb (cont_bb
);
5533 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5534 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5538 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5539 exit_bb
= region
->exit
;
5541 gsi
= gsi_last_bb (entry_bb
);
5543 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5544 if (fd
->collapse
> 1)
5546 int first_zero_iter
= -1;
5547 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5549 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5550 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5551 zero_iter_bb
, first_zero_iter
,
5556 /* Some counts[i] vars might be uninitialized if
5557 some loop has zero iterations. But the body shouldn't
5558 be executed in that case, so just avoid uninit warnings. */
5559 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5560 if (SSA_VAR_P (counts
[i
]))
5561 TREE_NO_WARNING (counts
[i
]) = 1;
5563 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5565 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5566 gsi
= gsi_last_bb (entry_bb
);
5567 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5568 get_immediate_dominator (CDI_DOMINATORS
,
5572 if (in_combined_parallel
)
5574 /* In a combined parallel loop, emit a call to
5575 GOMP_loop_foo_next. */
5576 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5577 build_fold_addr_expr (istart0
),
5578 build_fold_addr_expr (iend0
));
5582 tree t0
, t1
, t2
, t3
, t4
;
5583 /* If this is not a combined parallel loop, emit a call to
5584 GOMP_loop_foo_start in ENTRY_BB. */
5585 t4
= build_fold_addr_expr (iend0
);
5586 t3
= build_fold_addr_expr (istart0
);
5587 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5590 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5592 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5593 OMP_CLAUSE__LOOPTEMP_
);
5594 gcc_assert (innerc
);
5595 t0
= OMP_CLAUSE_DECL (innerc
);
5596 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5597 OMP_CLAUSE__LOOPTEMP_
);
5598 gcc_assert (innerc
);
5599 t1
= OMP_CLAUSE_DECL (innerc
);
5601 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5602 && TYPE_PRECISION (TREE_TYPE (t0
))
5603 != TYPE_PRECISION (fd
->iter_type
))
5605 /* Avoid casting pointers to integer of a different size. */
5606 tree itype
= signed_type_for (type
);
5607 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5608 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5612 t1
= fold_convert (fd
->iter_type
, t1
);
5613 t0
= fold_convert (fd
->iter_type
, t0
);
5617 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5618 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5620 if (fd
->iter_type
== long_integer_type_node
)
5624 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5625 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5626 6, t0
, t1
, t2
, t
, t3
, t4
);
5629 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5630 5, t0
, t1
, t2
, t3
, t4
);
5638 /* The GOMP_loop_ull_*start functions have additional boolean
5639 argument, true for < loops and false for > loops.
5640 In Fortran, the C bool type can be different from
5641 boolean_type_node. */
5642 bfn_decl
= builtin_decl_explicit (start_fn
);
5643 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5644 t5
= build_int_cst (c_bool_type
,
5645 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5648 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5649 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5650 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5653 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5654 6, t5
, t0
, t1
, t2
, t3
, t4
);
5657 if (TREE_TYPE (t
) != boolean_type_node
)
5658 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5659 t
, build_int_cst (TREE_TYPE (t
), 0));
5660 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5661 true, GSI_SAME_STMT
);
5662 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5664 /* Remove the GIMPLE_OMP_FOR statement. */
5665 gsi_remove (&gsi
, true);
5667 /* Iteration setup for sequential loop goes in L0_BB. */
5668 tree startvar
= fd
->loop
.v
;
5669 tree endvar
= NULL_TREE
;
5671 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5673 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5674 && gimple_omp_for_kind (inner_stmt
)
5675 == GF_OMP_FOR_KIND_SIMD
);
5676 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5677 OMP_CLAUSE__LOOPTEMP_
);
5678 gcc_assert (innerc
);
5679 startvar
= OMP_CLAUSE_DECL (innerc
);
5680 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5681 OMP_CLAUSE__LOOPTEMP_
);
5682 gcc_assert (innerc
);
5683 endvar
= OMP_CLAUSE_DECL (innerc
);
5686 gsi
= gsi_start_bb (l0_bb
);
5689 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5690 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5691 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5692 t
= fold_convert (TREE_TYPE (startvar
), t
);
5693 t
= force_gimple_operand_gsi (&gsi
, t
,
5695 && TREE_ADDRESSABLE (startvar
),
5696 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5697 stmt
= gimple_build_assign (startvar
, t
);
5698 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5702 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5703 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5704 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5705 t
= fold_convert (TREE_TYPE (startvar
), t
);
5706 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5707 false, GSI_CONTINUE_LINKING
);
5710 stmt
= gimple_build_assign (endvar
, iend
);
5711 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5712 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5713 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5715 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5717 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5719 if (fd
->collapse
> 1)
5720 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5724 /* Code to control the increment and predicate for the sequential
5725 loop goes in the CONT_BB. */
5726 gsi
= gsi_last_bb (cont_bb
);
5727 stmt
= gsi_stmt (gsi
);
5728 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5729 vmain
= gimple_omp_continue_control_use (stmt
);
5730 vback
= gimple_omp_continue_control_def (stmt
);
5732 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5734 if (POINTER_TYPE_P (type
))
5735 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5737 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5738 t
= force_gimple_operand_gsi (&gsi
, t
,
5740 && TREE_ADDRESSABLE (vback
),
5741 NULL_TREE
, true, GSI_SAME_STMT
);
5742 stmt
= gimple_build_assign (vback
, t
);
5743 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5745 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5746 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5748 stmt
= gimple_build_cond_empty (t
);
5749 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5752 /* Remove GIMPLE_OMP_CONTINUE. */
5753 gsi_remove (&gsi
, true);
5755 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5756 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5758 /* Emit code to get the next parallel iteration in L2_BB. */
5759 gsi
= gsi_start_bb (l2_bb
);
5761 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5762 build_fold_addr_expr (istart0
),
5763 build_fold_addr_expr (iend0
));
5764 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5765 false, GSI_CONTINUE_LINKING
);
5766 if (TREE_TYPE (t
) != boolean_type_node
)
5767 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5768 t
, build_int_cst (TREE_TYPE (t
), 0));
5769 stmt
= gimple_build_cond_empty (t
);
5770 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5773 /* Add the loop cleanup function. */
5774 gsi
= gsi_last_bb (exit_bb
);
5775 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5776 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5777 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5778 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5780 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5781 stmt
= gimple_build_call (t
, 0);
5782 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5783 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5784 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5785 gsi_remove (&gsi
, true);
5787 /* Connect the new blocks. */
5788 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5789 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5795 e
= find_edge (cont_bb
, l3_bb
);
5796 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5798 phis
= phi_nodes (l3_bb
);
5799 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5801 gimple phi
= gsi_stmt (gsi
);
5802 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5803 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5807 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5809 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5810 e
= find_edge (cont_bb
, l1_bb
);
5811 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5816 else if (fd
->collapse
> 1)
5819 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5822 e
->flags
= EDGE_TRUE_VALUE
;
5825 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5826 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5830 e
= find_edge (cont_bb
, l2_bb
);
5831 e
->flags
= EDGE_FALLTHRU
;
5833 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5835 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5836 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5837 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5838 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5839 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5840 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5841 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5842 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5844 struct loop
*outer_loop
= alloc_loop ();
5845 outer_loop
->header
= l0_bb
;
5846 outer_loop
->latch
= l2_bb
;
5847 add_loop (outer_loop
, l0_bb
->loop_father
);
5849 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5851 struct loop
*loop
= alloc_loop ();
5852 loop
->header
= l1_bb
;
5853 /* The loop may have multiple latches. */
5854 add_loop (loop
, outer_loop
);
5860 /* A subroutine of expand_omp_for. Generate code for a parallel
5861 loop with static schedule and no specified chunk size. Given
5864 for (V = N1; V cond N2; V += STEP) BODY;
5866 where COND is "<" or ">", we generate pseudocode
5868 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5873 if ((__typeof (V)) -1 > 0 && cond is >)
5874 n = -(adj + N2 - N1) / -STEP;
5876 n = (adj + N2 - N1) / STEP;
5879 if (threadid < tt) goto L3; else goto L4;
5884 s0 = q * threadid + tt;
5887 if (s0 >= e0) goto L2; else goto L0;
5893 if (V cond e) goto L1;
5898 expand_omp_for_static_nochunk (struct omp_region
*region
,
5899 struct omp_for_data
*fd
,
5902 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5903 tree type
, itype
, vmain
, vback
;
5904 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5905 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5907 gimple_stmt_iterator gsi
;
5910 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5911 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5912 bool broken_loop
= region
->cont
== NULL
;
5913 tree
*counts
= NULL
;
5916 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5917 if (POINTER_TYPE_P (type
))
5918 itype
= signed_type_for (type
);
5920 entry_bb
= region
->entry
;
5921 cont_bb
= region
->cont
;
5922 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5923 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5924 gcc_assert (broken_loop
5925 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5926 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5927 body_bb
= single_succ (seq_start_bb
);
5930 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5931 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5933 exit_bb
= region
->exit
;
5935 /* Iteration space partitioning goes in ENTRY_BB. */
5936 gsi
= gsi_last_bb (entry_bb
);
5937 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5939 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5941 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5942 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5945 if (fd
->collapse
> 1)
5947 int first_zero_iter
= -1;
5948 basic_block l2_dom_bb
= NULL
;
5950 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5951 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5952 fin_bb
, first_zero_iter
,
5956 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5957 t
= integer_one_node
;
5959 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5960 fold_convert (type
, fd
->loop
.n1
),
5961 fold_convert (type
, fd
->loop
.n2
));
5962 if (fd
->collapse
== 1
5963 && TYPE_UNSIGNED (type
)
5964 && (t
== NULL_TREE
|| !integer_onep (t
)))
5966 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5967 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5968 true, GSI_SAME_STMT
);
5969 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5970 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5971 true, GSI_SAME_STMT
);
5972 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5973 NULL_TREE
, NULL_TREE
);
5974 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5975 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5976 expand_omp_regimplify_p
, NULL
, NULL
)
5977 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5978 expand_omp_regimplify_p
, NULL
, NULL
))
5980 gsi
= gsi_for_stmt (stmt
);
5981 gimple_regimplify_operands (stmt
, &gsi
);
5983 ep
= split_block (entry_bb
, stmt
);
5984 ep
->flags
= EDGE_TRUE_VALUE
;
5985 entry_bb
= ep
->dest
;
5986 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5987 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5988 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5989 if (gimple_in_ssa_p (cfun
))
5991 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5992 for (gsi
= gsi_start_phis (fin_bb
);
5993 !gsi_end_p (gsi
); gsi_next (&gsi
))
5995 gimple phi
= gsi_stmt (gsi
);
5996 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5997 ep
, UNKNOWN_LOCATION
);
6000 gsi
= gsi_last_bb (entry_bb
);
6003 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6004 t
= fold_convert (itype
, t
);
6005 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6006 true, GSI_SAME_STMT
);
6008 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6009 t
= fold_convert (itype
, t
);
6010 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6011 true, GSI_SAME_STMT
);
6015 step
= fd
->loop
.step
;
6016 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6018 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6019 OMP_CLAUSE__LOOPTEMP_
);
6020 gcc_assert (innerc
);
6021 n1
= OMP_CLAUSE_DECL (innerc
);
6022 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6023 OMP_CLAUSE__LOOPTEMP_
);
6024 gcc_assert (innerc
);
6025 n2
= OMP_CLAUSE_DECL (innerc
);
6027 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6028 true, NULL_TREE
, true, GSI_SAME_STMT
);
6029 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6030 true, NULL_TREE
, true, GSI_SAME_STMT
);
6031 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6032 true, NULL_TREE
, true, GSI_SAME_STMT
);
6034 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6035 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6036 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6037 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6038 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6039 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6040 fold_build1 (NEGATE_EXPR
, itype
, t
),
6041 fold_build1 (NEGATE_EXPR
, itype
, step
));
6043 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6044 t
= fold_convert (itype
, t
);
6045 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6047 q
= create_tmp_reg (itype
, "q");
6048 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6049 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6050 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6052 tt
= create_tmp_reg (itype
, "tt");
6053 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6054 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6055 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6057 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6058 stmt
= gimple_build_cond_empty (t
);
6059 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6061 second_bb
= split_block (entry_bb
, stmt
)->dest
;
6062 gsi
= gsi_last_bb (second_bb
);
6063 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6065 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6067 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6068 build_int_cst (itype
, 1));
6069 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6071 third_bb
= split_block (second_bb
, stmt
)->dest
;
6072 gsi
= gsi_last_bb (third_bb
);
6073 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6075 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6076 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6077 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6079 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6080 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6082 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6083 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6085 /* Remove the GIMPLE_OMP_FOR statement. */
6086 gsi_remove (&gsi
, true);
6088 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6089 gsi
= gsi_start_bb (seq_start_bb
);
6091 tree startvar
= fd
->loop
.v
;
6092 tree endvar
= NULL_TREE
;
6094 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6096 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6097 ? gimple_omp_parallel_clauses (inner_stmt
)
6098 : gimple_omp_for_clauses (inner_stmt
);
6099 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6100 gcc_assert (innerc
);
6101 startvar
= OMP_CLAUSE_DECL (innerc
);
6102 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6103 OMP_CLAUSE__LOOPTEMP_
);
6104 gcc_assert (innerc
);
6105 endvar
= OMP_CLAUSE_DECL (innerc
);
6107 t
= fold_convert (itype
, s0
);
6108 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6109 if (POINTER_TYPE_P (type
))
6110 t
= fold_build_pointer_plus (n1
, t
);
6112 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6113 t
= fold_convert (TREE_TYPE (startvar
), t
);
6114 t
= force_gimple_operand_gsi (&gsi
, t
,
6116 && TREE_ADDRESSABLE (startvar
),
6117 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6118 stmt
= gimple_build_assign (startvar
, t
);
6119 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6121 t
= fold_convert (itype
, e0
);
6122 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6123 if (POINTER_TYPE_P (type
))
6124 t
= fold_build_pointer_plus (n1
, t
);
6126 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6127 t
= fold_convert (TREE_TYPE (startvar
), t
);
6128 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6129 false, GSI_CONTINUE_LINKING
);
6132 stmt
= gimple_build_assign (endvar
, e
);
6133 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6134 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6135 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6137 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6139 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6141 if (fd
->collapse
> 1)
6142 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6146 /* The code controlling the sequential loop replaces the
6147 GIMPLE_OMP_CONTINUE. */
6148 gsi
= gsi_last_bb (cont_bb
);
6149 stmt
= gsi_stmt (gsi
);
6150 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6151 vmain
= gimple_omp_continue_control_use (stmt
);
6152 vback
= gimple_omp_continue_control_def (stmt
);
6154 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6156 if (POINTER_TYPE_P (type
))
6157 t
= fold_build_pointer_plus (vmain
, step
);
6159 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6160 t
= force_gimple_operand_gsi (&gsi
, t
,
6162 && TREE_ADDRESSABLE (vback
),
6163 NULL_TREE
, true, GSI_SAME_STMT
);
6164 stmt
= gimple_build_assign (vback
, t
);
6165 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6167 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6168 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6170 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6173 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6174 gsi_remove (&gsi
, true);
6176 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6177 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6180 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6181 gsi
= gsi_last_bb (exit_bb
);
6182 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6184 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6185 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6187 gsi_remove (&gsi
, true);
6189 /* Connect all the blocks. */
6190 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6191 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6192 ep
= find_edge (entry_bb
, second_bb
);
6193 ep
->flags
= EDGE_TRUE_VALUE
;
6194 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6195 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6196 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6200 ep
= find_edge (cont_bb
, body_bb
);
6201 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6206 else if (fd
->collapse
> 1)
6209 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6212 ep
->flags
= EDGE_TRUE_VALUE
;
6213 find_edge (cont_bb
, fin_bb
)->flags
6214 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6217 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6218 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6219 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6221 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6222 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6223 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6224 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6226 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6228 struct loop
*loop
= alloc_loop ();
6229 loop
->header
= body_bb
;
6230 if (collapse_bb
== NULL
)
6231 loop
->latch
= cont_bb
;
6232 add_loop (loop
, body_bb
->loop_father
);
6237 /* A subroutine of expand_omp_for. Generate code for a parallel
6238 loop with static schedule and a specified chunk size. Given
6241 for (V = N1; V cond N2; V += STEP) BODY;
6243 where COND is "<" or ">", we generate pseudocode
6245 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6250 if ((__typeof (V)) -1 > 0 && cond is >)
6251 n = -(adj + N2 - N1) / -STEP;
6253 n = (adj + N2 - N1) / STEP;
6255 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6256 here so that V is defined
6257 if the loop is not entered
6259 s0 = (trip * nthreads + threadid) * CHUNK;
6260 e0 = min(s0 + CHUNK, n);
6261 if (s0 < n) goto L1; else goto L4;
6268 if (V cond e) goto L2; else goto L3;
6276 expand_omp_for_static_chunk (struct omp_region
*region
,
6277 struct omp_for_data
*fd
, gimple inner_stmt
)
6279 tree n
, s0
, e0
, e
, t
;
6280 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6281 tree type
, itype
, v_main
, v_back
, v_extra
;
6282 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6283 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6284 gimple_stmt_iterator si
;
6287 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6288 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6289 bool broken_loop
= region
->cont
== NULL
;
6290 tree
*counts
= NULL
;
6293 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6294 if (POINTER_TYPE_P (type
))
6295 itype
= signed_type_for (type
);
6297 entry_bb
= region
->entry
;
6298 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6300 iter_part_bb
= se
->dest
;
6301 cont_bb
= region
->cont
;
6302 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6303 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6304 gcc_assert (broken_loop
6305 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6306 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6307 body_bb
= single_succ (seq_start_bb
);
6310 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6311 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6312 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6314 exit_bb
= region
->exit
;
6316 /* Trip and adjustment setup goes in ENTRY_BB. */
6317 si
= gsi_last_bb (entry_bb
);
6318 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6320 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6322 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6323 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6326 if (fd
->collapse
> 1)
6328 int first_zero_iter
= -1;
6329 basic_block l2_dom_bb
= NULL
;
6331 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6332 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6333 fin_bb
, first_zero_iter
,
6337 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6338 t
= integer_one_node
;
6340 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6341 fold_convert (type
, fd
->loop
.n1
),
6342 fold_convert (type
, fd
->loop
.n2
));
6343 if (fd
->collapse
== 1
6344 && TYPE_UNSIGNED (type
)
6345 && (t
== NULL_TREE
|| !integer_onep (t
)))
6347 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6348 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6349 true, GSI_SAME_STMT
);
6350 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6351 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6352 true, GSI_SAME_STMT
);
6353 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6354 NULL_TREE
, NULL_TREE
);
6355 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6356 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6357 expand_omp_regimplify_p
, NULL
, NULL
)
6358 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6359 expand_omp_regimplify_p
, NULL
, NULL
))
6361 si
= gsi_for_stmt (stmt
);
6362 gimple_regimplify_operands (stmt
, &si
);
6364 se
= split_block (entry_bb
, stmt
);
6365 se
->flags
= EDGE_TRUE_VALUE
;
6366 entry_bb
= se
->dest
;
6367 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6368 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6369 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6370 if (gimple_in_ssa_p (cfun
))
6372 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6373 for (si
= gsi_start_phis (fin_bb
);
6374 !gsi_end_p (si
); gsi_next (&si
))
6376 gimple phi
= gsi_stmt (si
);
6377 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6378 se
, UNKNOWN_LOCATION
);
6381 si
= gsi_last_bb (entry_bb
);
6384 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6385 t
= fold_convert (itype
, t
);
6386 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6387 true, GSI_SAME_STMT
);
6389 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6390 t
= fold_convert (itype
, t
);
6391 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6392 true, GSI_SAME_STMT
);
6396 step
= fd
->loop
.step
;
6397 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6399 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6400 OMP_CLAUSE__LOOPTEMP_
);
6401 gcc_assert (innerc
);
6402 n1
= OMP_CLAUSE_DECL (innerc
);
6403 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6404 OMP_CLAUSE__LOOPTEMP_
);
6405 gcc_assert (innerc
);
6406 n2
= OMP_CLAUSE_DECL (innerc
);
6408 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6409 true, NULL_TREE
, true, GSI_SAME_STMT
);
6410 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6411 true, NULL_TREE
, true, GSI_SAME_STMT
);
6412 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6413 true, NULL_TREE
, true, GSI_SAME_STMT
);
6415 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6416 true, NULL_TREE
, true, GSI_SAME_STMT
);
6418 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6419 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6420 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6421 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6422 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6423 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6424 fold_build1 (NEGATE_EXPR
, itype
, t
),
6425 fold_build1 (NEGATE_EXPR
, itype
, step
));
6427 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6428 t
= fold_convert (itype
, t
);
6429 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6430 true, GSI_SAME_STMT
);
6432 trip_var
= create_tmp_reg (itype
, ".trip");
6433 if (gimple_in_ssa_p (cfun
))
6435 trip_init
= make_ssa_name (trip_var
, NULL
);
6436 trip_main
= make_ssa_name (trip_var
, NULL
);
6437 trip_back
= make_ssa_name (trip_var
, NULL
);
6441 trip_init
= trip_var
;
6442 trip_main
= trip_var
;
6443 trip_back
= trip_var
;
6446 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6447 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6449 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6450 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6451 if (POINTER_TYPE_P (type
))
6452 t
= fold_build_pointer_plus (n1
, t
);
6454 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6455 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6456 true, GSI_SAME_STMT
);
6458 /* Remove the GIMPLE_OMP_FOR. */
6459 gsi_remove (&si
, true);
6461 /* Iteration space partitioning goes in ITER_PART_BB. */
6462 si
= gsi_last_bb (iter_part_bb
);
6464 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6465 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6466 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6467 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6468 false, GSI_CONTINUE_LINKING
);
6470 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6471 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6472 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6473 false, GSI_CONTINUE_LINKING
);
6475 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6476 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6478 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6479 si
= gsi_start_bb (seq_start_bb
);
6481 tree startvar
= fd
->loop
.v
;
6482 tree endvar
= NULL_TREE
;
6484 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6486 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6487 ? gimple_omp_parallel_clauses (inner_stmt
)
6488 : gimple_omp_for_clauses (inner_stmt
);
6489 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6490 gcc_assert (innerc
);
6491 startvar
= OMP_CLAUSE_DECL (innerc
);
6492 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6493 OMP_CLAUSE__LOOPTEMP_
);
6494 gcc_assert (innerc
);
6495 endvar
= OMP_CLAUSE_DECL (innerc
);
6498 t
= fold_convert (itype
, s0
);
6499 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6500 if (POINTER_TYPE_P (type
))
6501 t
= fold_build_pointer_plus (n1
, t
);
6503 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6504 t
= fold_convert (TREE_TYPE (startvar
), t
);
6505 t
= force_gimple_operand_gsi (&si
, t
,
6507 && TREE_ADDRESSABLE (startvar
),
6508 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6509 stmt
= gimple_build_assign (startvar
, t
);
6510 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6512 t
= fold_convert (itype
, e0
);
6513 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6514 if (POINTER_TYPE_P (type
))
6515 t
= fold_build_pointer_plus (n1
, t
);
6517 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6518 t
= fold_convert (TREE_TYPE (startvar
), t
);
6519 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6520 false, GSI_CONTINUE_LINKING
);
6523 stmt
= gimple_build_assign (endvar
, e
);
6524 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6525 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6526 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6528 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6530 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6532 if (fd
->collapse
> 1)
6533 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6537 /* The code controlling the sequential loop goes in CONT_BB,
6538 replacing the GIMPLE_OMP_CONTINUE. */
6539 si
= gsi_last_bb (cont_bb
);
6540 stmt
= gsi_stmt (si
);
6541 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6542 v_main
= gimple_omp_continue_control_use (stmt
);
6543 v_back
= gimple_omp_continue_control_def (stmt
);
6545 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6547 if (POINTER_TYPE_P (type
))
6548 t
= fold_build_pointer_plus (v_main
, step
);
6550 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6551 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6552 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6553 true, GSI_SAME_STMT
);
6554 stmt
= gimple_build_assign (v_back
, t
);
6555 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6557 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6558 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6560 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6563 /* Remove GIMPLE_OMP_CONTINUE. */
6564 gsi_remove (&si
, true);
6566 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6567 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6569 /* Trip update code goes into TRIP_UPDATE_BB. */
6570 si
= gsi_start_bb (trip_update_bb
);
6572 t
= build_int_cst (itype
, 1);
6573 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6574 stmt
= gimple_build_assign (trip_back
, t
);
6575 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6578 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6579 si
= gsi_last_bb (exit_bb
);
6580 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6582 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6583 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6585 gsi_remove (&si
, true);
6587 /* Connect the new blocks. */
6588 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6589 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6593 se
= find_edge (cont_bb
, body_bb
);
6594 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6599 else if (fd
->collapse
> 1)
6602 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6605 se
->flags
= EDGE_TRUE_VALUE
;
6606 find_edge (cont_bb
, trip_update_bb
)->flags
6607 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6609 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6612 if (gimple_in_ssa_p (cfun
))
6614 gimple_stmt_iterator psi
;
6617 edge_var_map_vector
*head
;
6621 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6623 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6624 remove arguments of the phi nodes in fin_bb. We need to create
6625 appropriate phi nodes in iter_part_bb instead. */
6626 se
= single_pred_edge (fin_bb
);
6627 re
= single_succ_edge (trip_update_bb
);
6628 head
= redirect_edge_var_map_vector (re
);
6629 ene
= single_succ_edge (entry_bb
);
6631 psi
= gsi_start_phis (fin_bb
);
6632 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6633 gsi_next (&psi
), ++i
)
6636 source_location locus
;
6638 phi
= gsi_stmt (psi
);
6639 t
= gimple_phi_result (phi
);
6640 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6641 nphi
= create_phi_node (t
, iter_part_bb
);
6643 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6644 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6646 /* A special case -- fd->loop.v is not yet computed in
6647 iter_part_bb, we need to use v_extra instead. */
6648 if (t
== fd
->loop
.v
)
6650 add_phi_arg (nphi
, t
, ene
, locus
);
6651 locus
= redirect_edge_var_map_location (vm
);
6652 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6654 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6655 redirect_edge_var_map_clear (re
);
6658 psi
= gsi_start_phis (fin_bb
);
6659 if (gsi_end_p (psi
))
6661 remove_phi_node (&psi
, false);
6664 /* Make phi node for trip. */
6665 phi
= create_phi_node (trip_main
, iter_part_bb
);
6666 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6668 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6673 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6674 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6675 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6676 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6677 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6678 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6679 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6680 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6681 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6685 struct loop
*trip_loop
= alloc_loop ();
6686 trip_loop
->header
= iter_part_bb
;
6687 trip_loop
->latch
= trip_update_bb
;
6688 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6690 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6692 struct loop
*loop
= alloc_loop ();
6693 loop
->header
= body_bb
;
6694 if (collapse_bb
== NULL
)
6695 loop
->latch
= cont_bb
;
6696 add_loop (loop
, trip_loop
);
6702 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6703 loop. Given parameters:
6705 for (V = N1; V cond N2; V += STEP) BODY;
6707 where COND is "<" or ">", we generate pseudocode
6715 if (V cond N2) goto L0; else goto L2;
6718 For collapsed loops, given parameters:
6720 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6721 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6722 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6725 we generate pseudocode
6731 count3 = (adj + N32 - N31) / STEP3;
6736 count2 = (adj + N22 - N21) / STEP2;
6741 count1 = (adj + N12 - N11) / STEP1;
6742 count = count1 * count2 * count3;
6752 V2 += (V3 cond3 N32) ? 0 : STEP2;
6753 V3 = (V3 cond3 N32) ? V3 : N31;
6754 V1 += (V2 cond2 N22) ? 0 : STEP1;
6755 V2 = (V2 cond2 N22) ? V2 : N21;
6757 if (V < count) goto L0; else goto L2;
6763 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6766 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6767 gimple_stmt_iterator gsi
;
6769 bool broken_loop
= region
->cont
== NULL
;
6771 tree
*counts
= NULL
;
6773 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6774 OMP_CLAUSE_SAFELEN
);
6775 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6776 OMP_CLAUSE__SIMDUID_
);
6779 type
= TREE_TYPE (fd
->loop
.v
);
6780 entry_bb
= region
->entry
;
6781 cont_bb
= region
->cont
;
6782 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6783 gcc_assert (broken_loop
6784 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6785 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6788 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6789 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6790 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6791 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6795 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6796 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6797 l2_bb
= single_succ (l1_bb
);
6799 exit_bb
= region
->exit
;
6802 gsi
= gsi_last_bb (entry_bb
);
6804 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6805 /* Not needed in SSA form right now. */
6806 gcc_assert (!gimple_in_ssa_p (cfun
));
6807 if (fd
->collapse
> 1)
6809 int first_zero_iter
= -1;
6810 basic_block zero_iter_bb
= l2_bb
;
6812 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6813 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6814 zero_iter_bb
, first_zero_iter
,
6817 if (l2_dom_bb
== NULL
)
6822 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6824 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6825 OMP_CLAUSE__LOOPTEMP_
);
6826 gcc_assert (innerc
);
6827 n1
= OMP_CLAUSE_DECL (innerc
);
6828 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6829 OMP_CLAUSE__LOOPTEMP_
);
6830 gcc_assert (innerc
);
6831 n2
= OMP_CLAUSE_DECL (innerc
);
6832 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6833 fold_convert (type
, n1
));
6834 if (fd
->collapse
> 1)
6837 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6843 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6844 fold_convert (type
, fd
->loop
.n1
));
6845 if (fd
->collapse
> 1)
6846 for (i
= 0; i
< fd
->collapse
; i
++)
6848 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6849 if (POINTER_TYPE_P (itype
))
6850 itype
= signed_type_for (itype
);
6851 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6852 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6856 /* Remove the GIMPLE_OMP_FOR statement. */
6857 gsi_remove (&gsi
, true);
6861 /* Code to control the increment goes in the CONT_BB. */
6862 gsi
= gsi_last_bb (cont_bb
);
6863 stmt
= gsi_stmt (gsi
);
6864 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6866 if (POINTER_TYPE_P (type
))
6867 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6869 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6870 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6872 if (fd
->collapse
> 1)
6874 i
= fd
->collapse
- 1;
6875 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6877 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6878 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6882 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6884 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6887 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6889 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6891 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6892 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6893 if (POINTER_TYPE_P (itype2
))
6894 itype2
= signed_type_for (itype2
);
6895 t
= build3 (COND_EXPR
, itype2
,
6896 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6898 fold_convert (itype
, fd
->loops
[i
].n2
)),
6899 build_int_cst (itype2
, 0),
6900 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6901 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6902 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6904 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6905 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6907 t
= build3 (COND_EXPR
, itype
,
6908 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6910 fold_convert (itype
, fd
->loops
[i
].n2
)),
6912 fold_convert (itype
, fd
->loops
[i
].n1
));
6913 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6917 /* Remove GIMPLE_OMP_CONTINUE. */
6918 gsi_remove (&gsi
, true);
6921 /* Emit the condition in L1_BB. */
6922 gsi
= gsi_start_bb (l1_bb
);
6924 t
= fold_convert (type
, n2
);
6925 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6926 false, GSI_CONTINUE_LINKING
);
6927 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6928 stmt
= gimple_build_cond_empty (t
);
6929 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6930 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6932 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6935 gsi
= gsi_for_stmt (stmt
);
6936 gimple_regimplify_operands (stmt
, &gsi
);
6939 /* Remove GIMPLE_OMP_RETURN. */
6940 gsi
= gsi_last_bb (exit_bb
);
6941 gsi_remove (&gsi
, true);
6943 /* Connect the new blocks. */
6944 remove_edge (FALLTHRU_EDGE (entry_bb
));
6948 remove_edge (BRANCH_EDGE (entry_bb
));
6949 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6951 e
= BRANCH_EDGE (l1_bb
);
6952 ne
= FALLTHRU_EDGE (l1_bb
);
6953 e
->flags
= EDGE_TRUE_VALUE
;
6957 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6959 ne
= single_succ_edge (l1_bb
);
6960 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6963 ne
->flags
= EDGE_FALSE_VALUE
;
6964 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6965 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6967 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6968 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6969 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6973 struct loop
*loop
= alloc_loop ();
6974 loop
->header
= l1_bb
;
6975 loop
->latch
= cont_bb
;
6976 add_loop (loop
, l1_bb
->loop_father
);
6977 if (safelen
== NULL_TREE
)
6978 loop
->safelen
= INT_MAX
;
6981 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6982 if (TREE_CODE (safelen
) != INTEGER_CST
)
6984 else if (!tree_fits_uhwi_p (safelen
)
6985 || tree_to_uhwi (safelen
) > INT_MAX
)
6986 loop
->safelen
= INT_MAX
;
6988 loop
->safelen
= tree_to_uhwi (safelen
);
6989 if (loop
->safelen
== 1)
6994 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6995 cfun
->has_simduid_loops
= true;
6997 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6999 if ((flag_tree_loop_vectorize
7000 || (!global_options_set
.x_flag_tree_loop_vectorize
7001 && !global_options_set
.x_flag_tree_vectorize
))
7002 && flag_tree_loop_optimize
7003 && loop
->safelen
> 1)
7005 loop
->force_vect
= true;
7006 cfun
->has_force_vect_loops
= true;
7012 /* Expand the OpenMP loop defined by REGION. */
7015 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7017 struct omp_for_data fd
;
7018 struct omp_for_data_loop
*loops
;
7021 = (struct omp_for_data_loop
*)
7022 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7023 * sizeof (struct omp_for_data_loop
));
7024 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7025 region
->sched_kind
= fd
.sched_kind
;
7027 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7028 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7029 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7032 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7033 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7034 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7037 /* If there isn't a continue then this is a degerate case where
7038 the introduction of abnormal edges during lowering will prevent
7039 original loops from being detected. Fix that up. */
7040 loops_state_set (LOOPS_NEED_FIXUP
);
7042 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_KIND_SIMD
)
7043 expand_omp_simd (region
, &fd
);
7044 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7045 && !fd
.have_ordered
)
7047 if (fd
.chunk_size
== NULL
)
7048 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7050 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7054 int fn_index
, start_ix
, next_ix
;
7056 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7057 == GF_OMP_FOR_KIND_FOR
);
7058 if (fd
.chunk_size
== NULL
7059 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7060 fd
.chunk_size
= integer_zero_node
;
7061 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7062 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7063 ? 3 : fd
.sched_kind
;
7064 fn_index
+= fd
.have_ordered
* 4;
7065 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7066 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7067 if (fd
.iter_type
== long_long_unsigned_type_node
)
7069 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7070 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7071 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7072 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7074 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7075 (enum built_in_function
) next_ix
, inner_stmt
);
7078 if (gimple_in_ssa_p (cfun
))
7079 update_ssa (TODO_update_ssa_only_virtuals
);
7083 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7085 v = GOMP_sections_start (n);
7102 v = GOMP_sections_next ();
7107 If this is a combined parallel sections, replace the call to
7108 GOMP_sections_start with call to GOMP_sections_next. */
7111 expand_omp_sections (struct omp_region
*region
)
7113 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7115 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7116 gimple_stmt_iterator si
, switch_si
;
7117 gimple sections_stmt
, stmt
, cont
;
7120 struct omp_region
*inner
;
7122 bool exit_reachable
= region
->cont
!= NULL
;
7124 gcc_assert (region
->exit
!= NULL
);
7125 entry_bb
= region
->entry
;
7126 l0_bb
= single_succ (entry_bb
);
7127 l1_bb
= region
->cont
;
7128 l2_bb
= region
->exit
;
7129 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7130 l2
= gimple_block_label (l2_bb
);
7133 /* This can happen if there are reductions. */
7134 len
= EDGE_COUNT (l0_bb
->succs
);
7135 gcc_assert (len
> 0);
7136 e
= EDGE_SUCC (l0_bb
, len
- 1);
7137 si
= gsi_last_bb (e
->dest
);
7140 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7141 l2
= gimple_block_label (e
->dest
);
7143 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7145 si
= gsi_last_bb (e
->dest
);
7147 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7149 l2
= gimple_block_label (e
->dest
);
7155 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7157 default_bb
= create_empty_bb (l0_bb
);
7159 /* We will build a switch() with enough cases for all the
7160 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7161 and a default case to abort if something goes wrong. */
7162 len
= EDGE_COUNT (l0_bb
->succs
);
7164 /* Use vec::quick_push on label_vec throughout, since we know the size
7166 auto_vec
<tree
> label_vec (len
);
7168 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7169 GIMPLE_OMP_SECTIONS statement. */
7170 si
= gsi_last_bb (entry_bb
);
7171 sections_stmt
= gsi_stmt (si
);
7172 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7173 vin
= gimple_omp_sections_control (sections_stmt
);
7174 if (!is_combined_parallel (region
))
7176 /* If we are not inside a combined parallel+sections region,
7177 call GOMP_sections_start. */
7178 t
= build_int_cst (unsigned_type_node
, len
- 1);
7179 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7180 stmt
= gimple_build_call (u
, 1, t
);
7184 /* Otherwise, call GOMP_sections_next. */
7185 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7186 stmt
= gimple_build_call (u
, 0);
7188 gimple_call_set_lhs (stmt
, vin
);
7189 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7190 gsi_remove (&si
, true);
7192 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7194 switch_si
= gsi_last_bb (l0_bb
);
7195 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7198 cont
= last_stmt (l1_bb
);
7199 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7200 vmain
= gimple_omp_continue_control_use (cont
);
7201 vnext
= gimple_omp_continue_control_def (cont
);
7209 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7210 label_vec
.quick_push (t
);
7213 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7214 for (inner
= region
->inner
, casei
= 1;
7216 inner
= inner
->next
, i
++, casei
++)
7218 basic_block s_entry_bb
, s_exit_bb
;
7220 /* Skip optional reduction region. */
7221 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7228 s_entry_bb
= inner
->entry
;
7229 s_exit_bb
= inner
->exit
;
7231 t
= gimple_block_label (s_entry_bb
);
7232 u
= build_int_cst (unsigned_type_node
, casei
);
7233 u
= build_case_label (u
, NULL
, t
);
7234 label_vec
.quick_push (u
);
7236 si
= gsi_last_bb (s_entry_bb
);
7237 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7238 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7239 gsi_remove (&si
, true);
7240 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7242 if (s_exit_bb
== NULL
)
7245 si
= gsi_last_bb (s_exit_bb
);
7246 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7247 gsi_remove (&si
, true);
7249 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7252 /* Error handling code goes in DEFAULT_BB. */
7253 t
= gimple_block_label (default_bb
);
7254 u
= build_case_label (NULL
, NULL
, t
);
7255 make_edge (l0_bb
, default_bb
, 0);
7257 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7259 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7260 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7261 gsi_remove (&switch_si
, true);
7263 si
= gsi_start_bb (default_bb
);
7264 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7265 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7271 /* Code to get the next section goes in L1_BB. */
7272 si
= gsi_last_bb (l1_bb
);
7273 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7275 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7276 stmt
= gimple_build_call (bfn_decl
, 0);
7277 gimple_call_set_lhs (stmt
, vnext
);
7278 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7279 gsi_remove (&si
, true);
7281 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7284 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7285 si
= gsi_last_bb (l2_bb
);
7286 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7287 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7288 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7289 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7291 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7292 stmt
= gimple_build_call (t
, 0);
7293 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7294 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7295 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7296 gsi_remove (&si
, true);
7298 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7302 /* Expand code for an OpenMP single directive. We've already expanded
7303 much of the code, here we simply place the GOMP_barrier call. */
7306 expand_omp_single (struct omp_region
*region
)
7308 basic_block entry_bb
, exit_bb
;
7309 gimple_stmt_iterator si
;
7311 entry_bb
= region
->entry
;
7312 exit_bb
= region
->exit
;
7314 si
= gsi_last_bb (entry_bb
);
7315 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7316 gsi_remove (&si
, true);
7317 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7319 si
= gsi_last_bb (exit_bb
);
7320 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7322 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7323 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7325 gsi_remove (&si
, true);
7326 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7330 /* Generic expansion for OpenMP synchronization directives: master,
7331 ordered and critical. All we need to do here is remove the entry
7332 and exit markers for REGION. */
7335 expand_omp_synch (struct omp_region
*region
)
7337 basic_block entry_bb
, exit_bb
;
7338 gimple_stmt_iterator si
;
7340 entry_bb
= region
->entry
;
7341 exit_bb
= region
->exit
;
7343 si
= gsi_last_bb (entry_bb
);
7344 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7345 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7346 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7347 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7348 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7349 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7350 gsi_remove (&si
, true);
7351 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7355 si
= gsi_last_bb (exit_bb
);
7356 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7357 gsi_remove (&si
, true);
7358 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7362 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7363 operation as a normal volatile load. */
7366 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7367 tree loaded_val
, int index
)
7369 enum built_in_function tmpbase
;
7370 gimple_stmt_iterator gsi
;
7371 basic_block store_bb
;
7374 tree decl
, call
, type
, itype
;
7376 gsi
= gsi_last_bb (load_bb
);
7377 stmt
= gsi_stmt (gsi
);
7378 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7379 loc
= gimple_location (stmt
);
7381 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7382 is smaller than word size, then expand_atomic_load assumes that the load
7383 is atomic. We could avoid the builtin entirely in this case. */
7385 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7386 decl
= builtin_decl_explicit (tmpbase
);
7387 if (decl
== NULL_TREE
)
7390 type
= TREE_TYPE (loaded_val
);
7391 itype
= TREE_TYPE (TREE_TYPE (decl
));
7393 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7394 build_int_cst (NULL
,
7395 gimple_omp_atomic_seq_cst_p (stmt
)
7397 : MEMMODEL_RELAXED
));
7398 if (!useless_type_conversion_p (type
, itype
))
7399 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7400 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7402 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7403 gsi_remove (&gsi
, true);
7405 store_bb
= single_succ (load_bb
);
7406 gsi
= gsi_last_bb (store_bb
);
7407 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7408 gsi_remove (&gsi
, true);
7410 if (gimple_in_ssa_p (cfun
))
7411 update_ssa (TODO_update_ssa_no_phi
);
7416 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7417 operation as a normal volatile store. */
7420 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7421 tree loaded_val
, tree stored_val
, int index
)
7423 enum built_in_function tmpbase
;
7424 gimple_stmt_iterator gsi
;
7425 basic_block store_bb
= single_succ (load_bb
);
7428 tree decl
, call
, type
, itype
;
7429 enum machine_mode imode
;
7432 gsi
= gsi_last_bb (load_bb
);
7433 stmt
= gsi_stmt (gsi
);
7434 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7436 /* If the load value is needed, then this isn't a store but an exchange. */
7437 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7439 gsi
= gsi_last_bb (store_bb
);
7440 stmt
= gsi_stmt (gsi
);
7441 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7442 loc
= gimple_location (stmt
);
7444 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7445 is smaller than word size, then expand_atomic_store assumes that the store
7446 is atomic. We could avoid the builtin entirely in this case. */
7448 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7449 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7450 decl
= builtin_decl_explicit (tmpbase
);
7451 if (decl
== NULL_TREE
)
7454 type
= TREE_TYPE (stored_val
);
7456 /* Dig out the type of the function's second argument. */
7457 itype
= TREE_TYPE (decl
);
7458 itype
= TYPE_ARG_TYPES (itype
);
7459 itype
= TREE_CHAIN (itype
);
7460 itype
= TREE_VALUE (itype
);
7461 imode
= TYPE_MODE (itype
);
7463 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7466 if (!useless_type_conversion_p (itype
, type
))
7467 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7468 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7469 build_int_cst (NULL
,
7470 gimple_omp_atomic_seq_cst_p (stmt
)
7472 : MEMMODEL_RELAXED
));
7475 if (!useless_type_conversion_p (type
, itype
))
7476 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7477 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7480 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7481 gsi_remove (&gsi
, true);
7483 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7484 gsi
= gsi_last_bb (load_bb
);
7485 gsi_remove (&gsi
, true);
7487 if (gimple_in_ssa_p (cfun
))
7488 update_ssa (TODO_update_ssa_no_phi
);
7493 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7494 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7495 size of the data type, and thus usable to find the index of the builtin
7496 decl. Returns false if the expression is not of the proper form. */
7499 expand_omp_atomic_fetch_op (basic_block load_bb
,
7500 tree addr
, tree loaded_val
,
7501 tree stored_val
, int index
)
7503 enum built_in_function oldbase
, newbase
, tmpbase
;
7504 tree decl
, itype
, call
;
7506 basic_block store_bb
= single_succ (load_bb
);
7507 gimple_stmt_iterator gsi
;
7510 enum tree_code code
;
7511 bool need_old
, need_new
;
7512 enum machine_mode imode
;
7515 /* We expect to find the following sequences:
7518 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7521 val = tmp OP something; (or: something OP tmp)
7522 GIMPLE_OMP_STORE (val)
7524 ???FIXME: Allow a more flexible sequence.
7525 Perhaps use data flow to pick the statements.
7529 gsi
= gsi_after_labels (store_bb
);
7530 stmt
= gsi_stmt (gsi
);
7531 loc
= gimple_location (stmt
);
7532 if (!is_gimple_assign (stmt
))
7535 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7537 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7538 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7539 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7540 gcc_checking_assert (!need_old
|| !need_new
);
7542 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7545 /* Check for one of the supported fetch-op operations. */
7546 code
= gimple_assign_rhs_code (stmt
);
7550 case POINTER_PLUS_EXPR
:
7551 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7552 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7555 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7556 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7559 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7560 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7563 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7564 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7567 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7568 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7574 /* Make sure the expression is of the proper form. */
7575 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7576 rhs
= gimple_assign_rhs2 (stmt
);
7577 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7578 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7579 rhs
= gimple_assign_rhs1 (stmt
);
7583 tmpbase
= ((enum built_in_function
)
7584 ((need_new
? newbase
: oldbase
) + index
+ 1));
7585 decl
= builtin_decl_explicit (tmpbase
);
7586 if (decl
== NULL_TREE
)
7588 itype
= TREE_TYPE (TREE_TYPE (decl
));
7589 imode
= TYPE_MODE (itype
);
7591 /* We could test all of the various optabs involved, but the fact of the
7592 matter is that (with the exception of i486 vs i586 and xadd) all targets
7593 that support any atomic operaton optab also implements compare-and-swap.
7594 Let optabs.c take care of expanding any compare-and-swap loop. */
7595 if (!can_compare_and_swap_p (imode
, true))
7598 gsi
= gsi_last_bb (load_bb
);
7599 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7601 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7602 It only requires that the operation happen atomically. Thus we can
7603 use the RELAXED memory model. */
7604 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7605 fold_convert_loc (loc
, itype
, rhs
),
7606 build_int_cst (NULL
,
7607 seq_cst
? MEMMODEL_SEQ_CST
7608 : MEMMODEL_RELAXED
));
7610 if (need_old
|| need_new
)
7612 lhs
= need_old
? loaded_val
: stored_val
;
7613 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7614 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7617 call
= fold_convert_loc (loc
, void_type_node
, call
);
7618 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7619 gsi_remove (&gsi
, true);
7621 gsi
= gsi_last_bb (store_bb
);
7622 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7623 gsi_remove (&gsi
, true);
7624 gsi
= gsi_last_bb (store_bb
);
7625 gsi_remove (&gsi
, true);
7627 if (gimple_in_ssa_p (cfun
))
7628 update_ssa (TODO_update_ssa_no_phi
);
7633 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7637 newval = rhs; // with oldval replacing *addr in rhs
7638 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7639 if (oldval != newval)
7642 INDEX is log2 of the size of the data type, and thus usable to find the
7643 index of the builtin decl. */
7646 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7647 tree addr
, tree loaded_val
, tree stored_val
,
7650 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7651 tree type
, itype
, cmpxchg
, iaddr
;
7652 gimple_stmt_iterator si
;
7653 basic_block loop_header
= single_succ (load_bb
);
7656 enum built_in_function fncode
;
7658 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7659 order to use the RELAXED memory model effectively. */
7660 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7662 cmpxchg
= builtin_decl_explicit (fncode
);
7663 if (cmpxchg
== NULL_TREE
)
7665 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7666 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7668 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7671 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7672 si
= gsi_last_bb (load_bb
);
7673 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7675 /* For floating-point values, we'll need to view-convert them to integers
7676 so that we can perform the atomic compare and swap. Simplify the
7677 following code by always setting up the "i"ntegral variables. */
7678 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7682 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7685 = force_gimple_operand_gsi (&si
,
7686 fold_convert (TREE_TYPE (iaddr
), addr
),
7687 false, NULL_TREE
, true, GSI_SAME_STMT
);
7688 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7689 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7690 loadedi
= create_tmp_var (itype
, NULL
);
7691 if (gimple_in_ssa_p (cfun
))
7692 loadedi
= make_ssa_name (loadedi
, NULL
);
7697 loadedi
= loaded_val
;
7700 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7701 tree loaddecl
= builtin_decl_explicit (fncode
);
7704 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
7705 build_call_expr (loaddecl
, 2, iaddr
,
7706 build_int_cst (NULL_TREE
,
7707 MEMMODEL_RELAXED
)));
7709 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
7710 build_int_cst (TREE_TYPE (iaddr
), 0));
7713 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
7716 /* Move the value to the LOADEDI temporary. */
7717 if (gimple_in_ssa_p (cfun
))
7719 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7720 phi
= create_phi_node (loadedi
, loop_header
);
7721 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7725 gsi_insert_before (&si
,
7726 gimple_build_assign (loadedi
, initial
),
7728 if (loadedi
!= loaded_val
)
7730 gimple_stmt_iterator gsi2
;
7733 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7734 gsi2
= gsi_start_bb (loop_header
);
7735 if (gimple_in_ssa_p (cfun
))
7738 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7739 true, GSI_SAME_STMT
);
7740 stmt
= gimple_build_assign (loaded_val
, x
);
7741 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7745 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7746 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7747 true, GSI_SAME_STMT
);
7750 gsi_remove (&si
, true);
7752 si
= gsi_last_bb (store_bb
);
7753 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7756 storedi
= stored_val
;
7759 force_gimple_operand_gsi (&si
,
7760 build1 (VIEW_CONVERT_EXPR
, itype
,
7761 stored_val
), true, NULL_TREE
, true,
7764 /* Build the compare&swap statement. */
7765 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7766 new_storedi
= force_gimple_operand_gsi (&si
,
7767 fold_convert (TREE_TYPE (loadedi
),
7770 true, GSI_SAME_STMT
);
7772 if (gimple_in_ssa_p (cfun
))
7776 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7777 stmt
= gimple_build_assign (old_vali
, loadedi
);
7778 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7780 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7781 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7784 /* Note that we always perform the comparison as an integer, even for
7785 floating point. This allows the atomic operation to properly
7786 succeed even with NaNs and -0.0. */
7787 stmt
= gimple_build_cond_empty
7788 (build2 (NE_EXPR
, boolean_type_node
,
7789 new_storedi
, old_vali
));
7790 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7793 e
= single_succ_edge (store_bb
);
7794 e
->flags
&= ~EDGE_FALLTHRU
;
7795 e
->flags
|= EDGE_FALSE_VALUE
;
7797 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7799 /* Copy the new value to loadedi (we already did that before the condition
7800 if we are not in SSA). */
7801 if (gimple_in_ssa_p (cfun
))
7803 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7804 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7807 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7808 gsi_remove (&si
, true);
7810 struct loop
*loop
= alloc_loop ();
7811 loop
->header
= loop_header
;
7812 loop
->latch
= store_bb
;
7813 add_loop (loop
, loop_header
->loop_father
);
7815 if (gimple_in_ssa_p (cfun
))
7816 update_ssa (TODO_update_ssa_no_phi
);
7821 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7823 GOMP_atomic_start ();
7827 The result is not globally atomic, but works so long as all parallel
7828 references are within #pragma omp atomic directives. According to
7829 responses received from omp@openmp.org, appears to be within spec.
7830 Which makes sense, since that's how several other compilers handle
7831 this situation as well.
7832 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7833 expanding. STORED_VAL is the operand of the matching
7834 GIMPLE_OMP_ATOMIC_STORE.
7837 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7841 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7846 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7847 tree addr
, tree loaded_val
, tree stored_val
)
7849 gimple_stmt_iterator si
;
7853 si
= gsi_last_bb (load_bb
);
7854 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7856 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7857 t
= build_call_expr (t
, 0);
7858 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7860 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7861 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7862 gsi_remove (&si
, true);
7864 si
= gsi_last_bb (store_bb
);
7865 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7867 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7869 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7871 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7872 t
= build_call_expr (t
, 0);
7873 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7874 gsi_remove (&si
, true);
7876 if (gimple_in_ssa_p (cfun
))
7877 update_ssa (TODO_update_ssa_no_phi
);
7881 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7882 using expand_omp_atomic_fetch_op. If it failed, we try to
7883 call expand_omp_atomic_pipeline, and if it fails too, the
7884 ultimate fallback is wrapping the operation in a mutex
7885 (expand_omp_atomic_mutex). REGION is the atomic region built
7886 by build_omp_regions_1(). */
7889 expand_omp_atomic (struct omp_region
*region
)
7891 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7892 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7893 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7894 tree addr
= gimple_omp_atomic_load_rhs (load
);
7895 tree stored_val
= gimple_omp_atomic_store_val (store
);
7896 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7897 HOST_WIDE_INT index
;
7899 /* Make sure the type is one of the supported sizes. */
7900 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7901 index
= exact_log2 (index
);
7902 if (index
>= 0 && index
<= 4)
7904 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7906 /* __sync builtins require strict data alignment. */
7907 if (exact_log2 (align
) >= index
)
7910 if (loaded_val
== stored_val
7911 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7912 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7913 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7914 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7918 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7919 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7920 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7921 && store_bb
== single_succ (load_bb
)
7922 && first_stmt (store_bb
) == store
7923 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7927 /* When possible, use specialized atomic update functions. */
7928 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7929 && store_bb
== single_succ (load_bb
)
7930 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7931 loaded_val
, stored_val
, index
))
7934 /* If we don't have specialized __sync builtins, try and implement
7935 as a compare and swap loop. */
7936 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7937 loaded_val
, stored_val
, index
))
7942 /* The ultimate fallback is wrapping the operation in a mutex. */
7943 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7947 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7950 expand_omp_target (struct omp_region
*region
)
7952 basic_block entry_bb
, exit_bb
, new_bb
;
7953 struct function
*child_cfun
= NULL
;
7954 tree child_fn
= NULL_TREE
, block
, t
;
7955 gimple_stmt_iterator gsi
;
7956 gimple entry_stmt
, stmt
;
7959 entry_stmt
= last_stmt (region
->entry
);
7960 new_bb
= region
->entry
;
7961 int kind
= gimple_omp_target_kind (entry_stmt
);
7962 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7964 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7965 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7968 entry_bb
= region
->entry
;
7969 exit_bb
= region
->exit
;
7971 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7973 unsigned srcidx
, dstidx
, num
;
7975 /* If the target region needs data sent from the parent
7976 function, then the very first statement (except possible
7977 tree profile counter updates) of the parallel body
7978 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7979 &.OMP_DATA_O is passed as an argument to the child function,
7980 we need to replace it with the argument as seen by the child
7983 In most cases, this will end up being the identity assignment
7984 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7985 a function call that has been inlined, the original PARM_DECL
7986 .OMP_DATA_I may have been converted into a different local
7987 variable. In which case, we need to keep the assignment. */
7988 if (gimple_omp_target_data_arg (entry_stmt
))
7990 basic_block entry_succ_bb
= single_succ (entry_bb
);
7991 gimple_stmt_iterator gsi
;
7993 gimple tgtcopy_stmt
= NULL
;
7995 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7997 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7999 gcc_assert (!gsi_end_p (gsi
));
8000 stmt
= gsi_stmt (gsi
);
8001 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8004 if (gimple_num_ops (stmt
) == 2)
8006 tree arg
= gimple_assign_rhs1 (stmt
);
8008 /* We're ignoring the subcode because we're
8009 effectively doing a STRIP_NOPS. */
8011 if (TREE_CODE (arg
) == ADDR_EXPR
8012 && TREE_OPERAND (arg
, 0) == sender
)
8014 tgtcopy_stmt
= stmt
;
8020 gcc_assert (tgtcopy_stmt
!= NULL
);
8021 arg
= DECL_ARGUMENTS (child_fn
);
8023 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8024 gsi_remove (&gsi
, true);
8027 /* Declare local variables needed in CHILD_CFUN. */
8028 block
= DECL_INITIAL (child_fn
);
8029 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8030 /* The gimplifier could record temporaries in target block
8031 rather than in containing function's local_decls chain,
8032 which would mean cgraph missed finalizing them. Do it now. */
8033 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8034 if (TREE_CODE (t
) == VAR_DECL
8036 && !DECL_EXTERNAL (t
))
8037 varpool_finalize_decl (t
);
8038 DECL_SAVED_TREE (child_fn
) = NULL
;
8039 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8040 gimple_set_body (child_fn
, NULL
);
8041 TREE_USED (block
) = 1;
8043 /* Reset DECL_CONTEXT on function arguments. */
8044 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8045 DECL_CONTEXT (t
) = child_fn
;
8047 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
8048 so that it can be moved to the child function. */
8049 gsi
= gsi_last_bb (entry_bb
);
8050 stmt
= gsi_stmt (gsi
);
8051 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
8052 && gimple_omp_target_kind (stmt
)
8053 == GF_OMP_TARGET_KIND_REGION
);
8054 gsi_remove (&gsi
, true);
8055 e
= split_block (entry_bb
, stmt
);
8057 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8059 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8062 gsi
= gsi_last_bb (exit_bb
);
8063 gcc_assert (!gsi_end_p (gsi
)
8064 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8065 stmt
= gimple_build_return (NULL
);
8066 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8067 gsi_remove (&gsi
, true);
8070 /* Move the target region into CHILD_CFUN. */
8072 block
= gimple_block (entry_stmt
);
8074 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8076 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8077 /* When the OMP expansion process cannot guarantee an up-to-date
8078 loop tree arrange for the child function to fixup loops. */
8079 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8080 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8082 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8083 num
= vec_safe_length (child_cfun
->local_decls
);
8084 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8086 t
= (*child_cfun
->local_decls
)[srcidx
];
8087 if (DECL_CONTEXT (t
) == cfun
->decl
)
8089 if (srcidx
!= dstidx
)
8090 (*child_cfun
->local_decls
)[dstidx
] = t
;
8094 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8096 /* Inform the callgraph about the new function. */
8097 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8098 cgraph_add_new_function (child_fn
, true);
8100 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8101 fixed in a following pass. */
8102 push_cfun (child_cfun
);
8103 rebuild_cgraph_edges ();
8105 /* Some EH regions might become dead, see PR34608. If
8106 pass_cleanup_cfg isn't the first pass to happen with the
8107 new child, these dead EH edges might cause problems.
8108 Clean them up now. */
8109 if (flag_exceptions
)
8112 bool changed
= false;
8114 FOR_EACH_BB_FN (bb
, cfun
)
8115 changed
|= gimple_purge_dead_eh_edges (bb
);
8117 cleanup_tree_cfg ();
8122 /* Emit a library call to launch the target region, or do data
8124 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8125 enum built_in_function start_ix
;
8126 location_t clause_loc
;
8128 clauses
= gimple_omp_target_clauses (entry_stmt
);
8130 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8131 start_ix
= BUILT_IN_GOMP_TARGET
;
8132 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8133 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8135 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8137 /* By default, the value of DEVICE is -1 (let runtime library choose)
8138 and there is no conditional. */
8140 device
= build_int_cst (integer_type_node
, -1);
8142 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8144 cond
= OMP_CLAUSE_IF_EXPR (c
);
8146 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8149 device
= OMP_CLAUSE_DEVICE_ID (c
);
8150 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8153 clause_loc
= gimple_location (entry_stmt
);
8155 /* Ensure 'device' is of the correct type. */
8156 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8158 /* If we found the clause 'if (cond)', build
8159 (cond ? device : -2). */
8162 cond
= gimple_boolify (cond
);
8164 basic_block cond_bb
, then_bb
, else_bb
;
8168 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8169 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8171 gsi
= gsi_last_bb (new_bb
);
8173 e
= split_block (new_bb
, gsi_stmt (gsi
));
8176 e
= split_block (new_bb
, NULL
);
8181 then_bb
= create_empty_bb (cond_bb
);
8182 else_bb
= create_empty_bb (then_bb
);
8183 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8184 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8186 stmt
= gimple_build_cond_empty (cond
);
8187 gsi
= gsi_last_bb (cond_bb
);
8188 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8190 gsi
= gsi_start_bb (then_bb
);
8191 stmt
= gimple_build_assign (tmp_var
, device
);
8192 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8194 gsi
= gsi_start_bb (else_bb
);
8195 stmt
= gimple_build_assign (tmp_var
,
8196 build_int_cst (integer_type_node
, -2));
8197 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8199 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8200 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8203 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8204 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8206 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8207 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8212 gsi
= gsi_last_bb (new_bb
);
8213 t
= gimple_omp_target_data_arg (entry_stmt
);
8216 t1
= size_zero_node
;
8217 t2
= build_zero_cst (ptr_type_node
);
8223 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8224 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8225 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8226 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8227 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8231 /* FIXME: This will be address of
8232 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8233 symbol, as soon as the linker plugin is able to create it for us. */
8234 tree openmp_target
= build_zero_cst (ptr_type_node
);
8235 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8237 tree fnaddr
= build_fold_addr_expr (child_fn
);
8238 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8239 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8242 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8243 device
, openmp_target
, t1
, t2
, t3
, t4
);
8244 gimple_set_location (g
, gimple_location (entry_stmt
));
8245 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8246 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8249 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8250 gsi_remove (&gsi
, true);
8252 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8254 gsi
= gsi_last_bb (region
->exit
);
8256 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8257 gsi_remove (&gsi
, true);
8262 /* Expand the parallel region tree rooted at REGION. Expansion
8263 proceeds in depth-first order. Innermost regions are expanded
8264 first. This way, parallel regions that require a new function to
8265 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8266 internal dependencies in their body. */
8269 expand_omp (struct omp_region
*region
)
8273 location_t saved_location
;
8274 gimple inner_stmt
= NULL
;
8276 /* First, determine whether this is a combined parallel+workshare
8278 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8279 determine_parallel_type (region
);
8281 if (region
->type
== GIMPLE_OMP_FOR
8282 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8283 inner_stmt
= last_stmt (region
->inner
->entry
);
8286 expand_omp (region
->inner
);
8288 saved_location
= input_location
;
8289 if (gimple_has_location (last_stmt (region
->entry
)))
8290 input_location
= gimple_location (last_stmt (region
->entry
));
8292 switch (region
->type
)
8294 case GIMPLE_OMP_PARALLEL
:
8295 case GIMPLE_OMP_TASK
:
8296 expand_omp_taskreg (region
);
8299 case GIMPLE_OMP_FOR
:
8300 expand_omp_for (region
, inner_stmt
);
8303 case GIMPLE_OMP_SECTIONS
:
8304 expand_omp_sections (region
);
8307 case GIMPLE_OMP_SECTION
:
8308 /* Individual omp sections are handled together with their
8309 parent GIMPLE_OMP_SECTIONS region. */
8312 case GIMPLE_OMP_SINGLE
:
8313 expand_omp_single (region
);
8316 case GIMPLE_OMP_MASTER
:
8317 case GIMPLE_OMP_TASKGROUP
:
8318 case GIMPLE_OMP_ORDERED
:
8319 case GIMPLE_OMP_CRITICAL
:
8320 case GIMPLE_OMP_TEAMS
:
8321 expand_omp_synch (region
);
8324 case GIMPLE_OMP_ATOMIC_LOAD
:
8325 expand_omp_atomic (region
);
8328 case GIMPLE_OMP_TARGET
:
8329 expand_omp_target (region
);
8336 input_location
= saved_location
;
8337 region
= region
->next
;
8342 /* Helper for build_omp_regions. Scan the dominator tree starting at
8343 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8344 true, the function ends once a single tree is built (otherwise, whole
8345 forest of OMP constructs may be built). */
8348 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8351 gimple_stmt_iterator gsi
;
8355 gsi
= gsi_last_bb (bb
);
8356 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8358 struct omp_region
*region
;
8359 enum gimple_code code
;
8361 stmt
= gsi_stmt (gsi
);
8362 code
= gimple_code (stmt
);
8363 if (code
== GIMPLE_OMP_RETURN
)
8365 /* STMT is the return point out of region PARENT. Mark it
8366 as the exit point and make PARENT the immediately
8367 enclosing region. */
8368 gcc_assert (parent
);
8371 parent
= parent
->outer
;
8373 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8375 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8376 GIMPLE_OMP_RETURN, but matches with
8377 GIMPLE_OMP_ATOMIC_LOAD. */
8378 gcc_assert (parent
);
8379 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8382 parent
= parent
->outer
;
8385 else if (code
== GIMPLE_OMP_CONTINUE
)
8387 gcc_assert (parent
);
8390 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8392 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8393 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8396 else if (code
== GIMPLE_OMP_TARGET
8397 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8398 new_omp_region (bb
, code
, parent
);
8401 /* Otherwise, this directive becomes the parent for a new
8403 region
= new_omp_region (bb
, code
, parent
);
8408 if (single_tree
&& !parent
)
8411 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8413 son
= next_dom_son (CDI_DOMINATORS
, son
))
8414 build_omp_regions_1 (son
, parent
, single_tree
);
8417 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8421 build_omp_regions_root (basic_block root
)
8423 gcc_assert (root_omp_region
== NULL
);
8424 build_omp_regions_1 (root
, NULL
, true);
8425 gcc_assert (root_omp_region
!= NULL
);
8428 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8431 omp_expand_local (basic_block head
)
8433 build_omp_regions_root (head
);
8434 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8436 fprintf (dump_file
, "\nOMP region tree\n\n");
8437 dump_omp_region (dump_file
, root_omp_region
, 0);
8438 fprintf (dump_file
, "\n");
8441 remove_exit_barriers (root_omp_region
);
8442 expand_omp (root_omp_region
);
8444 free_omp_regions ();
8447 /* Scan the CFG and build a tree of OMP regions. Return the root of
8448 the OMP region tree. */
8451 build_omp_regions (void)
8453 gcc_assert (root_omp_region
== NULL
);
8454 calculate_dominance_info (CDI_DOMINATORS
);
8455 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8458 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8461 execute_expand_omp (void)
8463 build_omp_regions ();
8465 if (!root_omp_region
)
8470 fprintf (dump_file
, "\nOMP region tree\n\n");
8471 dump_omp_region (dump_file
, root_omp_region
, 0);
8472 fprintf (dump_file
, "\n");
8475 remove_exit_barriers (root_omp_region
);
8477 expand_omp (root_omp_region
);
8479 cleanup_tree_cfg ();
8481 free_omp_regions ();
8486 /* OMP expansion -- the default pass, run before creation of SSA form. */
8489 gate_expand_omp (void)
8491 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8492 || flag_cilkplus
!= 0) && !seen_error ());
8497 const pass_data pass_data_expand_omp
=
8499 GIMPLE_PASS
, /* type */
8500 "ompexp", /* name */
8501 OPTGROUP_NONE
, /* optinfo_flags */
8502 true, /* has_gate */
8503 true, /* has_execute */
8504 TV_NONE
, /* tv_id */
8505 PROP_gimple_any
, /* properties_required */
8506 0, /* properties_provided */
8507 0, /* properties_destroyed */
8508 0, /* todo_flags_start */
8509 0, /* todo_flags_finish */
8512 class pass_expand_omp
: public gimple_opt_pass
8515 pass_expand_omp (gcc::context
*ctxt
)
8516 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8519 /* opt_pass methods: */
8520 bool gate () { return gate_expand_omp (); }
8521 unsigned int execute () { return execute_expand_omp (); }
8523 }; // class pass_expand_omp
8528 make_pass_expand_omp (gcc::context
*ctxt
)
8530 return new pass_expand_omp (ctxt
);
8533 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8535 /* If ctx is a worksharing context inside of a cancellable parallel
8536 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8537 and conditional branch to parallel's cancel_label to handle
8538 cancellation in the implicit barrier. */
8541 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8543 gimple omp_return
= gimple_seq_last_stmt (*body
);
8544 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8545 if (gimple_omp_return_nowait_p (omp_return
))
8548 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8549 && ctx
->outer
->cancellable
)
8551 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8552 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8553 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8554 gimple_omp_return_set_lhs (omp_return
, lhs
);
8555 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8556 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8557 fold_convert (c_bool_type
,
8558 boolean_false_node
),
8559 ctx
->outer
->cancel_label
, fallthru_label
);
8560 gimple_seq_add_stmt (body
, g
);
8561 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8565 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8566 CTX is the enclosing OMP context for the current statement. */
8569 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8571 tree block
, control
;
8572 gimple_stmt_iterator tgsi
;
8573 gimple stmt
, new_stmt
, bind
, t
;
8574 gimple_seq ilist
, dlist
, olist
, new_body
;
8576 stmt
= gsi_stmt (*gsi_p
);
8578 push_gimplify_context ();
8582 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8583 &ilist
, &dlist
, ctx
, NULL
);
8585 new_body
= gimple_omp_body (stmt
);
8586 gimple_omp_set_body (stmt
, NULL
);
8587 tgsi
= gsi_start (new_body
);
8588 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8593 sec_start
= gsi_stmt (tgsi
);
8594 sctx
= maybe_lookup_ctx (sec_start
);
8597 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8598 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8599 GSI_CONTINUE_LINKING
);
8600 gimple_omp_set_body (sec_start
, NULL
);
8602 if (gsi_one_before_end_p (tgsi
))
8604 gimple_seq l
= NULL
;
8605 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8607 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8608 gimple_omp_section_set_last (sec_start
);
8611 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8612 GSI_CONTINUE_LINKING
);
8615 block
= make_node (BLOCK
);
8616 bind
= gimple_build_bind (NULL
, new_body
, block
);
8619 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8621 block
= make_node (BLOCK
);
8622 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8623 gsi_replace (gsi_p
, new_stmt
, true);
8625 pop_gimplify_context (new_stmt
);
8626 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8627 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8628 if (BLOCK_VARS (block
))
8629 TREE_USED (block
) = 1;
8632 gimple_seq_add_seq (&new_body
, ilist
);
8633 gimple_seq_add_stmt (&new_body
, stmt
);
8634 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8635 gimple_seq_add_stmt (&new_body
, bind
);
8637 control
= create_tmp_var (unsigned_type_node
, ".section");
8638 t
= gimple_build_omp_continue (control
, control
);
8639 gimple_omp_sections_set_control (stmt
, control
);
8640 gimple_seq_add_stmt (&new_body
, t
);
8642 gimple_seq_add_seq (&new_body
, olist
);
8643 if (ctx
->cancellable
)
8644 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8645 gimple_seq_add_seq (&new_body
, dlist
);
8647 new_body
= maybe_catch_exception (new_body
);
8649 t
= gimple_build_omp_return
8650 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8651 OMP_CLAUSE_NOWAIT
));
8652 gimple_seq_add_stmt (&new_body
, t
);
8653 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8655 gimple_bind_set_body (new_stmt
, new_body
);
8659 /* A subroutine of lower_omp_single. Expand the simple form of
8660 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8662 if (GOMP_single_start ())
8664 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8666 FIXME. It may be better to delay expanding the logic of this until
8667 pass_expand_omp. The expanded logic may make the job more difficult
8668 to a synchronization analysis pass. */
8671 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8673 location_t loc
= gimple_location (single_stmt
);
8674 tree tlabel
= create_artificial_label (loc
);
8675 tree flabel
= create_artificial_label (loc
);
8679 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8680 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8681 call
= gimple_build_call (decl
, 0);
8682 gimple_call_set_lhs (call
, lhs
);
8683 gimple_seq_add_stmt (pre_p
, call
);
8685 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8686 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8689 gimple_seq_add_stmt (pre_p
, cond
);
8690 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8691 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8692 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8696 /* A subroutine of lower_omp_single. Expand the simple form of
8697 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8699 #pragma omp single copyprivate (a, b, c)
8701 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8704 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8710 GOMP_single_copy_end (©out);
8721 FIXME. It may be better to delay expanding the logic of this until
8722 pass_expand_omp. The expanded logic may make the job more difficult
8723 to a synchronization analysis pass. */
8726 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8728 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8729 gimple_seq copyin_seq
;
8730 location_t loc
= gimple_location (single_stmt
);
8732 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8734 ptr_type
= build_pointer_type (ctx
->record_type
);
8735 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8737 l0
= create_artificial_label (loc
);
8738 l1
= create_artificial_label (loc
);
8739 l2
= create_artificial_label (loc
);
8741 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8742 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8743 t
= fold_convert_loc (loc
, ptr_type
, t
);
8744 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8746 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8747 build_int_cst (ptr_type
, 0));
8748 t
= build3 (COND_EXPR
, void_type_node
, t
,
8749 build_and_jump (&l0
), build_and_jump (&l1
));
8750 gimplify_and_add (t
, pre_p
);
8752 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8754 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8757 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8760 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8761 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8762 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8763 gimplify_and_add (t
, pre_p
);
8765 t
= build_and_jump (&l2
);
8766 gimplify_and_add (t
, pre_p
);
8768 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8770 gimple_seq_add_seq (pre_p
, copyin_seq
);
8772 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8776 /* Expand code for an OpenMP single directive. */
8779 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8782 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8783 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8785 push_gimplify_context ();
8787 block
= make_node (BLOCK
);
8788 bind
= gimple_build_bind (NULL
, NULL
, block
);
8789 gsi_replace (gsi_p
, bind
, true);
8792 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8793 &bind_body
, &dlist
, ctx
, NULL
);
8794 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8796 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8798 if (ctx
->record_type
)
8799 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8801 lower_omp_single_simple (single_stmt
, &bind_body
);
8803 gimple_omp_set_body (single_stmt
, NULL
);
8805 gimple_seq_add_seq (&bind_body
, dlist
);
8807 bind_body
= maybe_catch_exception (bind_body
);
8809 t
= gimple_build_omp_return
8810 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8811 OMP_CLAUSE_NOWAIT
));
8812 gimple_seq_add_stmt (&bind_body_tail
, t
);
8813 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8814 if (ctx
->record_type
)
8816 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8817 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8818 TREE_THIS_VOLATILE (clobber
) = 1;
8819 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8820 clobber
), GSI_SAME_STMT
);
8822 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8823 gimple_bind_set_body (bind
, bind_body
);
8825 pop_gimplify_context (bind
);
8827 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8828 BLOCK_VARS (block
) = ctx
->block_vars
;
8829 if (BLOCK_VARS (block
))
8830 TREE_USED (block
) = 1;
8834 /* Expand code for an OpenMP master directive. */
8837 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8839 tree block
, lab
= NULL
, x
, bfn_decl
;
8840 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8841 location_t loc
= gimple_location (stmt
);
8844 push_gimplify_context ();
8846 block
= make_node (BLOCK
);
8847 bind
= gimple_build_bind (NULL
, NULL
, block
);
8848 gsi_replace (gsi_p
, bind
, true);
8849 gimple_bind_add_stmt (bind
, stmt
);
8851 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8852 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8853 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8854 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8856 gimplify_and_add (x
, &tseq
);
8857 gimple_bind_add_seq (bind
, tseq
);
8859 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8860 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8861 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8862 gimple_omp_set_body (stmt
, NULL
);
8864 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8866 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8868 pop_gimplify_context (bind
);
8870 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8871 BLOCK_VARS (block
) = ctx
->block_vars
;
8875 /* Expand code for an OpenMP taskgroup directive. */
8878 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8880 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8881 tree block
= make_node (BLOCK
);
8883 bind
= gimple_build_bind (NULL
, NULL
, block
);
8884 gsi_replace (gsi_p
, bind
, true);
8885 gimple_bind_add_stmt (bind
, stmt
);
8887 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8889 gimple_bind_add_stmt (bind
, x
);
8891 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8892 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8893 gimple_omp_set_body (stmt
, NULL
);
8895 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8897 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8898 BLOCK_VARS (block
) = ctx
->block_vars
;
8902 /* Expand code for an OpenMP ordered directive. */
8905 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8908 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8910 push_gimplify_context ();
8912 block
= make_node (BLOCK
);
8913 bind
= gimple_build_bind (NULL
, NULL
, block
);
8914 gsi_replace (gsi_p
, bind
, true);
8915 gimple_bind_add_stmt (bind
, stmt
);
8917 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8919 gimple_bind_add_stmt (bind
, x
);
8921 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8922 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8923 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8924 gimple_omp_set_body (stmt
, NULL
);
8926 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8927 gimple_bind_add_stmt (bind
, x
);
8929 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8931 pop_gimplify_context (bind
);
8933 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8934 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8938 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8939 substitution of a couple of function calls. But in the NAMED case,
8940 requires that languages coordinate a symbol name. It is therefore
8941 best put here in common code. */
8943 static GTY((param1_is (tree
), param2_is (tree
)))
8944 splay_tree critical_name_mutexes
;
8947 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8950 tree name
, lock
, unlock
;
8951 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8952 location_t loc
= gimple_location (stmt
);
8955 name
= gimple_omp_critical_name (stmt
);
8961 if (!critical_name_mutexes
)
8962 critical_name_mutexes
8963 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8964 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8965 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8967 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8972 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8974 new_str
= ACONCAT ((".gomp_critical_user_",
8975 IDENTIFIER_POINTER (name
), NULL
));
8976 DECL_NAME (decl
) = get_identifier (new_str
);
8977 TREE_PUBLIC (decl
) = 1;
8978 TREE_STATIC (decl
) = 1;
8979 DECL_COMMON (decl
) = 1;
8980 DECL_ARTIFICIAL (decl
) = 1;
8981 DECL_IGNORED_P (decl
) = 1;
8982 varpool_finalize_decl (decl
);
8984 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8985 (splay_tree_value
) decl
);
8988 decl
= (tree
) n
->value
;
8990 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8991 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8993 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8994 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8995 build_fold_addr_expr_loc (loc
, decl
));
8999 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9000 lock
= build_call_expr_loc (loc
, lock
, 0);
9002 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9003 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9006 push_gimplify_context ();
9008 block
= make_node (BLOCK
);
9009 bind
= gimple_build_bind (NULL
, NULL
, block
);
9010 gsi_replace (gsi_p
, bind
, true);
9011 gimple_bind_add_stmt (bind
, stmt
);
9013 tbody
= gimple_bind_body (bind
);
9014 gimplify_and_add (lock
, &tbody
);
9015 gimple_bind_set_body (bind
, tbody
);
9017 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9018 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9019 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9020 gimple_omp_set_body (stmt
, NULL
);
9022 tbody
= gimple_bind_body (bind
);
9023 gimplify_and_add (unlock
, &tbody
);
9024 gimple_bind_set_body (bind
, tbody
);
9026 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9028 pop_gimplify_context (bind
);
9029 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9030 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9034 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9035 for a lastprivate clause. Given a loop control predicate of (V
9036 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9037 is appended to *DLIST, iterator initialization is appended to
9041 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9042 gimple_seq
*dlist
, struct omp_context
*ctx
)
9044 tree clauses
, cond
, vinit
;
9045 enum tree_code cond_code
;
9048 cond_code
= fd
->loop
.cond_code
;
9049 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9051 /* When possible, use a strict equality expression. This can let VRP
9052 type optimizations deduce the value and remove a copy. */
9053 if (tree_fits_shwi_p (fd
->loop
.step
))
9055 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9056 if (step
== 1 || step
== -1)
9057 cond_code
= EQ_EXPR
;
9060 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
9062 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9064 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
9065 if (!gimple_seq_empty_p (stmts
))
9067 gimple_seq_add_seq (&stmts
, *dlist
);
9070 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9071 vinit
= fd
->loop
.n1
;
9072 if (cond_code
== EQ_EXPR
9073 && tree_fits_shwi_p (fd
->loop
.n2
)
9074 && ! integer_zerop (fd
->loop
.n2
))
9075 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9077 vinit
= unshare_expr (vinit
);
9079 /* Initialize the iterator variable, so that threads that don't execute
9080 any iterations don't execute the lastprivate clauses by accident. */
9081 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9086 /* Lower code for an OpenMP loop directive. */
9089 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9092 struct omp_for_data fd
, *fdp
= NULL
;
9093 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9094 gimple_seq omp_for_body
, body
, dlist
;
9097 push_gimplify_context ();
9099 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9101 block
= make_node (BLOCK
);
9102 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9103 /* Replace at gsi right away, so that 'stmt' is no member
9104 of a sequence anymore as we're going to add to to a different
9106 gsi_replace (gsi_p
, new_stmt
, true);
9108 /* Move declaration of temporaries in the loop body before we make
9110 omp_for_body
= gimple_omp_body (stmt
);
9111 if (!gimple_seq_empty_p (omp_for_body
)
9112 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9114 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9115 tree vars
= gimple_bind_vars (inner_bind
);
9116 gimple_bind_append_vars (new_stmt
, vars
);
9117 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9118 keep them on the inner_bind and it's block. */
9119 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9120 if (gimple_bind_block (inner_bind
))
9121 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9124 if (gimple_omp_for_combined_into_p (stmt
))
9126 extract_omp_for_data (stmt
, &fd
, NULL
);
9129 /* We need two temporaries with fd.loop.v type (istart/iend)
9130 and then (fd.collapse - 1) temporaries with the same
9131 type for count2 ... countN-1 vars if not constant. */
9133 tree type
= fd
.iter_type
;
9135 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9136 count
+= fd
.collapse
- 1;
9137 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9138 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9142 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9143 OMP_CLAUSE__LOOPTEMP_
);
9144 for (i
= 0; i
< count
; i
++)
9149 gcc_assert (outerc
);
9150 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9151 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9152 OMP_CLAUSE__LOOPTEMP_
);
9156 temp
= create_tmp_var (type
, NULL
);
9157 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9159 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9160 OMP_CLAUSE_DECL (*pc
) = temp
;
9161 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9166 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9169 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9171 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9173 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9175 /* Lower the header expressions. At this point, we can assume that
9176 the header is of the form:
9178 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9180 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9181 using the .omp_data_s mapping, if needed. */
9182 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9184 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9185 if (!is_gimple_min_invariant (*rhs_p
))
9186 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9188 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9189 if (!is_gimple_min_invariant (*rhs_p
))
9190 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9192 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9193 if (!is_gimple_min_invariant (*rhs_p
))
9194 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9197 /* Once lowered, extract the bounds and clauses. */
9198 extract_omp_for_data (stmt
, &fd
, NULL
);
9200 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9202 gimple_seq_add_stmt (&body
, stmt
);
9203 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9205 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9208 /* After the loop, add exit clauses. */
9209 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9211 if (ctx
->cancellable
)
9212 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9214 gimple_seq_add_seq (&body
, dlist
);
9216 body
= maybe_catch_exception (body
);
9218 /* Region exit marker goes at the end of the loop body. */
9219 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9220 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9221 pop_gimplify_context (new_stmt
);
9223 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9224 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9225 if (BLOCK_VARS (block
))
9226 TREE_USED (block
) = 1;
9228 gimple_bind_set_body (new_stmt
, body
);
9229 gimple_omp_set_body (stmt
, NULL
);
9230 gimple_omp_for_set_pre_body (stmt
, NULL
);
9233 /* Callback for walk_stmts. Check if the current statement only contains
9234 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9237 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9238 bool *handled_ops_p
,
9239 struct walk_stmt_info
*wi
)
9241 int *info
= (int *) wi
->info
;
9242 gimple stmt
= gsi_stmt (*gsi_p
);
9244 *handled_ops_p
= true;
9245 switch (gimple_code (stmt
))
9249 case GIMPLE_OMP_FOR
:
9250 case GIMPLE_OMP_SECTIONS
:
9251 *info
= *info
== 0 ? 1 : -1;
9260 struct omp_taskcopy_context
9262 /* This field must be at the beginning, as we do "inheritance": Some
9263 callback functions for tree-inline.c (e.g., omp_copy_decl)
9264 receive a copy_body_data pointer that is up-casted to an
9265 omp_context pointer. */
9271 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9273 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9275 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9276 return create_tmp_var (TREE_TYPE (var
), NULL
);
9282 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9284 tree name
, new_fields
= NULL
, type
, f
;
9286 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9287 name
= DECL_NAME (TYPE_NAME (orig_type
));
9288 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9289 TYPE_DECL
, name
, type
);
9290 TYPE_NAME (type
) = name
;
9292 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9294 tree new_f
= copy_node (f
);
9295 DECL_CONTEXT (new_f
) = type
;
9296 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9297 TREE_CHAIN (new_f
) = new_fields
;
9298 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9299 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9300 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9303 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9305 TYPE_FIELDS (type
) = nreverse (new_fields
);
9310 /* Create task copyfn. */
9313 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9315 struct function
*child_cfun
;
9316 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9317 tree record_type
, srecord_type
, bind
, list
;
9318 bool record_needs_remap
= false, srecord_needs_remap
= false;
9320 struct omp_taskcopy_context tcctx
;
9321 location_t loc
= gimple_location (task_stmt
);
9323 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9324 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9325 gcc_assert (child_cfun
->cfg
== NULL
);
9326 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9328 /* Reset DECL_CONTEXT on function arguments. */
9329 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9330 DECL_CONTEXT (t
) = child_fn
;
9332 /* Populate the function. */
9333 push_gimplify_context ();
9334 push_cfun (child_cfun
);
9336 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9337 TREE_SIDE_EFFECTS (bind
) = 1;
9339 DECL_SAVED_TREE (child_fn
) = bind
;
9340 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9342 /* Remap src and dst argument types if needed. */
9343 record_type
= ctx
->record_type
;
9344 srecord_type
= ctx
->srecord_type
;
9345 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9346 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9348 record_needs_remap
= true;
9351 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9352 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9354 srecord_needs_remap
= true;
9358 if (record_needs_remap
|| srecord_needs_remap
)
9360 memset (&tcctx
, '\0', sizeof (tcctx
));
9361 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9362 tcctx
.cb
.dst_fn
= child_fn
;
9363 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9364 gcc_checking_assert (tcctx
.cb
.src_node
);
9365 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9366 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9367 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9368 tcctx
.cb
.eh_lp_nr
= 0;
9369 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9370 tcctx
.cb
.decl_map
= pointer_map_create ();
9373 if (record_needs_remap
)
9374 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9375 if (srecord_needs_remap
)
9376 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9379 tcctx
.cb
.decl_map
= NULL
;
9381 arg
= DECL_ARGUMENTS (child_fn
);
9382 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9383 sarg
= DECL_CHAIN (arg
);
9384 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9386 /* First pass: initialize temporaries used in record_type and srecord_type
9387 sizes and field offsets. */
9388 if (tcctx
.cb
.decl_map
)
9389 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9390 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9394 decl
= OMP_CLAUSE_DECL (c
);
9395 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9398 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9399 sf
= (tree
) n
->value
;
9400 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9401 src
= build_simple_mem_ref_loc (loc
, sarg
);
9402 src
= omp_build_component_ref (src
, sf
);
9403 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9404 append_to_statement_list (t
, &list
);
9407 /* Second pass: copy shared var pointers and copy construct non-VLA
9408 firstprivate vars. */
9409 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9410 switch (OMP_CLAUSE_CODE (c
))
9412 case OMP_CLAUSE_SHARED
:
9413 decl
= OMP_CLAUSE_DECL (c
);
9414 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9417 f
= (tree
) n
->value
;
9418 if (tcctx
.cb
.decl_map
)
9419 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9420 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9421 sf
= (tree
) n
->value
;
9422 if (tcctx
.cb
.decl_map
)
9423 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9424 src
= build_simple_mem_ref_loc (loc
, sarg
);
9425 src
= omp_build_component_ref (src
, sf
);
9426 dst
= build_simple_mem_ref_loc (loc
, arg
);
9427 dst
= omp_build_component_ref (dst
, f
);
9428 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9429 append_to_statement_list (t
, &list
);
9431 case OMP_CLAUSE_FIRSTPRIVATE
:
9432 decl
= OMP_CLAUSE_DECL (c
);
9433 if (is_variable_sized (decl
))
9435 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9438 f
= (tree
) n
->value
;
9439 if (tcctx
.cb
.decl_map
)
9440 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9441 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9444 sf
= (tree
) n
->value
;
9445 if (tcctx
.cb
.decl_map
)
9446 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9447 src
= build_simple_mem_ref_loc (loc
, sarg
);
9448 src
= omp_build_component_ref (src
, sf
);
9449 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9450 src
= build_simple_mem_ref_loc (loc
, src
);
9454 dst
= build_simple_mem_ref_loc (loc
, arg
);
9455 dst
= omp_build_component_ref (dst
, f
);
9456 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9457 append_to_statement_list (t
, &list
);
9459 case OMP_CLAUSE_PRIVATE
:
9460 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9462 decl
= OMP_CLAUSE_DECL (c
);
9463 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9464 f
= (tree
) n
->value
;
9465 if (tcctx
.cb
.decl_map
)
9466 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9467 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9470 sf
= (tree
) n
->value
;
9471 if (tcctx
.cb
.decl_map
)
9472 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9473 src
= build_simple_mem_ref_loc (loc
, sarg
);
9474 src
= omp_build_component_ref (src
, sf
);
9475 if (use_pointer_for_field (decl
, NULL
))
9476 src
= build_simple_mem_ref_loc (loc
, src
);
9480 dst
= build_simple_mem_ref_loc (loc
, arg
);
9481 dst
= omp_build_component_ref (dst
, f
);
9482 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9483 append_to_statement_list (t
, &list
);
9489 /* Last pass: handle VLA firstprivates. */
9490 if (tcctx
.cb
.decl_map
)
9491 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9492 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9496 decl
= OMP_CLAUSE_DECL (c
);
9497 if (!is_variable_sized (decl
))
9499 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9502 f
= (tree
) n
->value
;
9503 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9504 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9505 ind
= DECL_VALUE_EXPR (decl
);
9506 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9507 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9508 n
= splay_tree_lookup (ctx
->sfield_map
,
9509 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9510 sf
= (tree
) n
->value
;
9511 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9512 src
= build_simple_mem_ref_loc (loc
, sarg
);
9513 src
= omp_build_component_ref (src
, sf
);
9514 src
= build_simple_mem_ref_loc (loc
, src
);
9515 dst
= build_simple_mem_ref_loc (loc
, arg
);
9516 dst
= omp_build_component_ref (dst
, f
);
9517 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9518 append_to_statement_list (t
, &list
);
9519 n
= splay_tree_lookup (ctx
->field_map
,
9520 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9521 df
= (tree
) n
->value
;
9522 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9523 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9524 ptr
= omp_build_component_ref (ptr
, df
);
9525 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9526 build_fold_addr_expr_loc (loc
, dst
));
9527 append_to_statement_list (t
, &list
);
9530 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9531 append_to_statement_list (t
, &list
);
9533 if (tcctx
.cb
.decl_map
)
9534 pointer_map_destroy (tcctx
.cb
.decl_map
);
9535 pop_gimplify_context (NULL
);
9536 BIND_EXPR_BODY (bind
) = list
;
9541 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9545 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9547 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9549 gcc_assert (clauses
);
9550 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9551 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9552 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9554 case OMP_CLAUSE_DEPEND_IN
:
9557 case OMP_CLAUSE_DEPEND_OUT
:
9558 case OMP_CLAUSE_DEPEND_INOUT
:
9564 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9565 tree array
= create_tmp_var (type
, NULL
);
9566 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9568 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9569 gimple_seq_add_stmt (iseq
, g
);
9570 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9572 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9573 gimple_seq_add_stmt (iseq
, g
);
9574 for (i
= 0; i
< 2; i
++)
9576 if ((i
? n_in
: n_out
) == 0)
9578 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9579 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9580 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9582 tree t
= OMP_CLAUSE_DECL (c
);
9583 t
= fold_convert (ptr_type_node
, t
);
9584 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9585 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9586 NULL_TREE
, NULL_TREE
);
9587 g
= gimple_build_assign (r
, t
);
9588 gimple_seq_add_stmt (iseq
, g
);
9591 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9592 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9593 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9594 OMP_CLAUSE_CHAIN (c
) = *p
;
9596 tree clobber
= build_constructor (type
, NULL
);
9597 TREE_THIS_VOLATILE (clobber
) = 1;
9598 g
= gimple_build_assign (array
, clobber
);
9599 gimple_seq_add_stmt (oseq
, g
);
9602 /* Lower the OpenMP parallel or task directive in the current statement
9603 in GSI_P. CTX holds context information for the directive. */
9606 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9610 gimple stmt
= gsi_stmt (*gsi_p
);
9611 gimple par_bind
, bind
, dep_bind
= NULL
;
9612 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9613 location_t loc
= gimple_location (stmt
);
9615 clauses
= gimple_omp_taskreg_clauses (stmt
);
9616 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9617 par_body
= gimple_bind_body (par_bind
);
9618 child_fn
= ctx
->cb
.dst_fn
;
9619 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9620 && !gimple_omp_parallel_combined_p (stmt
))
9622 struct walk_stmt_info wi
;
9625 memset (&wi
, 0, sizeof (wi
));
9628 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9630 gimple_omp_parallel_set_combined_p (stmt
, true);
9632 gimple_seq dep_ilist
= NULL
;
9633 gimple_seq dep_olist
= NULL
;
9634 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9635 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9637 push_gimplify_context ();
9638 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9639 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9642 if (ctx
->srecord_type
)
9643 create_task_copyfn (stmt
, ctx
);
9645 push_gimplify_context ();
9650 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9651 lower_omp (&par_body
, ctx
);
9652 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9653 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9655 /* Declare all the variables created by mapping and the variables
9656 declared in the scope of the parallel body. */
9657 record_vars_into (ctx
->block_vars
, child_fn
);
9658 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9660 if (ctx
->record_type
)
9663 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9664 : ctx
->record_type
, ".omp_data_o");
9665 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9666 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9667 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9672 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9673 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9675 if (ctx
->record_type
)
9677 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9678 TREE_THIS_VOLATILE (clobber
) = 1;
9679 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9683 /* Once all the expansions are done, sequence all the different
9684 fragments inside gimple_omp_body. */
9688 if (ctx
->record_type
)
9690 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9691 /* fixup_child_record_type might have changed receiver_decl's type. */
9692 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9693 gimple_seq_add_stmt (&new_body
,
9694 gimple_build_assign (ctx
->receiver_decl
, t
));
9697 gimple_seq_add_seq (&new_body
, par_ilist
);
9698 gimple_seq_add_seq (&new_body
, par_body
);
9699 gimple_seq_add_seq (&new_body
, par_rlist
);
9700 if (ctx
->cancellable
)
9701 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9702 gimple_seq_add_seq (&new_body
, par_olist
);
9703 new_body
= maybe_catch_exception (new_body
);
9704 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9705 gimple_omp_set_body (stmt
, new_body
);
9707 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9708 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9709 gimple_bind_add_seq (bind
, ilist
);
9710 gimple_bind_add_stmt (bind
, stmt
);
9711 gimple_bind_add_seq (bind
, olist
);
9713 pop_gimplify_context (NULL
);
9717 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9718 gimple_bind_add_stmt (dep_bind
, bind
);
9719 gimple_bind_add_seq (dep_bind
, dep_olist
);
9720 pop_gimplify_context (dep_bind
);
9724 /* Lower the OpenMP target directive in the current statement
9725 in GSI_P. CTX holds context information for the directive. */
9728 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9731 tree child_fn
, t
, c
;
9732 gimple stmt
= gsi_stmt (*gsi_p
);
9733 gimple tgt_bind
= NULL
, bind
;
9734 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9735 location_t loc
= gimple_location (stmt
);
9736 int kind
= gimple_omp_target_kind (stmt
);
9737 unsigned int map_cnt
= 0;
9739 clauses
= gimple_omp_target_clauses (stmt
);
9740 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9742 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9743 tgt_body
= gimple_bind_body (tgt_bind
);
9745 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9746 tgt_body
= gimple_omp_body (stmt
);
9747 child_fn
= ctx
->cb
.dst_fn
;
9749 push_gimplify_context ();
9751 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9752 switch (OMP_CLAUSE_CODE (c
))
9758 case OMP_CLAUSE_MAP
:
9760 case OMP_CLAUSE_FROM
:
9761 var
= OMP_CLAUSE_DECL (c
);
9764 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9765 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9771 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9773 tree var2
= DECL_VALUE_EXPR (var
);
9774 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9775 var2
= TREE_OPERAND (var2
, 0);
9776 gcc_assert (DECL_P (var2
));
9780 if (!maybe_lookup_field (var
, ctx
))
9783 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9785 x
= build_receiver_ref (var
, true, ctx
);
9786 tree new_var
= lookup_decl (var
, ctx
);
9787 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9788 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9789 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9790 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9791 x
= build_simple_mem_ref (x
);
9792 SET_DECL_VALUE_EXPR (new_var
, x
);
9793 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9798 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9800 target_nesting_level
++;
9801 lower_omp (&tgt_body
, ctx
);
9802 target_nesting_level
--;
9804 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9805 lower_omp (&tgt_body
, ctx
);
9807 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9809 /* Declare all the variables created by mapping and the variables
9810 declared in the scope of the target body. */
9811 record_vars_into (ctx
->block_vars
, child_fn
);
9812 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9817 if (ctx
->record_type
)
9820 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9821 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9822 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9823 t
= make_tree_vec (3);
9824 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9826 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9828 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9829 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9830 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9832 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9835 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9836 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9837 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9838 gimple_omp_target_set_data_arg (stmt
, t
);
9840 vec
<constructor_elt
, va_gc
> *vsize
;
9841 vec
<constructor_elt
, va_gc
> *vkind
;
9842 vec_alloc (vsize
, map_cnt
);
9843 vec_alloc (vkind
, map_cnt
);
9844 unsigned int map_idx
= 0;
9846 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9847 switch (OMP_CLAUSE_CODE (c
))
9853 case OMP_CLAUSE_MAP
:
9855 case OMP_CLAUSE_FROM
:
9857 ovar
= OMP_CLAUSE_DECL (c
);
9860 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9861 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9863 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9864 == get_base_address (ovar
));
9865 nc
= OMP_CLAUSE_CHAIN (c
);
9866 ovar
= OMP_CLAUSE_DECL (nc
);
9870 tree x
= build_sender_ref (ovar
, ctx
);
9872 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9873 gimplify_assign (x
, v
, &ilist
);
9879 if (DECL_SIZE (ovar
)
9880 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9882 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9883 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9884 ovar2
= TREE_OPERAND (ovar2
, 0);
9885 gcc_assert (DECL_P (ovar2
));
9888 if (!maybe_lookup_field (ovar
, ctx
))
9892 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9893 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9894 talign
= DECL_ALIGN_UNIT (ovar
);
9897 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9898 tree x
= build_sender_ref (ovar
, ctx
);
9899 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9900 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9901 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9902 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9904 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9906 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9907 mark_addressable (avar
);
9908 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9909 talign
= DECL_ALIGN_UNIT (avar
);
9910 avar
= build_fold_addr_expr (avar
);
9911 gimplify_assign (x
, avar
, &ilist
);
9913 else if (is_gimple_reg (var
))
9915 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9916 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9917 mark_addressable (avar
);
9918 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9919 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9920 gimplify_assign (avar
, var
, &ilist
);
9921 avar
= build_fold_addr_expr (avar
);
9922 gimplify_assign (x
, avar
, &ilist
);
9923 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9924 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9925 && !TYPE_READONLY (TREE_TYPE (var
)))
9927 x
= build_sender_ref (ovar
, ctx
);
9928 x
= build_simple_mem_ref (x
);
9929 gimplify_assign (var
, x
, &olist
);
9934 var
= build_fold_addr_expr (var
);
9935 gimplify_assign (x
, var
, &ilist
);
9938 tree s
= OMP_CLAUSE_SIZE (c
);
9940 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9941 s
= fold_convert (size_type_node
, s
);
9942 tree purpose
= size_int (map_idx
++);
9943 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9944 if (TREE_CODE (s
) != INTEGER_CST
)
9945 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9947 unsigned char tkind
= 0;
9948 switch (OMP_CLAUSE_CODE (c
))
9950 case OMP_CLAUSE_MAP
:
9951 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9954 tkind
= OMP_CLAUSE_MAP_TO
;
9956 case OMP_CLAUSE_FROM
:
9957 tkind
= OMP_CLAUSE_MAP_FROM
;
9962 talign
= ceil_log2 (talign
);
9963 tkind
|= talign
<< 3;
9964 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9965 build_int_cst (unsigned_char_type_node
,
9971 gcc_assert (map_idx
== map_cnt
);
9973 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9974 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9975 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9976 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9977 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9979 gimple_seq initlist
= NULL
;
9980 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9981 TREE_VEC_ELT (t
, 1)),
9982 &initlist
, true, NULL_TREE
);
9983 gimple_seq_add_seq (&ilist
, initlist
);
9985 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
9987 TREE_THIS_VOLATILE (clobber
) = 1;
9988 gimple_seq_add_stmt (&olist
,
9989 gimple_build_assign (TREE_VEC_ELT (t
, 1),
9993 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9994 TREE_THIS_VOLATILE (clobber
) = 1;
9995 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9999 /* Once all the expansions are done, sequence all the different
10000 fragments inside gimple_omp_body. */
10004 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
10006 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10007 /* fixup_child_record_type might have changed receiver_decl's type. */
10008 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10009 gimple_seq_add_stmt (&new_body
,
10010 gimple_build_assign (ctx
->receiver_decl
, t
));
10013 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10015 gimple_seq_add_seq (&new_body
, tgt_body
);
10016 new_body
= maybe_catch_exception (new_body
);
10018 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10019 new_body
= tgt_body
;
10020 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
10022 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10023 gimple_omp_set_body (stmt
, new_body
);
10026 bind
= gimple_build_bind (NULL
, NULL
,
10027 tgt_bind
? gimple_bind_block (tgt_bind
)
10029 gsi_replace (gsi_p
, bind
, true);
10030 gimple_bind_add_seq (bind
, ilist
);
10031 gimple_bind_add_stmt (bind
, stmt
);
10032 gimple_bind_add_seq (bind
, olist
);
10034 pop_gimplify_context (NULL
);
10037 /* Expand code for an OpenMP teams directive. */
10040 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10042 gimple teams_stmt
= gsi_stmt (*gsi_p
);
10043 push_gimplify_context ();
10045 tree block
= make_node (BLOCK
);
10046 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
10047 gsi_replace (gsi_p
, bind
, true);
10048 gimple_seq bind_body
= NULL
;
10049 gimple_seq dlist
= NULL
;
10050 gimple_seq olist
= NULL
;
10052 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10053 OMP_CLAUSE_NUM_TEAMS
);
10054 if (num_teams
== NULL_TREE
)
10055 num_teams
= build_int_cst (unsigned_type_node
, 0);
10058 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
10059 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
10060 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
10062 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10063 OMP_CLAUSE_THREAD_LIMIT
);
10064 if (thread_limit
== NULL_TREE
)
10065 thread_limit
= build_int_cst (unsigned_type_node
, 0);
10068 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
10069 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
10070 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
10074 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
10075 &bind_body
, &dlist
, ctx
, NULL
);
10076 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
10077 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
10078 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
10080 location_t loc
= gimple_location (teams_stmt
);
10081 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
10082 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
10083 gimple_set_location (call
, loc
);
10084 gimple_seq_add_stmt (&bind_body
, call
);
10086 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10087 gimple_omp_set_body (teams_stmt
, NULL
);
10088 gimple_seq_add_seq (&bind_body
, olist
);
10089 gimple_seq_add_seq (&bind_body
, dlist
);
10090 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10091 gimple_bind_set_body (bind
, bind_body
);
10093 pop_gimplify_context (bind
);
10095 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10096 BLOCK_VARS (block
) = ctx
->block_vars
;
10097 if (BLOCK_VARS (block
))
10098 TREE_USED (block
) = 1;
10102 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10103 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10104 of OpenMP context, but with task_shared_vars set. */
10107 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10112 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10113 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10116 if (task_shared_vars
10118 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10121 /* If a global variable has been privatized, TREE_CONSTANT on
10122 ADDR_EXPR might be wrong. */
10123 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10124 recompute_tree_invariant_for_addr_expr (t
);
10126 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10131 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10133 gimple stmt
= gsi_stmt (*gsi_p
);
10134 struct walk_stmt_info wi
;
10136 if (gimple_has_location (stmt
))
10137 input_location
= gimple_location (stmt
);
10139 if (task_shared_vars
)
10140 memset (&wi
, '\0', sizeof (wi
));
10142 /* If we have issued syntax errors, avoid doing any heavy lifting.
10143 Just replace the OpenMP directives with a NOP to avoid
10144 confusing RTL expansion. */
10145 if (seen_error () && is_gimple_omp (stmt
))
10147 gsi_replace (gsi_p
, gimple_build_nop (), true);
10151 switch (gimple_code (stmt
))
10154 if ((ctx
|| task_shared_vars
)
10155 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10156 ctx
? NULL
: &wi
, NULL
)
10157 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10158 ctx
? NULL
: &wi
, NULL
)))
10159 gimple_regimplify_operands (stmt
, gsi_p
);
10162 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10164 case GIMPLE_EH_FILTER
:
10165 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10168 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10169 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10171 case GIMPLE_TRANSACTION
:
10172 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10175 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10177 case GIMPLE_OMP_PARALLEL
:
10178 case GIMPLE_OMP_TASK
:
10179 ctx
= maybe_lookup_ctx (stmt
);
10181 if (ctx
->cancellable
)
10182 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10183 lower_omp_taskreg (gsi_p
, ctx
);
10185 case GIMPLE_OMP_FOR
:
10186 ctx
= maybe_lookup_ctx (stmt
);
10188 if (ctx
->cancellable
)
10189 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10190 lower_omp_for (gsi_p
, ctx
);
10192 case GIMPLE_OMP_SECTIONS
:
10193 ctx
= maybe_lookup_ctx (stmt
);
10195 if (ctx
->cancellable
)
10196 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10197 lower_omp_sections (gsi_p
, ctx
);
10199 case GIMPLE_OMP_SINGLE
:
10200 ctx
= maybe_lookup_ctx (stmt
);
10202 lower_omp_single (gsi_p
, ctx
);
10204 case GIMPLE_OMP_MASTER
:
10205 ctx
= maybe_lookup_ctx (stmt
);
10207 lower_omp_master (gsi_p
, ctx
);
10209 case GIMPLE_OMP_TASKGROUP
:
10210 ctx
= maybe_lookup_ctx (stmt
);
10212 lower_omp_taskgroup (gsi_p
, ctx
);
10214 case GIMPLE_OMP_ORDERED
:
10215 ctx
= maybe_lookup_ctx (stmt
);
10217 lower_omp_ordered (gsi_p
, ctx
);
10219 case GIMPLE_OMP_CRITICAL
:
10220 ctx
= maybe_lookup_ctx (stmt
);
10222 lower_omp_critical (gsi_p
, ctx
);
10224 case GIMPLE_OMP_ATOMIC_LOAD
:
10225 if ((ctx
|| task_shared_vars
)
10226 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10227 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10228 gimple_regimplify_operands (stmt
, gsi_p
);
10230 case GIMPLE_OMP_TARGET
:
10231 ctx
= maybe_lookup_ctx (stmt
);
10233 lower_omp_target (gsi_p
, ctx
);
10235 case GIMPLE_OMP_TEAMS
:
10236 ctx
= maybe_lookup_ctx (stmt
);
10238 lower_omp_teams (gsi_p
, ctx
);
10242 fndecl
= gimple_call_fndecl (stmt
);
10244 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10245 switch (DECL_FUNCTION_CODE (fndecl
))
10247 case BUILT_IN_GOMP_BARRIER
:
10251 case BUILT_IN_GOMP_CANCEL
:
10252 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10255 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10256 cctx
= cctx
->outer
;
10257 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10258 if (!cctx
->cancellable
)
10260 if (DECL_FUNCTION_CODE (fndecl
)
10261 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10263 stmt
= gimple_build_nop ();
10264 gsi_replace (gsi_p
, stmt
, false);
10268 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10270 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10271 gimple_call_set_fndecl (stmt
, fndecl
);
10272 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10275 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10276 gimple_call_set_lhs (stmt
, lhs
);
10277 tree fallthru_label
;
10278 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10280 g
= gimple_build_label (fallthru_label
);
10281 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10282 g
= gimple_build_cond (NE_EXPR
, lhs
,
10283 fold_convert (TREE_TYPE (lhs
),
10284 boolean_false_node
),
10285 cctx
->cancel_label
, fallthru_label
);
10286 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10293 if ((ctx
|| task_shared_vars
)
10294 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10297 /* Just remove clobbers, this should happen only if we have
10298 "privatized" local addressable variables in SIMD regions,
10299 the clobber isn't needed in that case and gimplifying address
10300 of the ARRAY_REF into a pointer and creating MEM_REF based
10301 clobber would create worse code than we get with the clobber
10303 if (gimple_clobber_p (stmt
))
10305 gsi_replace (gsi_p
, gimple_build_nop (), true);
10308 gimple_regimplify_operands (stmt
, gsi_p
);
10315 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10317 location_t saved_location
= input_location
;
10318 gimple_stmt_iterator gsi
;
10319 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10320 lower_omp_1 (&gsi
, ctx
);
10321 /* During gimplification, we have not always invoked fold_stmt
10322 (gimplify.c:maybe_fold_stmt); call it now. */
10323 if (target_nesting_level
)
10324 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10326 input_location
= saved_location
;
10329 /* Main entry point. */
10331 static unsigned int
10332 execute_lower_omp (void)
10338 /* This pass always runs, to provide PROP_gimple_lomp.
10339 But there is nothing to do unless -fopenmp is given. */
10340 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10343 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10344 delete_omp_context
);
10346 body
= gimple_body (current_function_decl
);
10347 scan_omp (&body
, NULL
);
10348 gcc_assert (taskreg_nesting_level
== 0);
10349 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
10350 finish_taskreg_scan (ctx
);
10351 taskreg_contexts
.release ();
10353 if (all_contexts
->root
)
10355 if (task_shared_vars
)
10356 push_gimplify_context ();
10357 lower_omp (&body
, NULL
);
10358 if (task_shared_vars
)
10359 pop_gimplify_context (NULL
);
10364 splay_tree_delete (all_contexts
);
10365 all_contexts
= NULL
;
10367 BITMAP_FREE (task_shared_vars
);
10373 const pass_data pass_data_lower_omp
=
10375 GIMPLE_PASS
, /* type */
10376 "omplower", /* name */
10377 OPTGROUP_NONE
, /* optinfo_flags */
10378 false, /* has_gate */
10379 true, /* has_execute */
10380 TV_NONE
, /* tv_id */
10381 PROP_gimple_any
, /* properties_required */
10382 PROP_gimple_lomp
, /* properties_provided */
10383 0, /* properties_destroyed */
10384 0, /* todo_flags_start */
10385 0, /* todo_flags_finish */
10388 class pass_lower_omp
: public gimple_opt_pass
10391 pass_lower_omp (gcc::context
*ctxt
)
10392 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10395 /* opt_pass methods: */
10396 unsigned int execute () { return execute_lower_omp (); }
10398 }; // class pass_lower_omp
10400 } // anon namespace
10403 make_pass_lower_omp (gcc::context
*ctxt
)
10405 return new pass_lower_omp (ctxt
);
10408 /* The following is a utility to diagnose OpenMP structured block violations.
10409 It is not part of the "omplower" pass, as that's invoked too late. It
10410 should be invoked by the respective front ends after gimplification. */
10412 static splay_tree all_labels
;
10414 /* Check for mismatched contexts and generate an error if needed. Return
10415 true if an error is detected. */
10418 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10419 gimple branch_ctx
, gimple label_ctx
)
10421 if (label_ctx
== branch_ctx
)
10426 Previously we kept track of the label's entire context in diagnose_sb_[12]
10427 so we could traverse it and issue a correct "exit" or "enter" error
10428 message upon a structured block violation.
10430 We built the context by building a list with tree_cons'ing, but there is
10431 no easy counterpart in gimple tuples. It seems like far too much work
10432 for issuing exit/enter error messages. If someone really misses the
10433 distinct error message... patches welcome.
10437 /* Try to avoid confusing the user by producing and error message
10438 with correct "exit" or "enter" verbiage. We prefer "exit"
10439 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10440 if (branch_ctx
== NULL
)
10446 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10451 label_ctx
= TREE_CHAIN (label_ctx
);
10456 error ("invalid exit from OpenMP structured block");
10458 error ("invalid entry to OpenMP structured block");
10461 bool cilkplus_block
= false;
10465 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10466 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10468 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10469 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10470 cilkplus_block
= true;
10473 /* If it's obvious we have an invalid entry, be specific about the error. */
10474 if (branch_ctx
== NULL
)
10476 if (cilkplus_block
)
10477 error ("invalid entry to Cilk Plus structured block");
10479 error ("invalid entry to OpenMP structured block");
10483 /* Otherwise, be vague and lazy, but efficient. */
10484 if (cilkplus_block
)
10485 error ("invalid branch to/from a Cilk Plus structured block");
10487 error ("invalid branch to/from an OpenMP structured block");
10490 gsi_replace (gsi_p
, gimple_build_nop (), false);
10494 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10495 where each label is found. */
10498 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10499 struct walk_stmt_info
*wi
)
10501 gimple context
= (gimple
) wi
->info
;
10502 gimple inner_context
;
10503 gimple stmt
= gsi_stmt (*gsi_p
);
10505 *handled_ops_p
= true;
10507 switch (gimple_code (stmt
))
10511 case GIMPLE_OMP_PARALLEL
:
10512 case GIMPLE_OMP_TASK
:
10513 case GIMPLE_OMP_SECTIONS
:
10514 case GIMPLE_OMP_SINGLE
:
10515 case GIMPLE_OMP_SECTION
:
10516 case GIMPLE_OMP_MASTER
:
10517 case GIMPLE_OMP_ORDERED
:
10518 case GIMPLE_OMP_CRITICAL
:
10519 case GIMPLE_OMP_TARGET
:
10520 case GIMPLE_OMP_TEAMS
:
10521 case GIMPLE_OMP_TASKGROUP
:
10522 /* The minimal context here is just the current OMP construct. */
10523 inner_context
= stmt
;
10524 wi
->info
= inner_context
;
10525 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10526 wi
->info
= context
;
10529 case GIMPLE_OMP_FOR
:
10530 inner_context
= stmt
;
10531 wi
->info
= inner_context
;
10532 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10534 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10535 diagnose_sb_1
, NULL
, wi
);
10536 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10537 wi
->info
= context
;
10541 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10542 (splay_tree_value
) context
);
10552 /* Pass 2: Check each branch and see if its context differs from that of
10553 the destination label's context. */
10556 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10557 struct walk_stmt_info
*wi
)
10559 gimple context
= (gimple
) wi
->info
;
10561 gimple stmt
= gsi_stmt (*gsi_p
);
10563 *handled_ops_p
= true;
10565 switch (gimple_code (stmt
))
10569 case GIMPLE_OMP_PARALLEL
:
10570 case GIMPLE_OMP_TASK
:
10571 case GIMPLE_OMP_SECTIONS
:
10572 case GIMPLE_OMP_SINGLE
:
10573 case GIMPLE_OMP_SECTION
:
10574 case GIMPLE_OMP_MASTER
:
10575 case GIMPLE_OMP_ORDERED
:
10576 case GIMPLE_OMP_CRITICAL
:
10577 case GIMPLE_OMP_TARGET
:
10578 case GIMPLE_OMP_TEAMS
:
10579 case GIMPLE_OMP_TASKGROUP
:
10581 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10582 wi
->info
= context
;
10585 case GIMPLE_OMP_FOR
:
10587 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10589 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10590 diagnose_sb_2
, NULL
, wi
);
10591 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10592 wi
->info
= context
;
10597 tree lab
= gimple_cond_true_label (stmt
);
10600 n
= splay_tree_lookup (all_labels
,
10601 (splay_tree_key
) lab
);
10602 diagnose_sb_0 (gsi_p
, context
,
10603 n
? (gimple
) n
->value
: NULL
);
10605 lab
= gimple_cond_false_label (stmt
);
10608 n
= splay_tree_lookup (all_labels
,
10609 (splay_tree_key
) lab
);
10610 diagnose_sb_0 (gsi_p
, context
,
10611 n
? (gimple
) n
->value
: NULL
);
10618 tree lab
= gimple_goto_dest (stmt
);
10619 if (TREE_CODE (lab
) != LABEL_DECL
)
10622 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10623 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10627 case GIMPLE_SWITCH
:
10630 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10632 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10633 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10634 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10640 case GIMPLE_RETURN
:
10641 diagnose_sb_0 (gsi_p
, context
, NULL
);
10651 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10654 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10657 gimple last
= last_stmt (bb
);
10658 enum gimple_code code
= gimple_code (last
);
10659 struct omp_region
*cur_region
= *region
;
10660 bool fallthru
= false;
10664 case GIMPLE_OMP_PARALLEL
:
10665 case GIMPLE_OMP_TASK
:
10666 case GIMPLE_OMP_FOR
:
10667 case GIMPLE_OMP_SINGLE
:
10668 case GIMPLE_OMP_TEAMS
:
10669 case GIMPLE_OMP_MASTER
:
10670 case GIMPLE_OMP_TASKGROUP
:
10671 case GIMPLE_OMP_ORDERED
:
10672 case GIMPLE_OMP_CRITICAL
:
10673 case GIMPLE_OMP_SECTION
:
10674 cur_region
= new_omp_region (bb
, code
, cur_region
);
10678 case GIMPLE_OMP_TARGET
:
10679 cur_region
= new_omp_region (bb
, code
, cur_region
);
10681 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10682 cur_region
= cur_region
->outer
;
10685 case GIMPLE_OMP_SECTIONS
:
10686 cur_region
= new_omp_region (bb
, code
, cur_region
);
10690 case GIMPLE_OMP_SECTIONS_SWITCH
:
10694 case GIMPLE_OMP_ATOMIC_LOAD
:
10695 case GIMPLE_OMP_ATOMIC_STORE
:
10699 case GIMPLE_OMP_RETURN
:
10700 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10701 somewhere other than the next block. This will be
10703 cur_region
->exit
= bb
;
10704 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10705 cur_region
= cur_region
->outer
;
10708 case GIMPLE_OMP_CONTINUE
:
10709 cur_region
->cont
= bb
;
10710 switch (cur_region
->type
)
10712 case GIMPLE_OMP_FOR
:
10713 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10714 succs edges as abnormal to prevent splitting
10716 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10717 /* Make the loopback edge. */
10718 make_edge (bb
, single_succ (cur_region
->entry
),
10721 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10722 corresponds to the case that the body of the loop
10723 is not executed at all. */
10724 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10725 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10729 case GIMPLE_OMP_SECTIONS
:
10730 /* Wire up the edges into and out of the nested sections. */
10732 basic_block switch_bb
= single_succ (cur_region
->entry
);
10734 struct omp_region
*i
;
10735 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10737 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10738 make_edge (switch_bb
, i
->entry
, 0);
10739 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10742 /* Make the loopback edge to the block with
10743 GIMPLE_OMP_SECTIONS_SWITCH. */
10744 make_edge (bb
, switch_bb
, 0);
10746 /* Make the edge from the switch to exit. */
10747 make_edge (switch_bb
, bb
->next_bb
, 0);
10753 gcc_unreachable ();
10758 gcc_unreachable ();
10761 if (*region
!= cur_region
)
10763 *region
= cur_region
;
10765 *region_idx
= cur_region
->entry
->index
;
10773 static unsigned int
10774 diagnose_omp_structured_block_errors (void)
10776 struct walk_stmt_info wi
;
10777 gimple_seq body
= gimple_body (current_function_decl
);
10779 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10781 memset (&wi
, 0, sizeof (wi
));
10782 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10784 memset (&wi
, 0, sizeof (wi
));
10785 wi
.want_locations
= true;
10786 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10788 gimple_set_body (current_function_decl
, body
);
10790 splay_tree_delete (all_labels
);
10797 gate_diagnose_omp_blocks (void)
10799 return flag_openmp
|| flag_cilkplus
;
10804 const pass_data pass_data_diagnose_omp_blocks
=
10806 GIMPLE_PASS
, /* type */
10807 "*diagnose_omp_blocks", /* name */
10808 OPTGROUP_NONE
, /* optinfo_flags */
10809 true, /* has_gate */
10810 true, /* has_execute */
10811 TV_NONE
, /* tv_id */
10812 PROP_gimple_any
, /* properties_required */
10813 0, /* properties_provided */
10814 0, /* properties_destroyed */
10815 0, /* todo_flags_start */
10816 0, /* todo_flags_finish */
10819 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10822 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10823 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10826 /* opt_pass methods: */
10827 bool gate () { return gate_diagnose_omp_blocks (); }
10828 unsigned int execute () {
10829 return diagnose_omp_structured_block_errors ();
10832 }; // class pass_diagnose_omp_blocks
10834 } // anon namespace
10837 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10839 return new pass_diagnose_omp_blocks (ctxt
);
10842 /* SIMD clone supporting code. */
10844 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10845 of arguments to reserve space for. */
10847 static struct cgraph_simd_clone
*
10848 simd_clone_struct_alloc (int nargs
)
10850 struct cgraph_simd_clone
*clone_info
;
10851 size_t len
= (sizeof (struct cgraph_simd_clone
)
10852 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
10853 clone_info
= (struct cgraph_simd_clone
*)
10854 ggc_internal_cleared_alloc (len
);
10858 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10861 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
10862 struct cgraph_simd_clone
*from
)
10864 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
10865 + ((from
->nargs
- from
->inbranch
)
10866 * sizeof (struct cgraph_simd_clone_arg
))));
10869 /* Return vector of parameter types of function FNDECL. This uses
10870 TYPE_ARG_TYPES if available, otherwise falls back to types of
10871 DECL_ARGUMENTS types. */
10874 simd_clone_vector_of_formal_parm_types (tree fndecl
)
10876 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
10877 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
10878 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
10881 FOR_EACH_VEC_ELT (args
, i
, arg
)
10882 args
[i
] = TREE_TYPE (args
[i
]);
10886 /* Given a simd function in NODE, extract the simd specific
10887 information from the OMP clauses passed in CLAUSES, and return
10888 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10889 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10890 otherwise set to FALSE. */
10892 static struct cgraph_simd_clone
*
10893 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
10894 bool *inbranch_specified
)
10896 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
10899 *inbranch_specified
= false;
10901 n
= args
.length ();
10902 if (n
> 0 && args
.last () == void_type_node
)
10905 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10906 be cloned have a distinctive artificial label in addition to "omp
10910 && lookup_attribute ("cilk simd function",
10911 DECL_ATTRIBUTES (node
->decl
)));
10913 /* Allocate one more than needed just in case this is an in-branch
10914 clone which will require a mask argument. */
10915 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
10916 clone_info
->nargs
= n
;
10917 clone_info
->cilk_elemental
= cilk_clone
;
10924 clauses
= TREE_VALUE (clauses
);
10925 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
10928 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
10930 switch (OMP_CLAUSE_CODE (t
))
10932 case OMP_CLAUSE_INBRANCH
:
10933 clone_info
->inbranch
= 1;
10934 *inbranch_specified
= true;
10936 case OMP_CLAUSE_NOTINBRANCH
:
10937 clone_info
->inbranch
= 0;
10938 *inbranch_specified
= true;
10940 case OMP_CLAUSE_SIMDLEN
:
10941 clone_info
->simdlen
10942 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
10944 case OMP_CLAUSE_LINEAR
:
10946 tree decl
= OMP_CLAUSE_DECL (t
);
10947 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
10948 int argno
= TREE_INT_CST_LOW (decl
);
10949 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
10951 clone_info
->args
[argno
].arg_type
10952 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
10953 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10954 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
10955 && clone_info
->args
[argno
].linear_step
< n
);
10959 if (POINTER_TYPE_P (args
[argno
]))
10960 step
= fold_convert (ssizetype
, step
);
10961 if (!tree_fits_shwi_p (step
))
10963 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10964 "ignoring large linear step");
10968 else if (integer_zerop (step
))
10970 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10971 "ignoring zero linear step");
10977 clone_info
->args
[argno
].arg_type
10978 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
10979 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10984 case OMP_CLAUSE_UNIFORM
:
10986 tree decl
= OMP_CLAUSE_DECL (t
);
10987 int argno
= tree_to_uhwi (decl
);
10988 clone_info
->args
[argno
].arg_type
10989 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
10992 case OMP_CLAUSE_ALIGNED
:
10994 tree decl
= OMP_CLAUSE_DECL (t
);
10995 int argno
= tree_to_uhwi (decl
);
10996 clone_info
->args
[argno
].alignment
10997 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
11008 /* Given a SIMD clone in NODE, calculate the characteristic data
11009 type and return the coresponding type. The characteristic data
11010 type is computed as described in the Intel Vector ABI. */
11013 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
11014 struct cgraph_simd_clone
*clone_info
)
11016 tree type
= integer_type_node
;
11017 tree fndecl
= node
->decl
;
11019 /* a) For non-void function, the characteristic data type is the
11021 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
11022 type
= TREE_TYPE (TREE_TYPE (fndecl
));
11024 /* b) If the function has any non-uniform, non-linear parameters,
11025 then the characteristic data type is the type of the first
11029 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
11030 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
11031 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
11039 /* c) If the characteristic data type determined by a) or b) above
11040 is struct, union, or class type which is pass-by-value (except
11041 for the type that maps to the built-in complex data type), the
11042 characteristic data type is int. */
11043 if (RECORD_OR_UNION_TYPE_P (type
)
11044 && !aggregate_value_p (type
, NULL
)
11045 && TREE_CODE (type
) != COMPLEX_TYPE
)
11046 return integer_type_node
;
11048 /* d) If none of the above three classes is applicable, the
11049 characteristic data type is int. */
11053 /* e) For Intel Xeon Phi native and offload compilation, if the
11054 resulting characteristic data type is 8-bit or 16-bit integer
11055 data type, the characteristic data type is int. */
11056 /* Well, we don't handle Xeon Phi yet. */
11060 simd_clone_mangle (struct cgraph_node
*node
,
11061 struct cgraph_simd_clone
*clone_info
)
11063 char vecsize_mangle
= clone_info
->vecsize_mangle
;
11064 char mask
= clone_info
->inbranch
? 'M' : 'N';
11065 unsigned int simdlen
= clone_info
->simdlen
;
11069 gcc_assert (vecsize_mangle
&& simdlen
);
11071 pp_string (&pp
, "_ZGV");
11072 pp_character (&pp
, vecsize_mangle
);
11073 pp_character (&pp
, mask
);
11074 pp_decimal_int (&pp
, simdlen
);
11076 for (n
= 0; n
< clone_info
->nargs
; ++n
)
11078 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
11080 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
11081 pp_character (&pp
, 'u');
11082 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11084 gcc_assert (arg
.linear_step
!= 0);
11085 pp_character (&pp
, 'l');
11086 if (arg
.linear_step
> 1)
11087 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11088 else if (arg
.linear_step
< 0)
11090 pp_character (&pp
, 'n');
11091 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
11095 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11097 pp_character (&pp
, 's');
11098 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11101 pp_character (&pp
, 'v');
11104 pp_character (&pp
, 'a');
11105 pp_decimal_int (&pp
, arg
.alignment
);
11109 pp_underscore (&pp
);
11110 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
11113 pp_string (&pp
, str
);
11114 str
= pp_formatted_text (&pp
);
11116 /* If there already is a SIMD clone with the same mangled name, don't
11117 add another one. This can happen e.g. for
11118 #pragma omp declare simd
11119 #pragma omp declare simd simdlen(8)
11120 int foo (int, int);
11121 if the simdlen is assumed to be 8 for the first one, etc. */
11122 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11123 clone
= clone
->simdclone
->next_clone
)
11124 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11128 return get_identifier (str
);
11131 /* Create a simd clone of OLD_NODE and return it. */
11133 static struct cgraph_node
*
11134 simd_clone_create (struct cgraph_node
*old_node
)
11136 struct cgraph_node
*new_node
;
11137 if (old_node
->definition
)
11139 if (!cgraph_function_with_gimple_body_p (old_node
))
11141 cgraph_get_body (old_node
);
11142 new_node
= cgraph_function_versioning (old_node
, vNULL
, NULL
, NULL
,
11143 false, NULL
, NULL
, "simdclone");
11147 tree old_decl
= old_node
->decl
;
11148 tree new_decl
= copy_node (old_node
->decl
);
11149 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11150 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11151 SET_DECL_RTL (new_decl
, NULL
);
11152 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11153 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11155 = cgraph_copy_node_for_versioning (old_node
, new_decl
, vNULL
, NULL
);
11156 cgraph_call_function_insertion_hooks (new_node
);
11158 if (new_node
== NULL
)
11161 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11163 /* The function cgraph_function_versioning () will force the new
11164 symbol local. Undo this, and inherit external visability from
11166 new_node
->local
.local
= old_node
->local
.local
;
11167 new_node
->externally_visible
= old_node
->externally_visible
;
11172 /* Adjust the return type of the given function to its appropriate
11173 vector counterpart. Returns a simd array to be used throughout the
11174 function as a return value. */
11177 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11179 tree fndecl
= node
->decl
;
11180 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11181 unsigned int veclen
;
11184 /* Adjust the function return type. */
11185 if (orig_rettype
== void_type_node
)
11187 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11188 t
= TREE_TYPE (TREE_TYPE (fndecl
));
11189 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
11190 veclen
= node
->simdclone
->vecsize_int
;
11192 veclen
= node
->simdclone
->vecsize_float
;
11193 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
11194 if (veclen
> node
->simdclone
->simdlen
)
11195 veclen
= node
->simdclone
->simdlen
;
11196 if (POINTER_TYPE_P (t
))
11197 t
= pointer_sized_int_node
;
11198 if (veclen
== node
->simdclone
->simdlen
)
11199 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
11202 t
= build_vector_type (t
, veclen
);
11203 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11205 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11206 if (!node
->definition
)
11209 t
= DECL_RESULT (fndecl
);
11210 /* Adjust the DECL_RESULT. */
11211 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11212 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11215 tree atype
= build_array_type_nelts (orig_rettype
,
11216 node
->simdclone
->simdlen
);
11217 if (veclen
!= node
->simdclone
->simdlen
)
11218 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11220 /* Set up a SIMD array to use as the return value. */
11221 tree retval
= create_tmp_var_raw (atype
, "retval");
11222 gimple_add_tmp_var (retval
);
11226 /* Each vector argument has a corresponding array to be used locally
11227 as part of the eventual loop. Create such temporary array and
11230 PREFIX is the prefix to be used for the temporary.
11232 TYPE is the inner element type.
11234 SIMDLEN is the number of elements. */
11237 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11239 tree atype
= build_array_type_nelts (type
, simdlen
);
11240 tree avar
= create_tmp_var_raw (atype
, prefix
);
11241 gimple_add_tmp_var (avar
);
11245 /* Modify the function argument types to their corresponding vector
11246 counterparts if appropriate. Also, create one array for each simd
11247 argument to be used locally when using the function arguments as
11250 NODE is the function whose arguments are to be adjusted.
11252 Returns an adjustment vector that will be filled describing how the
11253 argument types will be adjusted. */
11255 static ipa_parm_adjustment_vec
11256 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11259 ipa_parm_adjustment_vec adjustments
;
11261 if (node
->definition
)
11262 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11264 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11265 adjustments
.create (args
.length ());
11266 unsigned i
, j
, veclen
;
11267 struct ipa_parm_adjustment adj
;
11268 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11270 memset (&adj
, 0, sizeof (adj
));
11271 tree parm
= args
[i
];
11272 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11273 adj
.base_index
= i
;
11276 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11277 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11279 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11281 /* No adjustment necessary for scalar arguments. */
11282 adj
.op
= IPA_PARM_OP_COPY
;
11286 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11287 veclen
= node
->simdclone
->vecsize_int
;
11289 veclen
= node
->simdclone
->vecsize_float
;
11290 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11291 if (veclen
> node
->simdclone
->simdlen
)
11292 veclen
= node
->simdclone
->simdlen
;
11293 adj
.arg_prefix
= "simd";
11294 if (POINTER_TYPE_P (parm_type
))
11295 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
11297 adj
.type
= build_vector_type (parm_type
, veclen
);
11298 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11299 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11301 adjustments
.safe_push (adj
);
11304 memset (&adj
, 0, sizeof (adj
));
11305 adj
.op
= IPA_PARM_OP_NEW
;
11306 adj
.arg_prefix
= "simd";
11307 adj
.base_index
= i
;
11308 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11312 if (node
->definition
)
11313 node
->simdclone
->args
[i
].simd_array
11314 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11315 parm_type
, node
->simdclone
->simdlen
);
11317 adjustments
.safe_push (adj
);
11320 if (node
->simdclone
->inbranch
)
11323 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11326 memset (&adj
, 0, sizeof (adj
));
11327 adj
.op
= IPA_PARM_OP_NEW
;
11328 adj
.arg_prefix
= "mask";
11330 adj
.base_index
= i
;
11331 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11332 veclen
= node
->simdclone
->vecsize_int
;
11334 veclen
= node
->simdclone
->vecsize_float
;
11335 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11336 if (veclen
> node
->simdclone
->simdlen
)
11337 veclen
= node
->simdclone
->simdlen
;
11338 if (POINTER_TYPE_P (base_type
))
11339 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
11341 adj
.type
= build_vector_type (base_type
, veclen
);
11342 adjustments
.safe_push (adj
);
11344 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11345 adjustments
.safe_push (adj
);
11347 /* We have previously allocated one extra entry for the mask. Use
11349 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11351 if (node
->definition
)
11353 sc
->args
[i
].orig_arg
11354 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11355 sc
->args
[i
].simd_array
11356 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11358 sc
->args
[i
].orig_type
= base_type
;
11359 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11362 if (node
->definition
)
11363 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11366 tree new_arg_types
= NULL_TREE
, new_reversed
;
11367 bool last_parm_void
= false;
11368 if (args
.length () > 0 && args
.last () == void_type_node
)
11369 last_parm_void
= true;
11371 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11372 j
= adjustments
.length ();
11373 for (i
= 0; i
< j
; i
++)
11375 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11377 if (adj
->op
== IPA_PARM_OP_COPY
)
11378 ptype
= args
[adj
->base_index
];
11381 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11383 new_reversed
= nreverse (new_arg_types
);
11384 if (last_parm_void
)
11387 TREE_CHAIN (new_arg_types
) = void_list_node
;
11389 new_reversed
= void_list_node
;
11392 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11393 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11394 TREE_TYPE (node
->decl
) = new_type
;
11396 adjustments
.release ();
11399 return adjustments
;
11402 /* Initialize and copy the function arguments in NODE to their
11403 corresponding local simd arrays. Returns a fresh gimple_seq with
11404 the instruction sequence generated. */
11407 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11408 ipa_parm_adjustment_vec adjustments
)
11410 gimple_seq seq
= NULL
;
11411 unsigned i
= 0, j
= 0, k
;
11413 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11415 arg
= DECL_CHAIN (arg
), i
++, j
++)
11417 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11420 node
->simdclone
->args
[i
].vector_arg
= arg
;
11422 tree array
= node
->simdclone
->args
[i
].simd_array
;
11423 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11425 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11426 tree ptr
= build_fold_addr_expr (array
);
11427 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11428 build_int_cst (ptype
, 0));
11429 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11430 gimplify_and_add (t
, &seq
);
11434 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11435 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11436 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11438 tree ptr
= build_fold_addr_expr (array
);
11442 arg
= DECL_CHAIN (arg
);
11446 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11447 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11448 build_int_cst (ptype
, k
* elemsize
));
11449 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11450 gimplify_and_add (t
, &seq
);
11457 /* Callback info for ipa_simd_modify_stmt_ops below. */
11459 struct modify_stmt_info
{
11460 ipa_parm_adjustment_vec adjustments
;
11462 /* True if the parent statement was modified by
11463 ipa_simd_modify_stmt_ops. */
11467 /* Callback for walk_gimple_op.
11469 Adjust operands from a given statement as specified in the
11470 adjustments vector in the callback data. */
11473 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11475 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11476 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11477 tree
*orig_tp
= tp
;
11478 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11479 tp
= &TREE_OPERAND (*tp
, 0);
11480 struct ipa_parm_adjustment
*cand
= NULL
;
11481 if (TREE_CODE (*tp
) == PARM_DECL
)
11482 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11486 *walk_subtrees
= 0;
11489 tree repl
= NULL_TREE
;
11491 repl
= unshare_expr (cand
->new_decl
);
11496 *walk_subtrees
= 0;
11497 bool modified
= info
->modified
;
11498 info
->modified
= false;
11499 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11500 if (!info
->modified
)
11502 info
->modified
= modified
;
11505 info
->modified
= modified
;
11514 repl
= build_fold_addr_expr (repl
);
11516 if (is_gimple_debug (info
->stmt
))
11518 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
11519 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
11520 DECL_ARTIFICIAL (vexpr
) = 1;
11521 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
11522 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
11527 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
11529 repl
= gimple_assign_lhs (stmt
);
11531 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11532 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11535 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11537 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11543 info
->modified
= true;
11547 /* Traverse the function body and perform all modifications as
11548 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11549 modified such that the replacement/reduction value will now be an
11550 offset into the corresponding simd_array.
11552 This function will replace all function argument uses with their
11553 corresponding simd array elements, and ajust the return values
11557 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11558 ipa_parm_adjustment_vec adjustments
,
11559 tree retval_array
, tree iter
)
11562 unsigned int i
, j
, l
;
11564 /* Re-use the adjustments array, but this time use it to replace
11565 every function argument use to an offset into the corresponding
11567 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11569 if (!node
->simdclone
->args
[i
].vector_arg
)
11572 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11573 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11574 adjustments
[j
].new_decl
11575 = build4 (ARRAY_REF
,
11577 node
->simdclone
->args
[i
].simd_array
,
11579 NULL_TREE
, NULL_TREE
);
11580 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11581 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11582 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11585 l
= adjustments
.length ();
11586 for (i
= 1; i
< num_ssa_names
; i
++)
11588 tree name
= ssa_name (i
);
11590 && SSA_NAME_VAR (name
)
11591 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11593 for (j
= 0; j
< l
; j
++)
11594 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11595 && adjustments
[j
].new_decl
)
11598 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11601 = copy_var_decl (adjustments
[j
].base
,
11602 DECL_NAME (adjustments
[j
].base
),
11603 TREE_TYPE (adjustments
[j
].base
));
11604 adjustments
[j
].new_ssa_base
= base_var
;
11607 base_var
= adjustments
[j
].new_ssa_base
;
11608 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11610 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11611 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11612 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11613 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11614 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11615 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11616 gimple stmt
= gimple_build_assign (name
, new_decl
);
11617 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11620 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11625 struct modify_stmt_info info
;
11626 info
.adjustments
= adjustments
;
11628 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11630 gimple_stmt_iterator gsi
;
11632 gsi
= gsi_start_bb (bb
);
11633 while (!gsi_end_p (gsi
))
11635 gimple stmt
= gsi_stmt (gsi
);
11637 struct walk_stmt_info wi
;
11639 memset (&wi
, 0, sizeof (wi
));
11640 info
.modified
= false;
11642 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11644 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11646 tree retval
= gimple_return_retval (stmt
);
11649 gsi_remove (&gsi
, true);
11653 /* Replace `return foo' with `retval_array[iter] = foo'. */
11654 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11655 retval_array
, iter
, NULL
, NULL
);
11656 stmt
= gimple_build_assign (ref
, retval
);
11657 gsi_replace (&gsi
, stmt
, true);
11658 info
.modified
= true;
11663 update_stmt (stmt
);
11664 if (maybe_clean_eh_stmt (stmt
))
11665 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11672 /* Adjust the argument types in NODE to their appropriate vector
11676 simd_clone_adjust (struct cgraph_node
*node
)
11678 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11680 targetm
.simd_clone
.adjust (node
);
11682 tree retval
= simd_clone_adjust_return_type (node
);
11683 ipa_parm_adjustment_vec adjustments
11684 = simd_clone_adjust_argument_types (node
);
11686 push_gimplify_context ();
11688 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11690 /* Adjust all uses of vector arguments accordingly. Adjust all
11691 return values accordingly. */
11692 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11693 tree iter1
= make_ssa_name (iter
, NULL
);
11694 tree iter2
= make_ssa_name (iter
, NULL
);
11695 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11697 /* Initialize the iteration variable. */
11698 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11699 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11700 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11701 /* Insert the SIMD array and iv initialization at function
11703 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11705 pop_gimplify_context (NULL
);
11707 /* Create a new BB right before the original exit BB, to hold the
11708 iteration increment and the condition/branch. */
11709 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11710 basic_block incr_bb
= create_empty_bb (orig_exit
);
11711 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11712 flag. Set it now to be a FALLTHRU_EDGE. */
11713 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11714 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11715 for (unsigned i
= 0;
11716 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
11718 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
11719 redirect_edge_succ (e
, incr_bb
);
11721 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
11722 e
->probability
= REG_BR_PROB_BASE
;
11723 gsi
= gsi_last_bb (incr_bb
);
11724 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
11725 build_int_cst (unsigned_type_node
,
11727 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11729 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11730 struct loop
*loop
= alloc_loop ();
11731 cfun
->has_force_vect_loops
= true;
11732 loop
->safelen
= node
->simdclone
->simdlen
;
11733 loop
->force_vect
= true;
11734 loop
->header
= body_bb
;
11735 add_bb_to_loop (incr_bb
, loop
);
11737 /* Branch around the body if the mask applies. */
11738 if (node
->simdclone
->inbranch
)
11740 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
11742 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
11743 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
11744 tree aref
= build4 (ARRAY_REF
,
11745 TREE_TYPE (TREE_TYPE (mask_array
)),
11748 g
= gimple_build_assign (mask
, aref
);
11749 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11750 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
11751 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
11753 aref
= build1 (VIEW_CONVERT_EXPR
,
11754 build_nonstandard_integer_type (bitsize
, 0), mask
);
11755 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
11756 g
= gimple_build_assign (mask
, aref
);
11757 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11760 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
11762 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11763 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
11764 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
11767 /* Generate the condition. */
11768 g
= gimple_build_cond (LT_EXPR
,
11770 build_int_cst (unsigned_type_node
,
11771 node
->simdclone
->simdlen
),
11773 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11774 e
= split_block (incr_bb
, gsi_stmt (gsi
));
11775 basic_block latch_bb
= e
->dest
;
11776 basic_block new_exit_bb
= e
->dest
;
11777 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
11778 loop
->latch
= latch_bb
;
11780 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
11782 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
11783 /* The successor of incr_bb is already pointing to latch_bb; just
11785 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11786 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
11788 gimple phi
= create_phi_node (iter1
, body_bb
);
11789 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
11790 edge latch_edge
= single_succ_edge (latch_bb
);
11791 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
11793 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11795 /* Generate the new return. */
11796 gsi
= gsi_last_bb (new_exit_bb
);
11798 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
11799 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
11800 retval
= TREE_OPERAND (retval
, 0);
11803 retval
= build1 (VIEW_CONVERT_EXPR
,
11804 TREE_TYPE (TREE_TYPE (node
->decl
)),
11806 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
11807 false, GSI_CONTINUE_LINKING
);
11809 g
= gimple_build_return (retval
);
11810 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11812 /* Handle aligned clauses by replacing default defs of the aligned
11813 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11814 lhs. Handle linear by adding PHIs. */
11815 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
11816 if (node
->simdclone
->args
[i
].alignment
11817 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
11818 && (node
->simdclone
->args
[i
].alignment
11819 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
11820 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
11823 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
11824 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11825 tree def
= ssa_default_def (cfun
, orig_arg
);
11826 if (def
&& !has_zero_uses (def
))
11828 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
11829 gimple_seq seq
= NULL
;
11830 bool need_cvt
= false;
11832 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
11834 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
11837 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
11838 gimple_call_set_lhs (g
, t
);
11839 gimple_seq_add_stmt_without_update (&seq
, g
);
11842 t
= make_ssa_name (orig_arg
, NULL
);
11843 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
11844 gimple_call_lhs (g
),
11846 gimple_seq_add_stmt_without_update (&seq
, g
);
11848 gsi_insert_seq_on_edge_immediate
11849 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
11851 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11852 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
11854 cgraph_create_edge (node
, cgraph_get_create_node (fn
),
11855 call
, entry_bb
->count
, freq
);
11857 imm_use_iterator iter
;
11858 use_operand_p use_p
;
11860 tree repl
= gimple_get_lhs (g
);
11861 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11862 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
11865 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11866 SET_USE (use_p
, repl
);
11869 else if (node
->simdclone
->args
[i
].arg_type
11870 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11872 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11873 tree def
= ssa_default_def (cfun
, orig_arg
);
11874 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11875 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
11876 if (def
&& !has_zero_uses (def
))
11878 iter1
= make_ssa_name (orig_arg
, NULL
);
11879 iter2
= make_ssa_name (orig_arg
, NULL
);
11880 phi
= create_phi_node (iter1
, body_bb
);
11881 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
11882 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11883 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11884 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
11885 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11886 ? TREE_TYPE (orig_arg
) : sizetype
;
11888 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
11889 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
11890 gsi
= gsi_last_bb (incr_bb
);
11891 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11893 imm_use_iterator iter
;
11894 use_operand_p use_p
;
11896 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11897 if (use_stmt
== phi
)
11900 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11901 SET_USE (use_p
, iter1
);
11905 calculate_dominance_info (CDI_DOMINATORS
);
11906 add_loop (loop
, loop
->header
->loop_father
);
11907 update_ssa (TODO_update_ssa
);
11912 /* If the function in NODE is tagged as an elemental SIMD function,
11913 create the appropriate SIMD clones. */
11916 expand_simd_clones (struct cgraph_node
*node
)
11918 tree attr
= lookup_attribute ("omp declare simd",
11919 DECL_ATTRIBUTES (node
->decl
));
11920 if (attr
== NULL_TREE
11921 || node
->global
.inlined_to
11922 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
11926 #pragma omp declare simd
11928 in C, there we don't know the argument types at all. */
11929 if (!node
->definition
11930 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
11935 /* Start with parsing the "omp declare simd" attribute(s). */
11936 bool inbranch_clause_specified
;
11937 struct cgraph_simd_clone
*clone_info
11938 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
11939 &inbranch_clause_specified
);
11940 if (clone_info
== NULL
)
11943 int orig_simdlen
= clone_info
->simdlen
;
11944 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
11945 /* The target can return 0 (no simd clones should be created),
11946 1 (just one ISA of simd clones should be created) or higher
11947 count of ISA variants. In that case, clone_info is initialized
11948 for the first ISA variant. */
11950 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
11955 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11956 also create one inbranch and one !inbranch clone of it. */
11957 for (int i
= 0; i
< count
* 2; i
++)
11959 struct cgraph_simd_clone
*clone
= clone_info
;
11960 if (inbranch_clause_specified
&& (i
& 1) != 0)
11965 clone
= simd_clone_struct_alloc (clone_info
->nargs
11967 simd_clone_struct_copy (clone
, clone_info
);
11968 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11969 and simd_clone_adjust_argument_types did to the first
11971 clone
->nargs
-= clone_info
->inbranch
;
11972 clone
->simdlen
= orig_simdlen
;
11973 /* And call the target hook again to get the right ISA. */
11974 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
11978 clone
->inbranch
= 1;
11981 /* simd_clone_mangle might fail if such a clone has been created
11983 tree id
= simd_clone_mangle (node
, clone
);
11984 if (id
== NULL_TREE
)
11987 /* Only when we are sure we want to create the clone actually
11988 clone the function (or definitions) or create another
11989 extern FUNCTION_DECL (for prototypes without definitions). */
11990 struct cgraph_node
*n
= simd_clone_create (node
);
11994 n
->simdclone
= clone
;
11995 clone
->origin
= node
;
11996 clone
->next_clone
= NULL
;
11997 if (node
->simd_clones
== NULL
)
11999 clone
->prev_clone
= n
;
12000 node
->simd_clones
= n
;
12004 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
12005 clone
->prev_clone
->simdclone
->next_clone
= n
;
12006 node
->simd_clones
->simdclone
->prev_clone
= n
;
12008 change_decl_assembler_name (n
->decl
, id
);
12009 /* And finally adjust the return type, parameters and for
12010 definitions also function body. */
12011 if (node
->definition
)
12012 simd_clone_adjust (n
);
12015 simd_clone_adjust_return_type (n
);
12016 simd_clone_adjust_argument_types (n
);
12020 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
12023 /* Entry point for IPA simd clone creation pass. */
12025 static unsigned int
12026 ipa_omp_simd_clone (void)
12028 struct cgraph_node
*node
;
12029 FOR_EACH_FUNCTION (node
)
12030 expand_simd_clones (node
);
12036 const pass_data pass_data_omp_simd_clone
=
12038 SIMPLE_IPA_PASS
, /* type */
12039 "simdclone", /* name */
12040 OPTGROUP_NONE
, /* optinfo_flags */
12041 true, /* has_gate */
12042 true, /* has_execute */
12043 TV_NONE
, /* tv_id */
12044 ( PROP_ssa
| PROP_cfg
), /* properties_required */
12045 0, /* properties_provided */
12046 0, /* properties_destroyed */
12047 0, /* todo_flags_start */
12048 0, /* todo_flags_finish */
12051 class pass_omp_simd_clone
: public simple_ipa_opt_pass
12054 pass_omp_simd_clone(gcc::context
*ctxt
)
12055 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
12058 /* opt_pass methods: */
12059 bool gate () { return ((flag_openmp
|| flag_openmp_simd
12060 || flag_cilkplus
|| (in_lto_p
&& !flag_wpa
))
12061 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
12063 unsigned int execute () { return ipa_omp_simd_clone (); }
12066 } // anon namespace
12068 simple_ipa_opt_pass
*
12069 make_pass_omp_simd_clone (gcc::context
*ctxt
)
12071 return new pass_omp_simd_clone (ctxt
);
12074 #include "gt-omp-low.h"