1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
37 #include "hard-reg-set.h"
40 #include "dominance.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
60 #include "plugin-api.h"
64 #include "tree-phinodes.h"
65 #include "ssa-iterators.h"
66 #include "tree-ssanames.h"
67 #include "tree-into-ssa.h"
73 #include "tree-pass.h"
75 #include "splay-tree.h"
76 #include "insn-codes.h"
80 #include "common/common-target.h"
82 #include "gimple-low.h"
83 #include "tree-cfgcleanup.h"
84 #include "pretty-print.h"
85 #include "alloc-pool.h"
86 #include "symbol-summary.h"
88 #include "tree-nested.h"
92 #include "lto-section-names.h"
95 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
96 phases. The first phase scans the function looking for OMP statements
97 and then for variables that must be replaced to satisfy data sharing
98 clauses. The second phase expands code for the constructs, as well as
99 re-gimplifying things when variables have been replaced with complex
102 Final code generation is done by pass_expand_omp. The flowgraph is
103 scanned for parallel regions which are then moved to a new
104 function, to be invoked by the thread library. */
106 /* Parallel region information. Every parallel and workshare
107 directive is enclosed between two markers, the OMP_* directive
108 and a corresponding OMP_RETURN statement. */
112 /* The enclosing region. */
113 struct omp_region
*outer
;
115 /* First child region. */
116 struct omp_region
*inner
;
118 /* Next peer region. */
119 struct omp_region
*next
;
121 /* Block containing the omp directive as its last stmt. */
124 /* Block containing the OMP_RETURN as its last stmt. */
127 /* Block containing the OMP_CONTINUE as its last stmt. */
130 /* If this is a combined parallel+workshare region, this is a list
131 of additional arguments needed by the combined parallel+workshare
133 vec
<tree
, va_gc
> *ws_args
;
135 /* The code for the omp directive of this region. */
136 enum gimple_code type
;
138 /* Schedule kind, only used for OMP_FOR type regions. */
139 enum omp_clause_schedule_kind sched_kind
;
141 /* True if this is a combined parallel+workshare region. */
142 bool is_combined_parallel
;
145 /* Context structure. Used to store information about each parallel
146 directive in the code. */
148 typedef struct omp_context
150 /* This field must be at the beginning, as we do "inheritance": Some
151 callback functions for tree-inline.c (e.g., omp_copy_decl)
152 receive a copy_body_data pointer that is up-casted to an
153 omp_context pointer. */
156 /* The tree of contexts corresponding to the encountered constructs. */
157 struct omp_context
*outer
;
160 /* Map variables to fields in a structure that allows communication
161 between sending and receiving threads. */
162 splay_tree field_map
;
167 /* These are used just by task contexts, if task firstprivate fn is
168 needed. srecord_type is used to communicate from the thread
169 that encountered the task construct to task firstprivate fn,
170 record_type is allocated by GOMP_task, initialized by task firstprivate
171 fn and passed to the task body fn. */
172 splay_tree sfield_map
;
175 /* A chain of variables to add to the top-level block surrounding the
176 construct. In the case of a parallel, this is in the child function. */
179 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
180 barriers should jump to during omplower pass. */
183 /* What to do with variables with implicitly determined sharing
185 enum omp_clause_default_kind default_kind
;
187 /* Nesting depth of this context. Used to beautify error messages re
188 invalid gotos. The outermost ctx is depth 1, with depth 0 being
189 reserved for the main body of the function. */
192 /* True if this parallel directive is nested within another. */
195 /* True if this construct can be cancelled. */
200 struct omp_for_data_loop
202 tree v
, n1
, n2
, step
;
203 enum tree_code cond_code
;
206 /* A structure describing the main elements of a parallel loop. */
210 struct omp_for_data_loop loop
;
215 bool have_nowait
, have_ordered
;
216 enum omp_clause_schedule_kind sched_kind
;
217 struct omp_for_data_loop
*loops
;
221 static splay_tree all_contexts
;
222 static int taskreg_nesting_level
;
223 static int target_nesting_level
;
224 static struct omp_region
*root_omp_region
;
225 static bitmap task_shared_vars
;
226 static vec
<omp_context
*> taskreg_contexts
;
228 static void scan_omp (gimple_seq
*, omp_context
*);
229 static tree
scan_omp_1_op (tree
*, int *, void *);
231 #define WALK_SUBSTMTS \
235 case GIMPLE_EH_FILTER: \
236 case GIMPLE_TRANSACTION: \
237 /* The sub-statements for these should be walked. */ \
238 *handled_ops_p = false; \
241 /* Holds offload tables with decls. */
242 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
244 /* Convenience function for calling scan_omp_1_op on tree operands. */
247 scan_omp_op (tree
*tp
, omp_context
*ctx
)
249 struct walk_stmt_info wi
;
251 memset (&wi
, 0, sizeof (wi
));
253 wi
.want_locations
= true;
255 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
258 static void lower_omp (gimple_seq
*, omp_context
*);
259 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
260 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
262 /* Find an OpenMP clause of type KIND within CLAUSES. */
265 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
267 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
268 if (OMP_CLAUSE_CODE (clauses
) == kind
)
274 /* Return true if CTX is for an omp parallel. */
277 is_parallel_ctx (omp_context
*ctx
)
279 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
283 /* Return true if CTX is for an omp target region. */
286 is_targetreg_ctx (omp_context
*ctx
)
288 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
289 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
;
293 /* Return true if CTX is for an omp task. */
296 is_task_ctx (omp_context
*ctx
)
298 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
302 /* Return true if CTX is for an omp parallel or omp task. */
305 is_taskreg_ctx (omp_context
*ctx
)
307 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
308 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
312 /* Return true if REGION is a combined parallel+workshare region. */
315 is_combined_parallel (struct omp_region
*region
)
317 return region
->is_combined_parallel
;
321 /* Extract the header elements of parallel loop FOR_STMT and store
325 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
326 struct omp_for_data_loop
*loops
)
328 tree t
, var
, *collapse_iter
, *collapse_count
;
329 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
330 struct omp_for_data_loop
*loop
;
332 struct omp_for_data_loop dummy_loop
;
333 location_t loc
= gimple_location (for_stmt
);
334 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
335 bool distribute
= gimple_omp_for_kind (for_stmt
)
336 == GF_OMP_FOR_KIND_DISTRIBUTE
;
338 fd
->for_stmt
= for_stmt
;
340 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
341 if (fd
->collapse
> 1)
344 fd
->loops
= &fd
->loop
;
346 fd
->have_nowait
= distribute
|| simd
;
347 fd
->have_ordered
= false;
348 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
349 fd
->chunk_size
= NULL_TREE
;
350 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
351 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
352 collapse_iter
= NULL
;
353 collapse_count
= NULL
;
355 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
356 switch (OMP_CLAUSE_CODE (t
))
358 case OMP_CLAUSE_NOWAIT
:
359 fd
->have_nowait
= true;
361 case OMP_CLAUSE_ORDERED
:
362 fd
->have_ordered
= true;
364 case OMP_CLAUSE_SCHEDULE
:
365 gcc_assert (!distribute
);
366 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
367 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
369 case OMP_CLAUSE_DIST_SCHEDULE
:
370 gcc_assert (distribute
);
371 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
373 case OMP_CLAUSE_COLLAPSE
:
374 if (fd
->collapse
> 1)
376 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
377 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
384 /* FIXME: for now map schedule(auto) to schedule(static).
385 There should be analysis to determine whether all iterations
386 are approximately the same amount of work (then schedule(static)
387 is best) or if it varies (then schedule(dynamic,N) is better). */
388 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
390 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
391 gcc_assert (fd
->chunk_size
== NULL
);
393 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
394 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
395 gcc_assert (fd
->chunk_size
== NULL
);
396 else if (fd
->chunk_size
== NULL
)
398 /* We only need to compute a default chunk size for ordered
399 static loops and dynamic loops. */
400 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
402 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
403 ? integer_zero_node
: integer_one_node
;
406 for (i
= 0; i
< fd
->collapse
; i
++)
408 if (fd
->collapse
== 1)
410 else if (loops
!= NULL
)
415 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
416 gcc_assert (SSA_VAR_P (loop
->v
));
417 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
418 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
419 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
420 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
422 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
423 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
424 switch (loop
->cond_code
)
430 gcc_assert (gimple_omp_for_kind (for_stmt
)
431 == GF_OMP_FOR_KIND_CILKSIMD
432 || (gimple_omp_for_kind (for_stmt
)
433 == GF_OMP_FOR_KIND_CILKFOR
));
436 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
437 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
439 loop
->n2
= fold_build2_loc (loc
,
440 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
441 build_int_cst (TREE_TYPE (loop
->n2
), 1));
442 loop
->cond_code
= LT_EXPR
;
445 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
446 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
448 loop
->n2
= fold_build2_loc (loc
,
449 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
450 build_int_cst (TREE_TYPE (loop
->n2
), 1));
451 loop
->cond_code
= GT_EXPR
;
457 t
= gimple_omp_for_incr (for_stmt
, i
);
458 gcc_assert (TREE_OPERAND (t
, 0) == var
);
459 switch (TREE_CODE (t
))
462 loop
->step
= TREE_OPERAND (t
, 1);
464 case POINTER_PLUS_EXPR
:
465 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
468 loop
->step
= TREE_OPERAND (t
, 1);
469 loop
->step
= fold_build1_loc (loc
,
470 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
478 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
479 && !fd
->have_ordered
))
481 if (fd
->collapse
== 1)
482 iter_type
= TREE_TYPE (loop
->v
);
484 || TYPE_PRECISION (iter_type
)
485 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
487 = build_nonstandard_integer_type
488 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
490 else if (iter_type
!= long_long_unsigned_type_node
)
492 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
493 iter_type
= long_long_unsigned_type_node
;
494 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
495 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
496 >= TYPE_PRECISION (iter_type
))
500 if (loop
->cond_code
== LT_EXPR
)
501 n
= fold_build2_loc (loc
,
502 PLUS_EXPR
, TREE_TYPE (loop
->v
),
503 loop
->n2
, loop
->step
);
506 if (TREE_CODE (n
) != INTEGER_CST
507 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
508 iter_type
= long_long_unsigned_type_node
;
510 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
511 > TYPE_PRECISION (iter_type
))
515 if (loop
->cond_code
== LT_EXPR
)
518 n2
= fold_build2_loc (loc
,
519 PLUS_EXPR
, TREE_TYPE (loop
->v
),
520 loop
->n2
, loop
->step
);
524 n1
= fold_build2_loc (loc
,
525 MINUS_EXPR
, TREE_TYPE (loop
->v
),
526 loop
->n2
, loop
->step
);
529 if (TREE_CODE (n1
) != INTEGER_CST
530 || TREE_CODE (n2
) != INTEGER_CST
531 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
532 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
533 iter_type
= long_long_unsigned_type_node
;
537 if (collapse_count
&& *collapse_count
== NULL
)
539 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
540 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
541 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
542 if (t
&& integer_zerop (t
))
543 count
= build_zero_cst (long_long_unsigned_type_node
);
544 else if ((i
== 0 || count
!= NULL_TREE
)
545 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
546 && TREE_CONSTANT (loop
->n1
)
547 && TREE_CONSTANT (loop
->n2
)
548 && TREE_CODE (loop
->step
) == INTEGER_CST
)
550 tree itype
= TREE_TYPE (loop
->v
);
552 if (POINTER_TYPE_P (itype
))
553 itype
= signed_type_for (itype
);
554 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
555 t
= fold_build2_loc (loc
,
557 fold_convert_loc (loc
, itype
, loop
->step
), t
);
558 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
559 fold_convert_loc (loc
, itype
, loop
->n2
));
560 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
561 fold_convert_loc (loc
, itype
, loop
->n1
));
562 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
563 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
564 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
565 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
566 fold_convert_loc (loc
, itype
,
569 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
570 fold_convert_loc (loc
, itype
, loop
->step
));
571 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
572 if (count
!= NULL_TREE
)
573 count
= fold_build2_loc (loc
,
574 MULT_EXPR
, long_long_unsigned_type_node
,
578 if (TREE_CODE (count
) != INTEGER_CST
)
581 else if (count
&& !integer_zerop (count
))
588 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
589 || fd
->have_ordered
))
591 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
592 iter_type
= long_long_unsigned_type_node
;
594 iter_type
= long_integer_type_node
;
596 else if (collapse_iter
&& *collapse_iter
!= NULL
)
597 iter_type
= TREE_TYPE (*collapse_iter
);
598 fd
->iter_type
= iter_type
;
599 if (collapse_iter
&& *collapse_iter
== NULL
)
600 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
601 if (collapse_count
&& *collapse_count
== NULL
)
604 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
606 *collapse_count
= create_tmp_var (iter_type
, ".count");
609 if (fd
->collapse
> 1)
611 fd
->loop
.v
= *collapse_iter
;
612 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
613 fd
->loop
.n2
= *collapse_count
;
614 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
615 fd
->loop
.cond_code
= LT_EXPR
;
620 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
621 is the immediate dominator of PAR_ENTRY_BB, return true if there
622 are no data dependencies that would prevent expanding the parallel
623 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
625 When expanding a combined parallel+workshare region, the call to
626 the child function may need additional arguments in the case of
627 GIMPLE_OMP_FOR regions. In some cases, these arguments are
628 computed out of variables passed in from the parent to the child
629 via 'struct .omp_data_s'. For instance:
631 #pragma omp parallel for schedule (guided, i * 4)
636 # BLOCK 2 (PAR_ENTRY_BB)
638 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
640 # BLOCK 3 (WS_ENTRY_BB)
641 .omp_data_i = &.omp_data_o;
642 D.1667 = .omp_data_i->i;
644 #pragma omp for schedule (guided, D.1598)
646 When we outline the parallel region, the call to the child function
647 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
648 that value is computed *after* the call site. So, in principle we
649 cannot do the transformation.
651 To see whether the code in WS_ENTRY_BB blocks the combined
652 parallel+workshare call, we collect all the variables used in the
653 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
654 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
657 FIXME. If we had the SSA form built at this point, we could merely
658 hoist the code in block 3 into block 2 and be done with it. But at
659 this point we don't have dataflow information and though we could
660 hack something up here, it is really not worth the aggravation. */
663 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
665 struct omp_for_data fd
;
666 gimple ws_stmt
= last_stmt (ws_entry_bb
);
668 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
671 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
673 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
675 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
677 if (fd
.iter_type
!= long_integer_type_node
)
680 /* FIXME. We give up too easily here. If any of these arguments
681 are not constants, they will likely involve variables that have
682 been mapped into fields of .omp_data_s for sharing with the child
683 function. With appropriate data flow, it would be possible to
685 if (!is_gimple_min_invariant (fd
.loop
.n1
)
686 || !is_gimple_min_invariant (fd
.loop
.n2
)
687 || !is_gimple_min_invariant (fd
.loop
.step
)
688 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
695 /* Collect additional arguments needed to emit a combined
696 parallel+workshare call. WS_STMT is the workshare directive being
699 static vec
<tree
, va_gc
> *
700 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
703 location_t loc
= gimple_location (ws_stmt
);
704 vec
<tree
, va_gc
> *ws_args
;
706 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
708 struct omp_for_data fd
;
711 extract_omp_for_data (for_stmt
, &fd
, NULL
);
715 if (gimple_omp_for_combined_into_p (for_stmt
))
718 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
719 OMP_CLAUSE__LOOPTEMP_
);
721 n1
= OMP_CLAUSE_DECL (innerc
);
722 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
723 OMP_CLAUSE__LOOPTEMP_
);
725 n2
= OMP_CLAUSE_DECL (innerc
);
728 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
730 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
731 ws_args
->quick_push (t
);
733 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
734 ws_args
->quick_push (t
);
736 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
737 ws_args
->quick_push (t
);
741 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
742 ws_args
->quick_push (t
);
747 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
749 /* Number of sections is equal to the number of edges from the
750 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
751 the exit of the sections region. */
752 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
753 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
754 vec_alloc (ws_args
, 1);
755 ws_args
->quick_push (t
);
763 /* Discover whether REGION is a combined parallel+workshare region. */
766 determine_parallel_type (struct omp_region
*region
)
768 basic_block par_entry_bb
, par_exit_bb
;
769 basic_block ws_entry_bb
, ws_exit_bb
;
771 if (region
== NULL
|| region
->inner
== NULL
772 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
773 || region
->inner
->cont
== NULL
)
776 /* We only support parallel+for and parallel+sections. */
777 if (region
->type
!= GIMPLE_OMP_PARALLEL
778 || (region
->inner
->type
!= GIMPLE_OMP_FOR
779 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
782 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
783 WS_EXIT_BB -> PAR_EXIT_BB. */
784 par_entry_bb
= region
->entry
;
785 par_exit_bb
= region
->exit
;
786 ws_entry_bb
= region
->inner
->entry
;
787 ws_exit_bb
= region
->inner
->exit
;
789 if (single_succ (par_entry_bb
) == ws_entry_bb
790 && single_succ (ws_exit_bb
) == par_exit_bb
791 && workshare_safe_to_combine_p (ws_entry_bb
)
792 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
793 || (last_and_only_stmt (ws_entry_bb
)
794 && last_and_only_stmt (par_exit_bb
))))
796 gimple par_stmt
= last_stmt (par_entry_bb
);
797 gimple ws_stmt
= last_stmt (ws_entry_bb
);
799 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
801 /* If this is a combined parallel loop, we need to determine
802 whether or not to use the combined library calls. There
803 are two cases where we do not apply the transformation:
804 static loops and any kind of ordered loop. In the first
805 case, we already open code the loop so there is no need
806 to do anything else. In the latter case, the combined
807 parallel loop call would still need extra synchronization
808 to implement ordered semantics, so there would not be any
809 gain in using the combined call. */
810 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
811 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
813 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
814 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
816 region
->is_combined_parallel
= false;
817 region
->inner
->is_combined_parallel
= false;
822 region
->is_combined_parallel
= true;
823 region
->inner
->is_combined_parallel
= true;
824 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
829 /* Return true if EXPR is variable sized. */
832 is_variable_sized (const_tree expr
)
834 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
837 /* Return true if DECL is a reference type. */
840 is_reference (tree decl
)
842 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
845 /* Lookup variables in the decl or field splay trees. The "maybe" form
846 allows for the variable form to not have been entered, otherwise we
847 assert that the variable must have been entered. */
850 lookup_decl (tree var
, omp_context
*ctx
)
852 tree
*n
= ctx
->cb
.decl_map
->get (var
);
857 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
859 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
860 return n
? *n
: NULL_TREE
;
864 lookup_field (tree var
, omp_context
*ctx
)
867 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
868 return (tree
) n
->value
;
872 lookup_sfield (tree var
, omp_context
*ctx
)
875 n
= splay_tree_lookup (ctx
->sfield_map
876 ? ctx
->sfield_map
: ctx
->field_map
,
877 (splay_tree_key
) var
);
878 return (tree
) n
->value
;
882 maybe_lookup_field (tree var
, omp_context
*ctx
)
885 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
886 return n
? (tree
) n
->value
: NULL_TREE
;
889 /* Return true if DECL should be copied by pointer. SHARED_CTX is
890 the parallel context if DECL is to be shared. */
893 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
895 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
898 /* We can only use copy-in/copy-out semantics for shared variables
899 when we know the value is not accessible from an outer scope. */
902 /* ??? Trivially accessible from anywhere. But why would we even
903 be passing an address in this case? Should we simply assert
904 this to be false, or should we have a cleanup pass that removes
905 these from the list of mappings? */
906 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
909 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
910 without analyzing the expression whether or not its location
911 is accessible to anyone else. In the case of nested parallel
912 regions it certainly may be. */
913 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
916 /* Do not use copy-in/copy-out for variables that have their
918 if (TREE_ADDRESSABLE (decl
))
921 /* lower_send_shared_vars only uses copy-in, but not copy-out
923 if (TREE_READONLY (decl
)
924 || ((TREE_CODE (decl
) == RESULT_DECL
925 || TREE_CODE (decl
) == PARM_DECL
)
926 && DECL_BY_REFERENCE (decl
)))
929 /* Disallow copy-in/out in nested parallel if
930 decl is shared in outer parallel, otherwise
931 each thread could store the shared variable
932 in its own copy-in location, making the
933 variable no longer really shared. */
934 if (shared_ctx
->is_nested
)
938 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
939 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
946 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
947 c
; c
= OMP_CLAUSE_CHAIN (c
))
948 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
949 && OMP_CLAUSE_DECL (c
) == decl
)
953 goto maybe_mark_addressable_and_ret
;
957 /* For tasks avoid using copy-in/out. As tasks can be
958 deferred or executed in different thread, when GOMP_task
959 returns, the task hasn't necessarily terminated. */
960 if (is_task_ctx (shared_ctx
))
963 maybe_mark_addressable_and_ret
:
964 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
965 if (is_gimple_reg (outer
))
967 /* Taking address of OUTER in lower_send_shared_vars
968 might need regimplification of everything that uses the
970 if (!task_shared_vars
)
971 task_shared_vars
= BITMAP_ALLOC (NULL
);
972 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
973 TREE_ADDRESSABLE (outer
) = 1;
982 /* Construct a new automatic decl similar to VAR. */
985 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
987 tree copy
= copy_var_decl (var
, name
, type
);
989 DECL_CONTEXT (copy
) = current_function_decl
;
990 DECL_CHAIN (copy
) = ctx
->block_vars
;
991 ctx
->block_vars
= copy
;
997 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
999 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1002 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1005 omp_build_component_ref (tree obj
, tree field
)
1007 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1008 if (TREE_THIS_VOLATILE (field
))
1009 TREE_THIS_VOLATILE (ret
) |= 1;
1010 if (TREE_READONLY (field
))
1011 TREE_READONLY (ret
) |= 1;
1015 /* Build tree nodes to access the field for VAR on the receiver side. */
1018 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1020 tree x
, field
= lookup_field (var
, ctx
);
1022 /* If the receiver record type was remapped in the child function,
1023 remap the field into the new record type. */
1024 x
= maybe_lookup_field (field
, ctx
);
1028 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1029 x
= omp_build_component_ref (x
, field
);
1031 x
= build_simple_mem_ref (x
);
1036 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1037 of a parallel, this is a component reference; for workshare constructs
1038 this is some variable. */
1041 build_outer_var_ref (tree var
, omp_context
*ctx
)
1045 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1047 else if (is_variable_sized (var
))
1049 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1050 x
= build_outer_var_ref (x
, ctx
);
1051 x
= build_simple_mem_ref (x
);
1053 else if (is_taskreg_ctx (ctx
))
1055 bool by_ref
= use_pointer_for_field (var
, NULL
);
1056 x
= build_receiver_ref (var
, by_ref
, ctx
);
1058 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1059 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1061 /* #pragma omp simd isn't a worksharing construct, and can reference even
1062 private vars in its linear etc. clauses. */
1064 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1065 x
= lookup_decl (var
, ctx
->outer
);
1066 else if (ctx
->outer
)
1067 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1071 else if (ctx
->outer
)
1072 x
= lookup_decl (var
, ctx
->outer
);
1073 else if (is_reference (var
))
1074 /* This can happen with orphaned constructs. If var is reference, it is
1075 possible it is shared and as such valid. */
1080 if (is_reference (var
))
1081 x
= build_simple_mem_ref (x
);
1086 /* Build tree nodes to access the field for VAR on the sender side. */
1089 build_sender_ref (tree var
, omp_context
*ctx
)
1091 tree field
= lookup_sfield (var
, ctx
);
1092 return omp_build_component_ref (ctx
->sender_decl
, field
);
1095 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1098 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1100 tree field
, type
, sfield
= NULL_TREE
;
1102 gcc_assert ((mask
& 1) == 0
1103 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1104 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1105 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1107 type
= TREE_TYPE (var
);
1110 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1111 type
= build_pointer_type (build_pointer_type (type
));
1114 type
= build_pointer_type (type
);
1115 else if ((mask
& 3) == 1 && is_reference (var
))
1116 type
= TREE_TYPE (type
);
1118 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1119 FIELD_DECL
, DECL_NAME (var
), type
);
1121 /* Remember what variable this field was created for. This does have a
1122 side effect of making dwarf2out ignore this member, so for helpful
1123 debugging we clear it later in delete_omp_context. */
1124 DECL_ABSTRACT_ORIGIN (field
) = var
;
1125 if (type
== TREE_TYPE (var
))
1127 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1128 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1129 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1132 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1134 if ((mask
& 3) == 3)
1136 insert_field_into_struct (ctx
->record_type
, field
);
1137 if (ctx
->srecord_type
)
1139 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1140 FIELD_DECL
, DECL_NAME (var
), type
);
1141 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1142 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1143 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1144 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1145 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1150 if (ctx
->srecord_type
== NULL_TREE
)
1154 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1155 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1156 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1158 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1159 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1160 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1161 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1162 splay_tree_insert (ctx
->sfield_map
,
1163 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1164 (splay_tree_value
) sfield
);
1168 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1169 : ctx
->srecord_type
, field
);
1173 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1174 (splay_tree_value
) field
);
1175 if ((mask
& 2) && ctx
->sfield_map
)
1176 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1177 (splay_tree_value
) sfield
);
1181 install_var_local (tree var
, omp_context
*ctx
)
1183 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1184 insert_decl_map (&ctx
->cb
, var
, new_var
);
1188 /* Adjust the replacement for DECL in CTX for the new context. This means
1189 copying the DECL_VALUE_EXPR, and fixing up the type. */
1192 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1194 tree new_decl
, size
;
1196 new_decl
= lookup_decl (decl
, ctx
);
1198 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1200 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1201 && DECL_HAS_VALUE_EXPR_P (decl
))
1203 tree ve
= DECL_VALUE_EXPR (decl
);
1204 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1205 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1206 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1209 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1211 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1212 if (size
== error_mark_node
)
1213 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1214 DECL_SIZE (new_decl
) = size
;
1216 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1217 if (size
== error_mark_node
)
1218 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1219 DECL_SIZE_UNIT (new_decl
) = size
;
1223 /* The callback for remap_decl. Search all containing contexts for a
1224 mapping of the variable; this avoids having to duplicate the splay
1225 tree ahead of time. We know a mapping doesn't already exist in the
1226 given context. Create new mappings to implement default semantics. */
1229 omp_copy_decl (tree var
, copy_body_data
*cb
)
1231 omp_context
*ctx
= (omp_context
*) cb
;
1234 if (TREE_CODE (var
) == LABEL_DECL
)
1236 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1237 DECL_CONTEXT (new_var
) = current_function_decl
;
1238 insert_decl_map (&ctx
->cb
, var
, new_var
);
1242 while (!is_taskreg_ctx (ctx
))
1247 new_var
= maybe_lookup_decl (var
, ctx
);
1252 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1255 return error_mark_node
;
1259 /* Debugging dumps for parallel regions. */
1260 void dump_omp_region (FILE *, struct omp_region
*, int);
1261 void debug_omp_region (struct omp_region
*);
1262 void debug_all_omp_regions (void);
1264 /* Dump the parallel region tree rooted at REGION. */
1267 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1269 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1270 gimple_code_name
[region
->type
]);
1273 dump_omp_region (file
, region
->inner
, indent
+ 4);
1277 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1278 region
->cont
->index
);
1282 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1283 region
->exit
->index
);
1285 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1288 dump_omp_region (file
, region
->next
, indent
);
1292 debug_omp_region (struct omp_region
*region
)
1294 dump_omp_region (stderr
, region
, 0);
1298 debug_all_omp_regions (void)
1300 dump_omp_region (stderr
, root_omp_region
, 0);
1304 /* Create a new parallel region starting at STMT inside region PARENT. */
1306 static struct omp_region
*
1307 new_omp_region (basic_block bb
, enum gimple_code type
,
1308 struct omp_region
*parent
)
1310 struct omp_region
*region
= XCNEW (struct omp_region
);
1312 region
->outer
= parent
;
1314 region
->type
= type
;
1318 /* This is a nested region. Add it to the list of inner
1319 regions in PARENT. */
1320 region
->next
= parent
->inner
;
1321 parent
->inner
= region
;
1325 /* This is a toplevel region. Add it to the list of toplevel
1326 regions in ROOT_OMP_REGION. */
1327 region
->next
= root_omp_region
;
1328 root_omp_region
= region
;
1334 /* Release the memory associated with the region tree rooted at REGION. */
1337 free_omp_region_1 (struct omp_region
*region
)
1339 struct omp_region
*i
, *n
;
1341 for (i
= region
->inner
; i
; i
= n
)
1344 free_omp_region_1 (i
);
1350 /* Release the memory for the entire omp region tree. */
1353 free_omp_regions (void)
1355 struct omp_region
*r
, *n
;
1356 for (r
= root_omp_region
; r
; r
= n
)
1359 free_omp_region_1 (r
);
1361 root_omp_region
= NULL
;
1365 /* Create a new context, with OUTER_CTX being the surrounding context. */
1367 static omp_context
*
1368 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1370 omp_context
*ctx
= XCNEW (omp_context
);
1372 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1373 (splay_tree_value
) ctx
);
1378 ctx
->outer
= outer_ctx
;
1379 ctx
->cb
= outer_ctx
->cb
;
1380 ctx
->cb
.block
= NULL
;
1381 ctx
->depth
= outer_ctx
->depth
+ 1;
1385 ctx
->cb
.src_fn
= current_function_decl
;
1386 ctx
->cb
.dst_fn
= current_function_decl
;
1387 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1388 gcc_checking_assert (ctx
->cb
.src_node
);
1389 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1390 ctx
->cb
.src_cfun
= cfun
;
1391 ctx
->cb
.copy_decl
= omp_copy_decl
;
1392 ctx
->cb
.eh_lp_nr
= 0;
1393 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1397 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1402 static gimple_seq
maybe_catch_exception (gimple_seq
);
1404 /* Finalize task copyfn. */
1407 finalize_task_copyfn (gomp_task
*task_stmt
)
1409 struct function
*child_cfun
;
1411 gimple_seq seq
= NULL
, new_seq
;
1414 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1415 if (child_fn
== NULL_TREE
)
1418 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1419 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1421 push_cfun (child_cfun
);
1422 bind
= gimplify_body (child_fn
, false);
1423 gimple_seq_add_stmt (&seq
, bind
);
1424 new_seq
= maybe_catch_exception (seq
);
1427 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1429 gimple_seq_add_stmt (&seq
, bind
);
1431 gimple_set_body (child_fn
, seq
);
1434 /* Inform the callgraph about the new function. */
1435 cgraph_node::add_new_function (child_fn
, false);
1438 /* Destroy a omp_context data structures. Called through the splay tree
1439 value delete callback. */
1442 delete_omp_context (splay_tree_value value
)
1444 omp_context
*ctx
= (omp_context
*) value
;
1446 delete ctx
->cb
.decl_map
;
1449 splay_tree_delete (ctx
->field_map
);
1450 if (ctx
->sfield_map
)
1451 splay_tree_delete (ctx
->sfield_map
);
1453 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1454 it produces corrupt debug information. */
1455 if (ctx
->record_type
)
1458 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1459 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1461 if (ctx
->srecord_type
)
1464 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1465 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1468 if (is_task_ctx (ctx
))
1469 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1474 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1478 fixup_child_record_type (omp_context
*ctx
)
1480 tree f
, type
= ctx
->record_type
;
1482 /* ??? It isn't sufficient to just call remap_type here, because
1483 variably_modified_type_p doesn't work the way we expect for
1484 record types. Testing each field for whether it needs remapping
1485 and creating a new record by hand works, however. */
1486 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1487 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1491 tree name
, new_fields
= NULL
;
1493 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1494 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1495 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1496 TYPE_DECL
, name
, type
);
1497 TYPE_NAME (type
) = name
;
1499 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1501 tree new_f
= copy_node (f
);
1502 DECL_CONTEXT (new_f
) = type
;
1503 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1504 DECL_CHAIN (new_f
) = new_fields
;
1505 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1506 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1508 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1512 /* Arrange to be able to look up the receiver field
1513 given the sender field. */
1514 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1515 (splay_tree_value
) new_f
);
1517 TYPE_FIELDS (type
) = nreverse (new_fields
);
1521 TREE_TYPE (ctx
->receiver_decl
)
1522 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1525 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1526 specified by CLAUSES. */
1529 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1532 bool scan_array_reductions
= false;
1534 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1538 switch (OMP_CLAUSE_CODE (c
))
1540 case OMP_CLAUSE_PRIVATE
:
1541 decl
= OMP_CLAUSE_DECL (c
);
1542 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1544 else if (!is_variable_sized (decl
))
1545 install_var_local (decl
, ctx
);
1548 case OMP_CLAUSE_SHARED
:
1549 decl
= OMP_CLAUSE_DECL (c
);
1550 /* Ignore shared directives in teams construct. */
1551 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1553 /* Global variables don't need to be copied,
1554 the receiver side will use them directly. */
1555 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1556 if (is_global_var (odecl
))
1558 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1561 gcc_assert (is_taskreg_ctx (ctx
));
1562 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1563 || !is_variable_sized (decl
));
1564 /* Global variables don't need to be copied,
1565 the receiver side will use them directly. */
1566 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1568 by_ref
= use_pointer_for_field (decl
, ctx
);
1569 if (! TREE_READONLY (decl
)
1570 || TREE_ADDRESSABLE (decl
)
1572 || is_reference (decl
))
1574 install_var_field (decl
, by_ref
, 3, ctx
);
1575 install_var_local (decl
, ctx
);
1578 /* We don't need to copy const scalar vars back. */
1579 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1582 case OMP_CLAUSE_LASTPRIVATE
:
1583 /* Let the corresponding firstprivate clause create
1585 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1589 case OMP_CLAUSE_FIRSTPRIVATE
:
1590 case OMP_CLAUSE_REDUCTION
:
1591 case OMP_CLAUSE_LINEAR
:
1592 decl
= OMP_CLAUSE_DECL (c
);
1594 if (is_variable_sized (decl
))
1596 if (is_task_ctx (ctx
))
1597 install_var_field (decl
, false, 1, ctx
);
1600 else if (is_taskreg_ctx (ctx
))
1603 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1604 by_ref
= use_pointer_for_field (decl
, NULL
);
1606 if (is_task_ctx (ctx
)
1607 && (global
|| by_ref
|| is_reference (decl
)))
1609 install_var_field (decl
, false, 1, ctx
);
1611 install_var_field (decl
, by_ref
, 2, ctx
);
1614 install_var_field (decl
, by_ref
, 3, ctx
);
1616 install_var_local (decl
, ctx
);
1619 case OMP_CLAUSE__LOOPTEMP_
:
1620 gcc_assert (is_parallel_ctx (ctx
));
1621 decl
= OMP_CLAUSE_DECL (c
);
1622 install_var_field (decl
, false, 3, ctx
);
1623 install_var_local (decl
, ctx
);
1626 case OMP_CLAUSE_COPYPRIVATE
:
1627 case OMP_CLAUSE_COPYIN
:
1628 decl
= OMP_CLAUSE_DECL (c
);
1629 by_ref
= use_pointer_for_field (decl
, NULL
);
1630 install_var_field (decl
, by_ref
, 3, ctx
);
1633 case OMP_CLAUSE_DEFAULT
:
1634 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1637 case OMP_CLAUSE_FINAL
:
1639 case OMP_CLAUSE_NUM_THREADS
:
1640 case OMP_CLAUSE_NUM_TEAMS
:
1641 case OMP_CLAUSE_THREAD_LIMIT
:
1642 case OMP_CLAUSE_DEVICE
:
1643 case OMP_CLAUSE_SCHEDULE
:
1644 case OMP_CLAUSE_DIST_SCHEDULE
:
1645 case OMP_CLAUSE_DEPEND
:
1646 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1648 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1652 case OMP_CLAUSE_FROM
:
1653 case OMP_CLAUSE_MAP
:
1655 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1656 decl
= OMP_CLAUSE_DECL (c
);
1657 /* Global variables with "omp declare target" attribute
1658 don't need to be copied, the receiver side will use them
1660 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1662 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1663 && varpool_node::get_create (decl
)->offloadable
)
1665 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1666 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1668 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1669 #pragma omp target data, there is nothing to map for
1671 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1672 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1677 if (DECL_SIZE (decl
)
1678 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1680 tree decl2
= DECL_VALUE_EXPR (decl
);
1681 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1682 decl2
= TREE_OPERAND (decl2
, 0);
1683 gcc_assert (DECL_P (decl2
));
1684 install_var_field (decl2
, true, 3, ctx
);
1685 install_var_local (decl2
, ctx
);
1686 install_var_local (decl
, ctx
);
1690 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1691 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1692 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1693 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1694 install_var_field (decl
, true, 7, ctx
);
1696 install_var_field (decl
, true, 3, ctx
);
1697 if (gimple_omp_target_kind (ctx
->stmt
)
1698 == GF_OMP_TARGET_KIND_REGION
)
1699 install_var_local (decl
, ctx
);
1704 tree base
= get_base_address (decl
);
1705 tree nc
= OMP_CLAUSE_CHAIN (c
);
1708 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1709 && OMP_CLAUSE_DECL (nc
) == base
1710 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1711 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1713 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1714 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1720 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1721 decl
= OMP_CLAUSE_DECL (c
);
1723 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1724 (splay_tree_key
) decl
));
1726 = build_decl (OMP_CLAUSE_LOCATION (c
),
1727 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1728 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1729 insert_field_into_struct (ctx
->record_type
, field
);
1730 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1731 (splay_tree_value
) field
);
1736 case OMP_CLAUSE_NOWAIT
:
1737 case OMP_CLAUSE_ORDERED
:
1738 case OMP_CLAUSE_COLLAPSE
:
1739 case OMP_CLAUSE_UNTIED
:
1740 case OMP_CLAUSE_MERGEABLE
:
1741 case OMP_CLAUSE_PROC_BIND
:
1742 case OMP_CLAUSE_SAFELEN
:
1745 case OMP_CLAUSE_ALIGNED
:
1746 decl
= OMP_CLAUSE_DECL (c
);
1747 if (is_global_var (decl
)
1748 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1749 install_var_local (decl
, ctx
);
1757 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1759 switch (OMP_CLAUSE_CODE (c
))
1761 case OMP_CLAUSE_LASTPRIVATE
:
1762 /* Let the corresponding firstprivate clause create
1764 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1765 scan_array_reductions
= true;
1766 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1770 case OMP_CLAUSE_PRIVATE
:
1771 case OMP_CLAUSE_FIRSTPRIVATE
:
1772 case OMP_CLAUSE_REDUCTION
:
1773 case OMP_CLAUSE_LINEAR
:
1774 decl
= OMP_CLAUSE_DECL (c
);
1775 if (is_variable_sized (decl
))
1776 install_var_local (decl
, ctx
);
1777 fixup_remapped_decl (decl
, ctx
,
1778 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1779 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1780 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1781 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1782 scan_array_reductions
= true;
1783 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1784 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1785 scan_array_reductions
= true;
1788 case OMP_CLAUSE_SHARED
:
1789 /* Ignore shared directives in teams construct. */
1790 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1792 decl
= OMP_CLAUSE_DECL (c
);
1793 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1794 fixup_remapped_decl (decl
, ctx
, false);
1797 case OMP_CLAUSE_MAP
:
1798 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1800 decl
= OMP_CLAUSE_DECL (c
);
1802 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1803 && varpool_node::get_create (decl
)->offloadable
)
1807 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1808 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1809 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1811 tree new_decl
= lookup_decl (decl
, ctx
);
1812 TREE_TYPE (new_decl
)
1813 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1815 else if (DECL_SIZE (decl
)
1816 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1818 tree decl2
= DECL_VALUE_EXPR (decl
);
1819 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1820 decl2
= TREE_OPERAND (decl2
, 0);
1821 gcc_assert (DECL_P (decl2
));
1822 fixup_remapped_decl (decl2
, ctx
, false);
1823 fixup_remapped_decl (decl
, ctx
, true);
1826 fixup_remapped_decl (decl
, ctx
, false);
1830 case OMP_CLAUSE_COPYPRIVATE
:
1831 case OMP_CLAUSE_COPYIN
:
1832 case OMP_CLAUSE_DEFAULT
:
1834 case OMP_CLAUSE_NUM_THREADS
:
1835 case OMP_CLAUSE_NUM_TEAMS
:
1836 case OMP_CLAUSE_THREAD_LIMIT
:
1837 case OMP_CLAUSE_DEVICE
:
1838 case OMP_CLAUSE_SCHEDULE
:
1839 case OMP_CLAUSE_DIST_SCHEDULE
:
1840 case OMP_CLAUSE_NOWAIT
:
1841 case OMP_CLAUSE_ORDERED
:
1842 case OMP_CLAUSE_COLLAPSE
:
1843 case OMP_CLAUSE_UNTIED
:
1844 case OMP_CLAUSE_FINAL
:
1845 case OMP_CLAUSE_MERGEABLE
:
1846 case OMP_CLAUSE_PROC_BIND
:
1847 case OMP_CLAUSE_SAFELEN
:
1848 case OMP_CLAUSE_ALIGNED
:
1849 case OMP_CLAUSE_DEPEND
:
1850 case OMP_CLAUSE__LOOPTEMP_
:
1852 case OMP_CLAUSE_FROM
:
1853 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1861 if (scan_array_reductions
)
1862 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1863 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1864 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1866 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1867 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1869 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1870 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1871 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1872 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1873 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1874 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1877 /* Create a new name for omp child function. Returns an identifier. If
1878 IS_CILK_FOR is true then the suffix for the child function is
1882 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
1885 return clone_function_name (current_function_decl
, "_cilk_for_fn");
1886 return clone_function_name (current_function_decl
,
1887 task_copy
? "_omp_cpyfn" : "_omp_fn");
1890 /* Returns the type of the induction variable for the child function for
1891 _Cilk_for and the types for _high and _low variables based on TYPE. */
1894 cilk_for_check_loop_diff_type (tree type
)
1896 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
1898 if (TYPE_UNSIGNED (type
))
1899 return uint32_type_node
;
1901 return integer_type_node
;
1905 if (TYPE_UNSIGNED (type
))
1906 return uint64_type_node
;
1908 return long_long_integer_type_node
;
1912 /* Build a decl for the omp child function. It'll not contain a body
1913 yet, just the bare decl. */
1916 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1918 tree decl
, type
, name
, t
;
1921 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
1922 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
1923 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
1924 tree cilk_var_type
= NULL_TREE
;
1926 name
= create_omp_child_function_name (task_copy
,
1927 cilk_for_count
!= NULL_TREE
);
1929 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1930 ptr_type_node
, NULL_TREE
);
1931 else if (cilk_for_count
)
1933 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
1934 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
1935 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1936 cilk_var_type
, cilk_var_type
, NULL_TREE
);
1939 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1941 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
1944 ctx
->cb
.dst_fn
= decl
;
1946 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1948 TREE_STATIC (decl
) = 1;
1949 TREE_USED (decl
) = 1;
1950 DECL_ARTIFICIAL (decl
) = 1;
1951 DECL_IGNORED_P (decl
) = 0;
1952 TREE_PUBLIC (decl
) = 0;
1953 DECL_UNINLINABLE (decl
) = 1;
1954 DECL_EXTERNAL (decl
) = 0;
1955 DECL_CONTEXT (decl
) = NULL_TREE
;
1956 DECL_INITIAL (decl
) = make_node (BLOCK
);
1957 if (cgraph_node::get (current_function_decl
)->offloadable
)
1958 cgraph_node::get_create (decl
)->offloadable
= 1;
1962 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1963 if (is_targetreg_ctx (octx
))
1965 cgraph_node::get_create (decl
)->offloadable
= 1;
1966 #ifdef ENABLE_OFFLOADING
1967 g
->have_offload
= true;
1973 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1974 RESULT_DECL
, NULL_TREE
, void_type_node
);
1975 DECL_ARTIFICIAL (t
) = 1;
1976 DECL_IGNORED_P (t
) = 1;
1977 DECL_CONTEXT (t
) = decl
;
1978 DECL_RESULT (decl
) = t
;
1980 /* _Cilk_for's child function requires two extra parameters called
1981 __low and __high that are set the by Cilk runtime when it calls this
1985 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1986 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
1987 DECL_ARTIFICIAL (t
) = 1;
1988 DECL_NAMELESS (t
) = 1;
1989 DECL_ARG_TYPE (t
) = ptr_type_node
;
1990 DECL_CONTEXT (t
) = current_function_decl
;
1992 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1993 DECL_ARGUMENTS (decl
) = t
;
1995 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1996 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
1997 DECL_ARTIFICIAL (t
) = 1;
1998 DECL_NAMELESS (t
) = 1;
1999 DECL_ARG_TYPE (t
) = ptr_type_node
;
2000 DECL_CONTEXT (t
) = current_function_decl
;
2002 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2003 DECL_ARGUMENTS (decl
) = t
;
2006 tree data_name
= get_identifier (".omp_data_i");
2007 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2009 DECL_ARTIFICIAL (t
) = 1;
2010 DECL_NAMELESS (t
) = 1;
2011 DECL_ARG_TYPE (t
) = ptr_type_node
;
2012 DECL_CONTEXT (t
) = current_function_decl
;
2015 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2016 DECL_ARGUMENTS (decl
) = t
;
2018 ctx
->receiver_decl
= t
;
2021 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2022 PARM_DECL
, get_identifier (".omp_data_o"),
2024 DECL_ARTIFICIAL (t
) = 1;
2025 DECL_NAMELESS (t
) = 1;
2026 DECL_ARG_TYPE (t
) = ptr_type_node
;
2027 DECL_CONTEXT (t
) = current_function_decl
;
2029 TREE_ADDRESSABLE (t
) = 1;
2030 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2031 DECL_ARGUMENTS (decl
) = t
;
2034 /* Allocate memory for the function structure. The call to
2035 allocate_struct_function clobbers CFUN, so we need to restore
2037 push_struct_function (decl
);
2038 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2042 /* Callback for walk_gimple_seq. Check if combined parallel
2043 contains gimple_omp_for_combined_into_p OMP_FOR. */
2046 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2047 bool *handled_ops_p
,
2048 struct walk_stmt_info
*wi
)
2050 gimple stmt
= gsi_stmt (*gsi_p
);
2052 *handled_ops_p
= true;
2053 switch (gimple_code (stmt
))
2057 case GIMPLE_OMP_FOR
:
2058 if (gimple_omp_for_combined_into_p (stmt
)
2059 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2062 return integer_zero_node
;
2071 /* Scan an OpenMP parallel directive. */
2074 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2078 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2080 /* Ignore parallel directives with empty bodies, unless there
2081 are copyin clauses. */
2083 && empty_body_p (gimple_omp_body (stmt
))
2084 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2085 OMP_CLAUSE_COPYIN
) == NULL
)
2087 gsi_replace (gsi
, gimple_build_nop (), false);
2091 if (gimple_omp_parallel_combined_p (stmt
))
2093 struct walk_stmt_info wi
;
2095 memset (&wi
, 0, sizeof (wi
));
2097 walk_gimple_seq (gimple_omp_body (stmt
),
2098 find_combined_for
, NULL
, &wi
);
2101 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2102 struct omp_for_data fd
;
2103 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2104 /* We need two temporaries with fd.loop.v type (istart/iend)
2105 and then (fd.collapse - 1) temporaries with the same
2106 type for count2 ... countN-1 vars if not constant. */
2107 size_t count
= 2, i
;
2108 tree type
= fd
.iter_type
;
2110 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2111 count
+= fd
.collapse
- 1;
2112 for (i
= 0; i
< count
; i
++)
2114 tree temp
= create_tmp_var (type
);
2115 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2116 OMP_CLAUSE__LOOPTEMP_
);
2117 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2118 OMP_CLAUSE_DECL (c
) = temp
;
2119 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2120 gimple_omp_parallel_set_clauses (stmt
, c
);
2125 ctx
= new_omp_context (stmt
, outer_ctx
);
2126 taskreg_contexts
.safe_push (ctx
);
2127 if (taskreg_nesting_level
> 1)
2128 ctx
->is_nested
= true;
2129 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2130 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2131 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2132 name
= create_tmp_var_name (".omp_data_s");
2133 name
= build_decl (gimple_location (stmt
),
2134 TYPE_DECL
, name
, ctx
->record_type
);
2135 DECL_ARTIFICIAL (name
) = 1;
2136 DECL_NAMELESS (name
) = 1;
2137 TYPE_NAME (ctx
->record_type
) = name
;
2138 create_omp_child_function (ctx
, false);
2139 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2141 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2142 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2144 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2145 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2148 /* Scan an OpenMP task directive. */
2151 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2155 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2157 /* Ignore task directives with empty bodies. */
2159 && empty_body_p (gimple_omp_body (stmt
)))
2161 gsi_replace (gsi
, gimple_build_nop (), false);
2165 ctx
= new_omp_context (stmt
, outer_ctx
);
2166 taskreg_contexts
.safe_push (ctx
);
2167 if (taskreg_nesting_level
> 1)
2168 ctx
->is_nested
= true;
2169 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2170 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2171 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2172 name
= create_tmp_var_name (".omp_data_s");
2173 name
= build_decl (gimple_location (stmt
),
2174 TYPE_DECL
, name
, ctx
->record_type
);
2175 DECL_ARTIFICIAL (name
) = 1;
2176 DECL_NAMELESS (name
) = 1;
2177 TYPE_NAME (ctx
->record_type
) = name
;
2178 create_omp_child_function (ctx
, false);
2179 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2181 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2183 if (ctx
->srecord_type
)
2185 name
= create_tmp_var_name (".omp_data_a");
2186 name
= build_decl (gimple_location (stmt
),
2187 TYPE_DECL
, name
, ctx
->srecord_type
);
2188 DECL_ARTIFICIAL (name
) = 1;
2189 DECL_NAMELESS (name
) = 1;
2190 TYPE_NAME (ctx
->srecord_type
) = name
;
2191 create_omp_child_function (ctx
, true);
2194 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2196 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2198 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2199 t
= build_int_cst (long_integer_type_node
, 0);
2200 gimple_omp_task_set_arg_size (stmt
, t
);
2201 t
= build_int_cst (long_integer_type_node
, 1);
2202 gimple_omp_task_set_arg_align (stmt
, t
);
2207 /* If any decls have been made addressable during scan_omp,
2208 adjust their fields if needed, and layout record types
2209 of parallel/task constructs. */
2212 finish_taskreg_scan (omp_context
*ctx
)
2214 if (ctx
->record_type
== NULL_TREE
)
2217 /* If any task_shared_vars were needed, verify all
2218 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2219 statements if use_pointer_for_field hasn't changed
2220 because of that. If it did, update field types now. */
2221 if (task_shared_vars
)
2225 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2226 c
; c
= OMP_CLAUSE_CHAIN (c
))
2227 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2229 tree decl
= OMP_CLAUSE_DECL (c
);
2231 /* Global variables don't need to be copied,
2232 the receiver side will use them directly. */
2233 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2235 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2236 || !use_pointer_for_field (decl
, ctx
))
2238 tree field
= lookup_field (decl
, ctx
);
2239 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2240 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2242 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2243 TREE_THIS_VOLATILE (field
) = 0;
2244 DECL_USER_ALIGN (field
) = 0;
2245 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2246 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2247 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2248 if (ctx
->srecord_type
)
2250 tree sfield
= lookup_sfield (decl
, ctx
);
2251 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2252 TREE_THIS_VOLATILE (sfield
) = 0;
2253 DECL_USER_ALIGN (sfield
) = 0;
2254 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2255 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2256 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2261 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2263 layout_type (ctx
->record_type
);
2264 fixup_child_record_type (ctx
);
2268 location_t loc
= gimple_location (ctx
->stmt
);
2269 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2270 /* Move VLA fields to the end. */
2271 p
= &TYPE_FIELDS (ctx
->record_type
);
2273 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2274 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2277 *p
= TREE_CHAIN (*p
);
2278 TREE_CHAIN (*q
) = NULL_TREE
;
2279 q
= &TREE_CHAIN (*q
);
2282 p
= &DECL_CHAIN (*p
);
2284 layout_type (ctx
->record_type
);
2285 fixup_child_record_type (ctx
);
2286 if (ctx
->srecord_type
)
2287 layout_type (ctx
->srecord_type
);
2288 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2289 TYPE_SIZE_UNIT (ctx
->record_type
));
2290 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2291 t
= build_int_cst (long_integer_type_node
,
2292 TYPE_ALIGN_UNIT (ctx
->record_type
));
2293 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2298 /* Scan an OpenMP loop directive. */
2301 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2306 ctx
= new_omp_context (stmt
, outer_ctx
);
2308 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2310 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2311 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2313 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2314 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2315 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2316 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2318 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2321 /* Scan an OpenMP sections directive. */
2324 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2328 ctx
= new_omp_context (stmt
, outer_ctx
);
2329 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2330 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2333 /* Scan an OpenMP single directive. */
2336 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2341 ctx
= new_omp_context (stmt
, outer_ctx
);
2342 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2343 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2344 name
= create_tmp_var_name (".omp_copy_s");
2345 name
= build_decl (gimple_location (stmt
),
2346 TYPE_DECL
, name
, ctx
->record_type
);
2347 TYPE_NAME (ctx
->record_type
) = name
;
2349 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2350 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2352 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2353 ctx
->record_type
= NULL
;
2355 layout_type (ctx
->record_type
);
2358 /* Scan an OpenMP target{, data, update} directive. */
2361 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2365 int kind
= gimple_omp_target_kind (stmt
);
2367 ctx
= new_omp_context (stmt
, outer_ctx
);
2368 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2369 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2370 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2371 name
= create_tmp_var_name (".omp_data_t");
2372 name
= build_decl (gimple_location (stmt
),
2373 TYPE_DECL
, name
, ctx
->record_type
);
2374 DECL_ARTIFICIAL (name
) = 1;
2375 DECL_NAMELESS (name
) = 1;
2376 TYPE_NAME (ctx
->record_type
) = name
;
2377 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2379 create_omp_child_function (ctx
, false);
2380 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2383 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2384 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2386 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2387 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2390 TYPE_FIELDS (ctx
->record_type
)
2391 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2392 #ifdef ENABLE_CHECKING
2394 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2395 for (field
= TYPE_FIELDS (ctx
->record_type
);
2397 field
= DECL_CHAIN (field
))
2398 gcc_assert (DECL_ALIGN (field
) == align
);
2400 layout_type (ctx
->record_type
);
2401 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2402 fixup_child_record_type (ctx
);
2406 /* Scan an OpenMP teams directive. */
2409 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2411 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2412 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2413 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2416 /* Check OpenMP nesting restrictions. */
2418 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2422 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2423 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2425 error_at (gimple_location (stmt
),
2426 "OpenMP constructs may not be nested inside simd region");
2429 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2431 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2432 || (gimple_omp_for_kind (stmt
)
2433 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2434 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2436 error_at (gimple_location (stmt
),
2437 "only distribute or parallel constructs are allowed to "
2438 "be closely nested inside teams construct");
2443 switch (gimple_code (stmt
))
2445 case GIMPLE_OMP_FOR
:
2446 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2448 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2450 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2452 error_at (gimple_location (stmt
),
2453 "distribute construct must be closely nested inside "
2461 if (is_gimple_call (stmt
)
2462 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2463 == BUILT_IN_GOMP_CANCEL
2464 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2465 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2467 const char *bad
= NULL
;
2468 const char *kind
= NULL
;
2471 error_at (gimple_location (stmt
), "orphaned %qs construct",
2472 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2473 == BUILT_IN_GOMP_CANCEL
2474 ? "#pragma omp cancel"
2475 : "#pragma omp cancellation point");
2478 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2479 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2483 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2484 bad
= "#pragma omp parallel";
2485 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2486 == BUILT_IN_GOMP_CANCEL
2487 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2488 ctx
->cancellable
= true;
2492 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2493 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2494 bad
= "#pragma omp for";
2495 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2496 == BUILT_IN_GOMP_CANCEL
2497 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2499 ctx
->cancellable
= true;
2500 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2502 warning_at (gimple_location (stmt
), 0,
2503 "%<#pragma omp cancel for%> inside "
2504 "%<nowait%> for construct");
2505 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2506 OMP_CLAUSE_ORDERED
))
2507 warning_at (gimple_location (stmt
), 0,
2508 "%<#pragma omp cancel for%> inside "
2509 "%<ordered%> for construct");
2514 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2515 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2516 bad
= "#pragma omp sections";
2517 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2518 == BUILT_IN_GOMP_CANCEL
2519 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2521 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2523 ctx
->cancellable
= true;
2524 if (find_omp_clause (gimple_omp_sections_clauses
2527 warning_at (gimple_location (stmt
), 0,
2528 "%<#pragma omp cancel sections%> inside "
2529 "%<nowait%> sections construct");
2533 gcc_assert (ctx
->outer
2534 && gimple_code (ctx
->outer
->stmt
)
2535 == GIMPLE_OMP_SECTIONS
);
2536 ctx
->outer
->cancellable
= true;
2537 if (find_omp_clause (gimple_omp_sections_clauses
2540 warning_at (gimple_location (stmt
), 0,
2541 "%<#pragma omp cancel sections%> inside "
2542 "%<nowait%> sections construct");
2548 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2549 bad
= "#pragma omp task";
2551 ctx
->cancellable
= true;
2555 error_at (gimple_location (stmt
), "invalid arguments");
2560 error_at (gimple_location (stmt
),
2561 "%<%s %s%> construct not closely nested inside of %qs",
2562 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2563 == BUILT_IN_GOMP_CANCEL
2564 ? "#pragma omp cancel"
2565 : "#pragma omp cancellation point", kind
, bad
);
2570 case GIMPLE_OMP_SECTIONS
:
2571 case GIMPLE_OMP_SINGLE
:
2572 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2573 switch (gimple_code (ctx
->stmt
))
2575 case GIMPLE_OMP_FOR
:
2576 case GIMPLE_OMP_SECTIONS
:
2577 case GIMPLE_OMP_SINGLE
:
2578 case GIMPLE_OMP_ORDERED
:
2579 case GIMPLE_OMP_MASTER
:
2580 case GIMPLE_OMP_TASK
:
2581 case GIMPLE_OMP_CRITICAL
:
2582 if (is_gimple_call (stmt
))
2584 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2585 != BUILT_IN_GOMP_BARRIER
)
2587 error_at (gimple_location (stmt
),
2588 "barrier region may not be closely nested inside "
2589 "of work-sharing, critical, ordered, master or "
2590 "explicit task region");
2593 error_at (gimple_location (stmt
),
2594 "work-sharing region may not be closely nested inside "
2595 "of work-sharing, critical, ordered, master or explicit "
2598 case GIMPLE_OMP_PARALLEL
:
2604 case GIMPLE_OMP_MASTER
:
2605 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2606 switch (gimple_code (ctx
->stmt
))
2608 case GIMPLE_OMP_FOR
:
2609 case GIMPLE_OMP_SECTIONS
:
2610 case GIMPLE_OMP_SINGLE
:
2611 case GIMPLE_OMP_TASK
:
2612 error_at (gimple_location (stmt
),
2613 "master region may not be closely nested inside "
2614 "of work-sharing or explicit task region");
2616 case GIMPLE_OMP_PARALLEL
:
2622 case GIMPLE_OMP_ORDERED
:
2623 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2624 switch (gimple_code (ctx
->stmt
))
2626 case GIMPLE_OMP_CRITICAL
:
2627 case GIMPLE_OMP_TASK
:
2628 error_at (gimple_location (stmt
),
2629 "ordered region may not be closely nested inside "
2630 "of critical or explicit task region");
2632 case GIMPLE_OMP_FOR
:
2633 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2634 OMP_CLAUSE_ORDERED
) == NULL
)
2636 error_at (gimple_location (stmt
),
2637 "ordered region must be closely nested inside "
2638 "a loop region with an ordered clause");
2642 case GIMPLE_OMP_PARALLEL
:
2643 error_at (gimple_location (stmt
),
2644 "ordered region must be closely nested inside "
2645 "a loop region with an ordered clause");
2651 case GIMPLE_OMP_CRITICAL
:
2654 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2655 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2656 if (gomp_critical
*other_crit
2657 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2658 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2660 error_at (gimple_location (stmt
),
2661 "critical region may not be nested inside a critical "
2662 "region with the same name");
2667 case GIMPLE_OMP_TEAMS
:
2669 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2670 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2672 error_at (gimple_location (stmt
),
2673 "teams construct not closely nested inside of target "
2678 case GIMPLE_OMP_TARGET
:
2679 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2680 if (is_targetreg_ctx (ctx
))
2683 switch (gimple_omp_target_kind (stmt
))
2685 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2686 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2687 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2688 default: gcc_unreachable ();
2690 warning_at (gimple_location (stmt
), 0,
2691 "%s construct inside of target region", name
);
2701 /* Helper function scan_omp.
2703 Callback for walk_tree or operators in walk_gimple_stmt used to
2704 scan for OpenMP directives in TP. */
2707 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2709 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2710 omp_context
*ctx
= (omp_context
*) wi
->info
;
2713 switch (TREE_CODE (t
))
2720 *tp
= remap_decl (t
, &ctx
->cb
);
2724 if (ctx
&& TYPE_P (t
))
2725 *tp
= remap_type (t
, &ctx
->cb
);
2726 else if (!DECL_P (t
))
2731 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2732 if (tem
!= TREE_TYPE (t
))
2734 if (TREE_CODE (t
) == INTEGER_CST
)
2735 *tp
= wide_int_to_tree (tem
, t
);
2737 TREE_TYPE (t
) = tem
;
2747 /* Return true if FNDECL is a setjmp or a longjmp. */
2750 setjmp_or_longjmp_p (const_tree fndecl
)
2752 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2753 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2754 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2757 tree declname
= DECL_NAME (fndecl
);
2760 const char *name
= IDENTIFIER_POINTER (declname
);
2761 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2765 /* Helper function for scan_omp.
2767 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2768 the current statement in GSI. */
2771 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2772 struct walk_stmt_info
*wi
)
2774 gimple stmt
= gsi_stmt (*gsi
);
2775 omp_context
*ctx
= (omp_context
*) wi
->info
;
2777 if (gimple_has_location (stmt
))
2778 input_location
= gimple_location (stmt
);
2780 /* Check the OpenMP nesting restrictions. */
2781 bool remove
= false;
2782 if (is_gimple_omp (stmt
))
2783 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2784 else if (is_gimple_call (stmt
))
2786 tree fndecl
= gimple_call_fndecl (stmt
);
2789 if (setjmp_or_longjmp_p (fndecl
)
2791 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2792 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2795 error_at (gimple_location (stmt
),
2796 "setjmp/longjmp inside simd construct");
2798 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2799 switch (DECL_FUNCTION_CODE (fndecl
))
2801 case BUILT_IN_GOMP_BARRIER
:
2802 case BUILT_IN_GOMP_CANCEL
:
2803 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2804 case BUILT_IN_GOMP_TASKYIELD
:
2805 case BUILT_IN_GOMP_TASKWAIT
:
2806 case BUILT_IN_GOMP_TASKGROUP_START
:
2807 case BUILT_IN_GOMP_TASKGROUP_END
:
2808 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2817 stmt
= gimple_build_nop ();
2818 gsi_replace (gsi
, stmt
, false);
2821 *handled_ops_p
= true;
2823 switch (gimple_code (stmt
))
2825 case GIMPLE_OMP_PARALLEL
:
2826 taskreg_nesting_level
++;
2827 scan_omp_parallel (gsi
, ctx
);
2828 taskreg_nesting_level
--;
2831 case GIMPLE_OMP_TASK
:
2832 taskreg_nesting_level
++;
2833 scan_omp_task (gsi
, ctx
);
2834 taskreg_nesting_level
--;
2837 case GIMPLE_OMP_FOR
:
2838 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
2841 case GIMPLE_OMP_SECTIONS
:
2842 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
2845 case GIMPLE_OMP_SINGLE
:
2846 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
2849 case GIMPLE_OMP_SECTION
:
2850 case GIMPLE_OMP_MASTER
:
2851 case GIMPLE_OMP_TASKGROUP
:
2852 case GIMPLE_OMP_ORDERED
:
2853 case GIMPLE_OMP_CRITICAL
:
2854 ctx
= new_omp_context (stmt
, ctx
);
2855 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2858 case GIMPLE_OMP_TARGET
:
2859 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
2862 case GIMPLE_OMP_TEAMS
:
2863 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
2870 *handled_ops_p
= false;
2872 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
2874 var
= DECL_CHAIN (var
))
2875 insert_decl_map (&ctx
->cb
, var
, var
);
2879 *handled_ops_p
= false;
2887 /* Scan all the statements starting at the current statement. CTX
2888 contains context information about the OpenMP directives and
2889 clauses found during the scan. */
2892 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2894 location_t saved_location
;
2895 struct walk_stmt_info wi
;
2897 memset (&wi
, 0, sizeof (wi
));
2899 wi
.want_locations
= true;
2901 saved_location
= input_location
;
2902 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2903 input_location
= saved_location
;
2906 /* Re-gimplification and code generation routines. */
2908 /* Build a call to GOMP_barrier. */
2911 build_omp_barrier (tree lhs
)
2913 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2914 : BUILT_IN_GOMP_BARRIER
);
2915 gcall
*g
= gimple_build_call (fndecl
, 0);
2917 gimple_call_set_lhs (g
, lhs
);
2921 /* If a context was created for STMT when it was scanned, return it. */
2923 static omp_context
*
2924 maybe_lookup_ctx (gimple stmt
)
2927 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2928 return n
? (omp_context
*) n
->value
: NULL
;
2932 /* Find the mapping for DECL in CTX or the immediately enclosing
2933 context that has a mapping for DECL.
2935 If CTX is a nested parallel directive, we may have to use the decl
2936 mappings created in CTX's parent context. Suppose that we have the
2937 following parallel nesting (variable UIDs showed for clarity):
2940 #omp parallel shared(iD.1562) -> outer parallel
2941 iD.1562 = iD.1562 + 1;
2943 #omp parallel shared (iD.1562) -> inner parallel
2944 iD.1562 = iD.1562 - 1;
2946 Each parallel structure will create a distinct .omp_data_s structure
2947 for copying iD.1562 in/out of the directive:
2949 outer parallel .omp_data_s.1.i -> iD.1562
2950 inner parallel .omp_data_s.2.i -> iD.1562
2952 A shared variable mapping will produce a copy-out operation before
2953 the parallel directive and a copy-in operation after it. So, in
2954 this case we would have:
2957 .omp_data_o.1.i = iD.1562;
2958 #omp parallel shared(iD.1562) -> outer parallel
2959 .omp_data_i.1 = &.omp_data_o.1
2960 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2962 .omp_data_o.2.i = iD.1562; -> **
2963 #omp parallel shared(iD.1562) -> inner parallel
2964 .omp_data_i.2 = &.omp_data_o.2
2965 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2968 ** This is a problem. The symbol iD.1562 cannot be referenced
2969 inside the body of the outer parallel region. But since we are
2970 emitting this copy operation while expanding the inner parallel
2971 directive, we need to access the CTX structure of the outer
2972 parallel directive to get the correct mapping:
2974 .omp_data_o.2.i = .omp_data_i.1->i
2976 Since there may be other workshare or parallel directives enclosing
2977 the parallel directive, it may be necessary to walk up the context
2978 parent chain. This is not a problem in general because nested
2979 parallelism happens only rarely. */
2982 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2987 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2988 t
= maybe_lookup_decl (decl
, up
);
2990 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2992 return t
? t
: decl
;
2996 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2997 in outer contexts. */
3000 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3005 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3006 t
= maybe_lookup_decl (decl
, up
);
3008 return t
? t
: decl
;
3012 /* Construct the initialization value for reduction CLAUSE. */
3015 omp_reduction_init (tree clause
, tree type
)
3017 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3018 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3025 case TRUTH_ORIF_EXPR
:
3026 case TRUTH_XOR_EXPR
:
3028 return build_zero_cst (type
);
3031 case TRUTH_AND_EXPR
:
3032 case TRUTH_ANDIF_EXPR
:
3034 return fold_convert_loc (loc
, type
, integer_one_node
);
3037 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3040 if (SCALAR_FLOAT_TYPE_P (type
))
3042 REAL_VALUE_TYPE max
, min
;
3043 if (HONOR_INFINITIES (type
))
3046 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3049 real_maxval (&min
, 1, TYPE_MODE (type
));
3050 return build_real (type
, min
);
3054 gcc_assert (INTEGRAL_TYPE_P (type
));
3055 return TYPE_MIN_VALUE (type
);
3059 if (SCALAR_FLOAT_TYPE_P (type
))
3061 REAL_VALUE_TYPE max
;
3062 if (HONOR_INFINITIES (type
))
3065 real_maxval (&max
, 0, TYPE_MODE (type
));
3066 return build_real (type
, max
);
3070 gcc_assert (INTEGRAL_TYPE_P (type
));
3071 return TYPE_MAX_VALUE (type
);
3079 /* Return alignment to be assumed for var in CLAUSE, which should be
3080 OMP_CLAUSE_ALIGNED. */
3083 omp_clause_aligned_alignment (tree clause
)
3085 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3086 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3088 /* Otherwise return implementation defined alignment. */
3089 unsigned int al
= 1;
3090 machine_mode mode
, vmode
;
3091 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3093 vs
= 1 << floor_log2 (vs
);
3094 static enum mode_class classes
[]
3095 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3096 for (int i
= 0; i
< 4; i
+= 2)
3097 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3099 mode
= GET_MODE_WIDER_MODE (mode
))
3101 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3102 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3105 && GET_MODE_SIZE (vmode
) < vs
3106 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3107 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3109 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3110 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3112 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3113 / GET_MODE_SIZE (mode
));
3114 if (TYPE_MODE (type
) != vmode
)
3116 if (TYPE_ALIGN_UNIT (type
) > al
)
3117 al
= TYPE_ALIGN_UNIT (type
);
3119 return build_int_cst (integer_type_node
, al
);
3122 /* Return maximum possible vectorization factor for the target. */
3129 || !flag_tree_loop_optimize
3130 || (!flag_tree_loop_vectorize
3131 && (global_options_set
.x_flag_tree_loop_vectorize
3132 || global_options_set
.x_flag_tree_vectorize
)))
3135 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3138 vs
= 1 << floor_log2 (vs
);
3141 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3142 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3143 return GET_MODE_NUNITS (vqimode
);
3147 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3151 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3152 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3156 max_vf
= omp_max_vf ();
3159 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3160 OMP_CLAUSE_SAFELEN
);
3161 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3163 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3165 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3169 idx
= create_tmp_var (unsigned_type_node
);
3170 lane
= create_tmp_var (unsigned_type_node
);
3176 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3177 tree avar
= create_tmp_var_raw (atype
);
3178 if (TREE_ADDRESSABLE (new_var
))
3179 TREE_ADDRESSABLE (avar
) = 1;
3180 DECL_ATTRIBUTES (avar
)
3181 = tree_cons (get_identifier ("omp simd array"), NULL
,
3182 DECL_ATTRIBUTES (avar
));
3183 gimple_add_tmp_var (avar
);
3184 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3185 NULL_TREE
, NULL_TREE
);
3186 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3187 NULL_TREE
, NULL_TREE
);
3188 if (DECL_P (new_var
))
3190 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3191 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3196 /* Helper function of lower_rec_input_clauses. For a reference
3197 in simd reduction, add an underlying variable it will reference. */
3200 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3202 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3203 if (TREE_CONSTANT (z
))
3205 const char *name
= NULL
;
3206 if (DECL_NAME (new_vard
))
3207 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3209 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3210 gimple_add_tmp_var (z
);
3211 TREE_ADDRESSABLE (z
) = 1;
3212 z
= build_fold_addr_expr_loc (loc
, z
);
3213 gimplify_assign (new_vard
, z
, ilist
);
3217 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3218 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3219 private variables. Initialization statements go in ILIST, while calls
3220 to destructors go in DLIST. */
3223 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3224 omp_context
*ctx
, struct omp_for_data
*fd
)
3226 tree c
, dtor
, copyin_seq
, x
, ptr
;
3227 bool copyin_by_ref
= false;
3228 bool lastprivate_firstprivate
= false;
3229 bool reduction_omp_orig_ref
= false;
3231 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3232 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3234 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3235 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3236 gimple_seq llist
[2] = { NULL
, NULL
};
3240 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3241 with data sharing clauses referencing variable sized vars. That
3242 is unnecessarily hard to support and very unlikely to result in
3243 vectorized code anyway. */
3245 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3246 switch (OMP_CLAUSE_CODE (c
))
3248 case OMP_CLAUSE_LINEAR
:
3249 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3252 case OMP_CLAUSE_REDUCTION
:
3253 case OMP_CLAUSE_PRIVATE
:
3254 case OMP_CLAUSE_FIRSTPRIVATE
:
3255 case OMP_CLAUSE_LASTPRIVATE
:
3256 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3263 /* Do all the fixed sized types in the first pass, and the variable sized
3264 types in the second pass. This makes sure that the scalar arguments to
3265 the variable sized types are processed before we use them in the
3266 variable sized operations. */
3267 for (pass
= 0; pass
< 2; ++pass
)
3269 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3271 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3274 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3278 case OMP_CLAUSE_PRIVATE
:
3279 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3282 case OMP_CLAUSE_SHARED
:
3283 /* Ignore shared directives in teams construct. */
3284 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3286 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3288 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3291 case OMP_CLAUSE_FIRSTPRIVATE
:
3292 case OMP_CLAUSE_COPYIN
:
3293 case OMP_CLAUSE_LINEAR
:
3295 case OMP_CLAUSE_REDUCTION
:
3296 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3297 reduction_omp_orig_ref
= true;
3299 case OMP_CLAUSE__LOOPTEMP_
:
3300 /* Handle _looptemp_ clauses only on parallel. */
3304 case OMP_CLAUSE_LASTPRIVATE
:
3305 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3307 lastprivate_firstprivate
= true;
3311 /* Even without corresponding firstprivate, if
3312 decl is Fortran allocatable, it needs outer var
3315 && lang_hooks
.decls
.omp_private_outer_ref
3316 (OMP_CLAUSE_DECL (c
)))
3317 lastprivate_firstprivate
= true;
3319 case OMP_CLAUSE_ALIGNED
:
3322 var
= OMP_CLAUSE_DECL (c
);
3323 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3324 && !is_global_var (var
))
3326 new_var
= maybe_lookup_decl (var
, ctx
);
3327 if (new_var
== NULL_TREE
)
3328 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3329 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3330 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3331 omp_clause_aligned_alignment (c
));
3332 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3333 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3334 gimplify_and_add (x
, ilist
);
3336 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3337 && is_global_var (var
))
3339 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3340 new_var
= lookup_decl (var
, ctx
);
3341 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3342 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3343 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3344 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3345 omp_clause_aligned_alignment (c
));
3346 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3347 x
= create_tmp_var (ptype
);
3348 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3349 gimplify_and_add (t
, ilist
);
3350 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3351 SET_DECL_VALUE_EXPR (new_var
, t
);
3352 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3359 new_var
= var
= OMP_CLAUSE_DECL (c
);
3360 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3361 new_var
= lookup_decl (var
, ctx
);
3363 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3368 else if (is_variable_sized (var
))
3370 /* For variable sized types, we need to allocate the
3371 actual storage here. Call alloca and store the
3372 result in the pointer decl that we created elsewhere. */
3376 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3381 ptr
= DECL_VALUE_EXPR (new_var
);
3382 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3383 ptr
= TREE_OPERAND (ptr
, 0);
3384 gcc_assert (DECL_P (ptr
));
3385 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3387 /* void *tmp = __builtin_alloca */
3388 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3389 stmt
= gimple_build_call (atmp
, 1, x
);
3390 tmp
= create_tmp_var_raw (ptr_type_node
);
3391 gimple_add_tmp_var (tmp
);
3392 gimple_call_set_lhs (stmt
, tmp
);
3394 gimple_seq_add_stmt (ilist
, stmt
);
3396 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3397 gimplify_assign (ptr
, x
, ilist
);
3400 else if (is_reference (var
))
3402 /* For references that are being privatized for Fortran,
3403 allocate new backing storage for the new pointer
3404 variable. This allows us to avoid changing all the
3405 code that expects a pointer to something that expects
3406 a direct variable. */
3410 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3411 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3413 x
= build_receiver_ref (var
, false, ctx
);
3414 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3416 else if (TREE_CONSTANT (x
))
3418 /* For reduction in SIMD loop, defer adding the
3419 initialization of the reference, because if we decide
3420 to use SIMD array for it, the initilization could cause
3422 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3426 const char *name
= NULL
;
3427 if (DECL_NAME (var
))
3428 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3430 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3432 gimple_add_tmp_var (x
);
3433 TREE_ADDRESSABLE (x
) = 1;
3434 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3439 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3440 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3445 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3446 gimplify_assign (new_var
, x
, ilist
);
3449 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3451 else if (c_kind
== OMP_CLAUSE_REDUCTION
3452 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3460 switch (OMP_CLAUSE_CODE (c
))
3462 case OMP_CLAUSE_SHARED
:
3463 /* Ignore shared directives in teams construct. */
3464 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3466 /* Shared global vars are just accessed directly. */
3467 if (is_global_var (new_var
))
3469 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3470 needs to be delayed until after fixup_child_record_type so
3471 that we get the correct type during the dereference. */
3472 by_ref
= use_pointer_for_field (var
, ctx
);
3473 x
= build_receiver_ref (var
, by_ref
, ctx
);
3474 SET_DECL_VALUE_EXPR (new_var
, x
);
3475 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3477 /* ??? If VAR is not passed by reference, and the variable
3478 hasn't been initialized yet, then we'll get a warning for
3479 the store into the omp_data_s structure. Ideally, we'd be
3480 able to notice this and not store anything at all, but
3481 we're generating code too early. Suppress the warning. */
3483 TREE_NO_WARNING (var
) = 1;
3486 case OMP_CLAUSE_LASTPRIVATE
:
3487 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3491 case OMP_CLAUSE_PRIVATE
:
3492 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3493 x
= build_outer_var_ref (var
, ctx
);
3494 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3496 if (is_task_ctx (ctx
))
3497 x
= build_receiver_ref (var
, false, ctx
);
3499 x
= build_outer_var_ref (var
, ctx
);
3505 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3508 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3509 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3510 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3511 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3512 idx
, lane
, ivar
, lvar
))
3515 x
= lang_hooks
.decls
.omp_clause_default_ctor
3516 (c
, unshare_expr (ivar
), x
);
3518 gimplify_and_add (x
, &llist
[0]);
3521 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3524 gimple_seq tseq
= NULL
;
3527 gimplify_stmt (&dtor
, &tseq
);
3528 gimple_seq_add_seq (&llist
[1], tseq
);
3535 gimplify_and_add (nx
, ilist
);
3539 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3542 gimple_seq tseq
= NULL
;
3545 gimplify_stmt (&dtor
, &tseq
);
3546 gimple_seq_add_seq (dlist
, tseq
);
3550 case OMP_CLAUSE_LINEAR
:
3551 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3552 goto do_firstprivate
;
3553 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3556 x
= build_outer_var_ref (var
, ctx
);
3559 case OMP_CLAUSE_FIRSTPRIVATE
:
3560 if (is_task_ctx (ctx
))
3562 if (is_reference (var
) || is_variable_sized (var
))
3564 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3566 || use_pointer_for_field (var
, NULL
))
3568 x
= build_receiver_ref (var
, false, ctx
);
3569 SET_DECL_VALUE_EXPR (new_var
, x
);
3570 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3575 x
= build_outer_var_ref (var
, ctx
);
3578 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3579 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3581 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3582 tree stept
= TREE_TYPE (t
);
3583 tree ct
= find_omp_clause (clauses
,
3584 OMP_CLAUSE__LOOPTEMP_
);
3586 tree l
= OMP_CLAUSE_DECL (ct
);
3587 tree n1
= fd
->loop
.n1
;
3588 tree step
= fd
->loop
.step
;
3589 tree itype
= TREE_TYPE (l
);
3590 if (POINTER_TYPE_P (itype
))
3591 itype
= signed_type_for (itype
);
3592 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3593 if (TYPE_UNSIGNED (itype
)
3594 && fd
->loop
.cond_code
== GT_EXPR
)
3595 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3596 fold_build1 (NEGATE_EXPR
, itype
, l
),
3597 fold_build1 (NEGATE_EXPR
,
3600 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3601 t
= fold_build2 (MULT_EXPR
, stept
,
3602 fold_convert (stept
, l
), t
);
3604 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3606 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3608 gimplify_and_add (x
, ilist
);
3612 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3613 x
= fold_build2 (POINTER_PLUS_EXPR
,
3614 TREE_TYPE (x
), x
, t
);
3616 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3619 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3620 || TREE_ADDRESSABLE (new_var
))
3621 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3622 idx
, lane
, ivar
, lvar
))
3624 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3626 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
3627 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3628 gimplify_and_add (x
, ilist
);
3629 gimple_stmt_iterator gsi
3630 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3632 = gimple_build_assign (unshare_expr (lvar
), iv
);
3633 gsi_insert_before_without_update (&gsi
, g
,
3635 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3636 enum tree_code code
= PLUS_EXPR
;
3637 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3638 code
= POINTER_PLUS_EXPR
;
3639 g
= gimple_build_assign (iv
, code
, iv
, t
);
3640 gsi_insert_before_without_update (&gsi
, g
,
3644 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3645 (c
, unshare_expr (ivar
), x
);
3646 gimplify_and_add (x
, &llist
[0]);
3647 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3650 gimple_seq tseq
= NULL
;
3653 gimplify_stmt (&dtor
, &tseq
);
3654 gimple_seq_add_seq (&llist
[1], tseq
);
3659 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3660 gimplify_and_add (x
, ilist
);
3663 case OMP_CLAUSE__LOOPTEMP_
:
3664 gcc_assert (is_parallel_ctx (ctx
));
3665 x
= build_outer_var_ref (var
, ctx
);
3666 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3667 gimplify_and_add (x
, ilist
);
3670 case OMP_CLAUSE_COPYIN
:
3671 by_ref
= use_pointer_for_field (var
, NULL
);
3672 x
= build_receiver_ref (var
, by_ref
, ctx
);
3673 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3674 append_to_statement_list (x
, ©in_seq
);
3675 copyin_by_ref
|= by_ref
;
3678 case OMP_CLAUSE_REDUCTION
:
3679 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3681 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3683 x
= build_outer_var_ref (var
, ctx
);
3685 if (is_reference (var
)
3686 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3688 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3689 SET_DECL_VALUE_EXPR (placeholder
, x
);
3690 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3691 tree new_vard
= new_var
;
3692 if (is_reference (var
))
3694 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3695 new_vard
= TREE_OPERAND (new_var
, 0);
3696 gcc_assert (DECL_P (new_vard
));
3699 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3700 idx
, lane
, ivar
, lvar
))
3702 if (new_vard
== new_var
)
3704 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3705 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3709 SET_DECL_VALUE_EXPR (new_vard
,
3710 build_fold_addr_expr (ivar
));
3711 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3713 x
= lang_hooks
.decls
.omp_clause_default_ctor
3714 (c
, unshare_expr (ivar
),
3715 build_outer_var_ref (var
, ctx
));
3717 gimplify_and_add (x
, &llist
[0]);
3718 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3720 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3721 lower_omp (&tseq
, ctx
);
3722 gimple_seq_add_seq (&llist
[0], tseq
);
3724 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3725 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3726 lower_omp (&tseq
, ctx
);
3727 gimple_seq_add_seq (&llist
[1], tseq
);
3728 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3729 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3730 if (new_vard
== new_var
)
3731 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3733 SET_DECL_VALUE_EXPR (new_vard
,
3734 build_fold_addr_expr (lvar
));
3735 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3740 gimplify_stmt (&dtor
, &tseq
);
3741 gimple_seq_add_seq (&llist
[1], tseq
);
3745 /* If this is a reference to constant size reduction var
3746 with placeholder, we haven't emitted the initializer
3747 for it because it is undesirable if SIMD arrays are used.
3748 But if they aren't used, we need to emit the deferred
3749 initialization now. */
3750 else if (is_reference (var
) && is_simd
)
3751 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3752 x
= lang_hooks
.decls
.omp_clause_default_ctor
3753 (c
, unshare_expr (new_var
),
3754 build_outer_var_ref (var
, ctx
));
3756 gimplify_and_add (x
, ilist
);
3757 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3759 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3760 lower_omp (&tseq
, ctx
);
3761 gimple_seq_add_seq (ilist
, tseq
);
3763 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3766 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3767 lower_omp (&tseq
, ctx
);
3768 gimple_seq_add_seq (dlist
, tseq
);
3769 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3771 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3776 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3777 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3778 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3780 /* reduction(-:var) sums up the partial results, so it
3781 acts identically to reduction(+:var). */
3782 if (code
== MINUS_EXPR
)
3785 tree new_vard
= new_var
;
3786 if (is_simd
&& is_reference (var
))
3788 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3789 new_vard
= TREE_OPERAND (new_var
, 0);
3790 gcc_assert (DECL_P (new_vard
));
3793 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3794 idx
, lane
, ivar
, lvar
))
3796 tree ref
= build_outer_var_ref (var
, ctx
);
3798 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3800 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3801 ref
= build_outer_var_ref (var
, ctx
);
3802 gimplify_assign (ref
, x
, &llist
[1]);
3804 if (new_vard
!= new_var
)
3806 SET_DECL_VALUE_EXPR (new_vard
,
3807 build_fold_addr_expr (lvar
));
3808 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3813 if (is_reference (var
) && is_simd
)
3814 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3815 gimplify_assign (new_var
, x
, ilist
);
3818 tree ref
= build_outer_var_ref (var
, ctx
);
3820 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3821 ref
= build_outer_var_ref (var
, ctx
);
3822 gimplify_assign (ref
, x
, dlist
);
3836 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3837 /* Don't want uninit warnings on simduid, it is always uninitialized,
3838 but we use it not for the value, but for the DECL_UID only. */
3839 TREE_NO_WARNING (uid
) = 1;
3841 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3842 gimple_call_set_lhs (g
, lane
);
3843 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3844 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3845 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3846 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3847 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3848 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3849 g
= gimple_build_assign (lane
, INTEGER_CST
,
3850 build_int_cst (unsigned_type_node
, 0));
3851 gimple_seq_add_stmt (ilist
, g
);
3852 for (int i
= 0; i
< 2; i
++)
3855 tree vf
= create_tmp_var (unsigned_type_node
);
3856 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3857 gimple_call_set_lhs (g
, vf
);
3858 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3859 gimple_seq_add_stmt (seq
, g
);
3860 tree t
= build_int_cst (unsigned_type_node
, 0);
3861 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
3862 gimple_seq_add_stmt (seq
, g
);
3863 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3864 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3865 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3866 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3867 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3868 gimple_seq_add_seq (seq
, llist
[i
]);
3869 t
= build_int_cst (unsigned_type_node
, 1);
3870 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
3871 gimple_seq_add_stmt (seq
, g
);
3872 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3873 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3874 gimple_seq_add_stmt (seq
, g
);
3875 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3879 /* The copyin sequence is not to be executed by the main thread, since
3880 that would result in self-copies. Perhaps not visible to scalars,
3881 but it certainly is to C++ operator=. */
3884 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3886 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3887 build_int_cst (TREE_TYPE (x
), 0));
3888 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3889 gimplify_and_add (x
, ilist
);
3892 /* If any copyin variable is passed by reference, we must ensure the
3893 master thread doesn't modify it before it is copied over in all
3894 threads. Similarly for variables in both firstprivate and
3895 lastprivate clauses we need to ensure the lastprivate copying
3896 happens after firstprivate copying in all threads. And similarly
3897 for UDRs if initializer expression refers to omp_orig. */
3898 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3900 /* Don't add any barrier for #pragma omp simd or
3901 #pragma omp distribute. */
3902 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3903 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3904 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3907 /* If max_vf is non-zero, then we can use only a vectorization factor
3908 up to the max_vf we chose. So stick it into the safelen clause. */
3911 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3912 OMP_CLAUSE_SAFELEN
);
3914 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3915 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3918 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3919 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3921 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3922 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3928 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3929 both parallel and workshare constructs. PREDICATE may be NULL if it's
3933 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3936 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3937 bool par_clauses
= false;
3938 tree simduid
= NULL
, lastlane
= NULL
;
3940 /* Early exit if there are no lastprivate or linear clauses. */
3941 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3942 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3943 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3944 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3946 if (clauses
== NULL
)
3948 /* If this was a workshare clause, see if it had been combined
3949 with its parallel. In that case, look for the clauses on the
3950 parallel statement itself. */
3951 if (is_parallel_ctx (ctx
))
3955 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3958 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3959 OMP_CLAUSE_LASTPRIVATE
);
3960 if (clauses
== NULL
)
3968 tree label_true
, arm1
, arm2
;
3970 label
= create_artificial_label (UNKNOWN_LOCATION
);
3971 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3972 arm1
= TREE_OPERAND (predicate
, 0);
3973 arm2
= TREE_OPERAND (predicate
, 1);
3974 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3975 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3976 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3978 gimple_seq_add_stmt (stmt_list
, stmt
);
3979 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3982 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3983 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3985 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3987 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3990 for (c
= clauses
; c
;)
3993 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3995 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3996 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3997 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3999 var
= OMP_CLAUSE_DECL (c
);
4000 new_var
= lookup_decl (var
, ctx
);
4002 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4004 tree val
= DECL_VALUE_EXPR (new_var
);
4005 if (TREE_CODE (val
) == ARRAY_REF
4006 && VAR_P (TREE_OPERAND (val
, 0))
4007 && lookup_attribute ("omp simd array",
4008 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4011 if (lastlane
== NULL
)
4013 lastlane
= create_tmp_var (unsigned_type_node
);
4015 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4017 TREE_OPERAND (val
, 1));
4018 gimple_call_set_lhs (g
, lastlane
);
4019 gimple_seq_add_stmt (stmt_list
, g
);
4021 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4022 TREE_OPERAND (val
, 0), lastlane
,
4023 NULL_TREE
, NULL_TREE
);
4027 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4028 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4030 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4031 gimple_seq_add_seq (stmt_list
,
4032 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4033 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4035 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4036 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4038 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4039 gimple_seq_add_seq (stmt_list
,
4040 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4041 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4044 x
= build_outer_var_ref (var
, ctx
);
4045 if (is_reference (var
))
4046 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4047 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4048 gimplify_and_add (x
, stmt_list
);
4050 c
= OMP_CLAUSE_CHAIN (c
);
4051 if (c
== NULL
&& !par_clauses
)
4053 /* If this was a workshare clause, see if it had been combined
4054 with its parallel. In that case, continue looking for the
4055 clauses also on the parallel statement itself. */
4056 if (is_parallel_ctx (ctx
))
4060 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4063 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4064 OMP_CLAUSE_LASTPRIVATE
);
4070 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4074 /* Generate code to implement the REDUCTION clauses. */
4077 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4079 gimple_seq sub_seq
= NULL
;
4084 /* SIMD reductions are handled in lower_rec_input_clauses. */
4085 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4086 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4089 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4090 update in that case, otherwise use a lock. */
4091 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4092 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4094 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4096 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4106 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4108 tree var
, ref
, new_var
;
4109 enum tree_code code
;
4110 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4112 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4115 var
= OMP_CLAUSE_DECL (c
);
4116 new_var
= lookup_decl (var
, ctx
);
4117 if (is_reference (var
))
4118 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4119 ref
= build_outer_var_ref (var
, ctx
);
4120 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4122 /* reduction(-:var) sums up the partial results, so it acts
4123 identically to reduction(+:var). */
4124 if (code
== MINUS_EXPR
)
4129 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4131 addr
= save_expr (addr
);
4132 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4133 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4134 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4135 gimplify_and_add (x
, stmt_seqp
);
4139 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4141 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4143 if (is_reference (var
)
4144 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4146 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4147 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4148 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4149 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4150 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4151 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4152 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4156 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4157 ref
= build_outer_var_ref (var
, ctx
);
4158 gimplify_assign (ref
, x
, &sub_seq
);
4162 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4164 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4166 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4168 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4170 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4174 /* Generate code to implement the COPYPRIVATE clauses. */
4177 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4182 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4184 tree var
, new_var
, ref
, x
;
4186 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4188 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4191 var
= OMP_CLAUSE_DECL (c
);
4192 by_ref
= use_pointer_for_field (var
, NULL
);
4194 ref
= build_sender_ref (var
, ctx
);
4195 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4198 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4199 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4201 gimplify_assign (ref
, x
, slist
);
4203 ref
= build_receiver_ref (var
, false, ctx
);
4206 ref
= fold_convert_loc (clause_loc
,
4207 build_pointer_type (TREE_TYPE (new_var
)),
4209 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4211 if (is_reference (var
))
4213 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4214 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4215 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4217 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4218 gimplify_and_add (x
, rlist
);
4223 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4224 and REDUCTION from the sender (aka parent) side. */
4227 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4232 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4234 tree val
, ref
, x
, var
;
4235 bool by_ref
, do_in
= false, do_out
= false;
4236 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4238 switch (OMP_CLAUSE_CODE (c
))
4240 case OMP_CLAUSE_PRIVATE
:
4241 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4244 case OMP_CLAUSE_FIRSTPRIVATE
:
4245 case OMP_CLAUSE_COPYIN
:
4246 case OMP_CLAUSE_LASTPRIVATE
:
4247 case OMP_CLAUSE_REDUCTION
:
4248 case OMP_CLAUSE__LOOPTEMP_
:
4254 val
= OMP_CLAUSE_DECL (c
);
4255 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4257 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4258 && is_global_var (var
))
4260 if (is_variable_sized (val
))
4262 by_ref
= use_pointer_for_field (val
, NULL
);
4264 switch (OMP_CLAUSE_CODE (c
))
4266 case OMP_CLAUSE_PRIVATE
:
4267 case OMP_CLAUSE_FIRSTPRIVATE
:
4268 case OMP_CLAUSE_COPYIN
:
4269 case OMP_CLAUSE__LOOPTEMP_
:
4273 case OMP_CLAUSE_LASTPRIVATE
:
4274 if (by_ref
|| is_reference (val
))
4276 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4283 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4288 case OMP_CLAUSE_REDUCTION
:
4290 do_out
= !(by_ref
|| is_reference (val
));
4299 ref
= build_sender_ref (val
, ctx
);
4300 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4301 gimplify_assign (ref
, x
, ilist
);
4302 if (is_task_ctx (ctx
))
4303 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4308 ref
= build_sender_ref (val
, ctx
);
4309 gimplify_assign (var
, ref
, olist
);
4314 /* Generate code to implement SHARED from the sender (aka parent)
4315 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4316 list things that got automatically shared. */
4319 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4321 tree var
, ovar
, nvar
, f
, x
, record_type
;
4323 if (ctx
->record_type
== NULL
)
4326 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4327 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4329 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4330 nvar
= maybe_lookup_decl (ovar
, ctx
);
4331 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4334 /* If CTX is a nested parallel directive. Find the immediately
4335 enclosing parallel or workshare construct that contains a
4336 mapping for OVAR. */
4337 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4339 if (use_pointer_for_field (ovar
, ctx
))
4341 x
= build_sender_ref (ovar
, ctx
);
4342 var
= build_fold_addr_expr (var
);
4343 gimplify_assign (x
, var
, ilist
);
4347 x
= build_sender_ref (ovar
, ctx
);
4348 gimplify_assign (x
, var
, ilist
);
4350 if (!TREE_READONLY (var
)
4351 /* We don't need to receive a new reference to a result
4352 or parm decl. In fact we may not store to it as we will
4353 invalidate any pending RSO and generate wrong gimple
4355 && !((TREE_CODE (var
) == RESULT_DECL
4356 || TREE_CODE (var
) == PARM_DECL
)
4357 && DECL_BY_REFERENCE (var
)))
4359 x
= build_sender_ref (ovar
, ctx
);
4360 gimplify_assign (var
, x
, olist
);
4367 /* A convenience function to build an empty GIMPLE_COND with just the
4371 gimple_build_cond_empty (tree cond
)
4373 enum tree_code pred_code
;
4376 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4377 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4381 /* Build the function calls to GOMP_parallel_start etc to actually
4382 generate the parallel operation. REGION is the parallel region
4383 being expanded. BB is the block where to insert the code. WS_ARGS
4384 will be set if this is a call to a combined parallel+workshare
4385 construct, it contains the list of additional arguments needed by
4386 the workshare construct. */
4389 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4390 gomp_parallel
*entry_stmt
,
4391 vec
<tree
, va_gc
> *ws_args
)
4393 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4394 gimple_stmt_iterator gsi
;
4396 enum built_in_function start_ix
;
4398 location_t clause_loc
;
4399 vec
<tree
, va_gc
> *args
;
4401 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4403 /* Determine what flavor of GOMP_parallel we will be
4405 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4406 if (is_combined_parallel (region
))
4408 switch (region
->inner
->type
)
4410 case GIMPLE_OMP_FOR
:
4411 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4412 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4413 + (region
->inner
->sched_kind
4414 == OMP_CLAUSE_SCHEDULE_RUNTIME
4415 ? 3 : region
->inner
->sched_kind
));
4416 start_ix
= (enum built_in_function
)start_ix2
;
4418 case GIMPLE_OMP_SECTIONS
:
4419 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4426 /* By default, the value of NUM_THREADS is zero (selected at run time)
4427 and there is no conditional. */
4429 val
= build_int_cst (unsigned_type_node
, 0);
4430 flags
= build_int_cst (unsigned_type_node
, 0);
4432 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4434 cond
= OMP_CLAUSE_IF_EXPR (c
);
4436 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4439 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4440 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4443 clause_loc
= gimple_location (entry_stmt
);
4445 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4447 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4449 /* Ensure 'val' is of the correct type. */
4450 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4452 /* If we found the clause 'if (cond)', build either
4453 (cond != 0) or (cond ? val : 1u). */
4456 cond
= gimple_boolify (cond
);
4458 if (integer_zerop (val
))
4459 val
= fold_build2_loc (clause_loc
,
4460 EQ_EXPR
, unsigned_type_node
, cond
,
4461 build_int_cst (TREE_TYPE (cond
), 0));
4464 basic_block cond_bb
, then_bb
, else_bb
;
4465 edge e
, e_then
, e_else
;
4466 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4468 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4469 if (gimple_in_ssa_p (cfun
))
4471 tmp_then
= make_ssa_name (tmp_var
);
4472 tmp_else
= make_ssa_name (tmp_var
);
4473 tmp_join
= make_ssa_name (tmp_var
);
4482 e
= split_block (bb
, NULL
);
4487 then_bb
= create_empty_bb (cond_bb
);
4488 else_bb
= create_empty_bb (then_bb
);
4489 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4490 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4492 stmt
= gimple_build_cond_empty (cond
);
4493 gsi
= gsi_start_bb (cond_bb
);
4494 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4496 gsi
= gsi_start_bb (then_bb
);
4497 stmt
= gimple_build_assign (tmp_then
, val
);
4498 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4500 gsi
= gsi_start_bb (else_bb
);
4501 stmt
= gimple_build_assign
4502 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4503 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4505 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4506 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4507 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4508 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4509 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4510 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4512 if (gimple_in_ssa_p (cfun
))
4514 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4515 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4516 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4522 gsi
= gsi_start_bb (bb
);
4523 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4524 false, GSI_CONTINUE_LINKING
);
4527 gsi
= gsi_last_bb (bb
);
4528 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4530 t1
= null_pointer_node
;
4532 t1
= build_fold_addr_expr (t
);
4533 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4535 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4536 args
->quick_push (t2
);
4537 args
->quick_push (t1
);
4538 args
->quick_push (val
);
4540 args
->splice (*ws_args
);
4541 args
->quick_push (flags
);
4543 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4544 builtin_decl_explicit (start_ix
), args
);
4546 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4547 false, GSI_CONTINUE_LINKING
);
4550 /* Insert a function call whose name is FUNC_NAME with the information from
4551 ENTRY_STMT into the basic_block BB. */
4554 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
4555 vec
<tree
, va_gc
> *ws_args
)
4558 gimple_stmt_iterator gsi
;
4559 vec
<tree
, va_gc
> *args
;
4561 gcc_assert (vec_safe_length (ws_args
) == 2);
4562 tree func_name
= (*ws_args
)[0];
4563 tree grain
= (*ws_args
)[1];
4565 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4566 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
4567 gcc_assert (count
!= NULL_TREE
);
4568 count
= OMP_CLAUSE_OPERAND (count
, 0);
4570 gsi
= gsi_last_bb (bb
);
4571 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4573 t1
= null_pointer_node
;
4575 t1
= build_fold_addr_expr (t
);
4576 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4578 vec_alloc (args
, 4);
4579 args
->quick_push (t2
);
4580 args
->quick_push (t1
);
4581 args
->quick_push (count
);
4582 args
->quick_push (grain
);
4583 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
4585 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
4586 GSI_CONTINUE_LINKING
);
4589 /* Build the function call to GOMP_task to actually
4590 generate the task operation. BB is the block where to insert the code. */
4593 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
4595 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4596 gimple_stmt_iterator gsi
;
4597 location_t loc
= gimple_location (entry_stmt
);
4599 clauses
= gimple_omp_task_clauses (entry_stmt
);
4601 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4603 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4605 cond
= boolean_true_node
;
4607 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4608 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4609 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4610 flags
= build_int_cst (unsigned_type_node
,
4611 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4613 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4616 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4617 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4618 build_int_cst (unsigned_type_node
, 2),
4619 build_int_cst (unsigned_type_node
, 0));
4620 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4623 depend
= OMP_CLAUSE_DECL (depend
);
4625 depend
= build_int_cst (ptr_type_node
, 0);
4627 gsi
= gsi_last_bb (bb
);
4628 t
= gimple_omp_task_data_arg (entry_stmt
);
4630 t2
= null_pointer_node
;
4632 t2
= build_fold_addr_expr_loc (loc
, t
);
4633 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4634 t
= gimple_omp_task_copy_fn (entry_stmt
);
4636 t3
= null_pointer_node
;
4638 t3
= build_fold_addr_expr_loc (loc
, t
);
4640 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4642 gimple_omp_task_arg_size (entry_stmt
),
4643 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4646 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4647 false, GSI_CONTINUE_LINKING
);
4651 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4652 catch handler and return it. This prevents programs from violating the
4653 structured block semantics with throws. */
4656 maybe_catch_exception (gimple_seq body
)
4661 if (!flag_exceptions
)
4664 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4665 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4667 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4669 g
= gimple_build_eh_must_not_throw (decl
);
4670 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4673 return gimple_seq_alloc_with_stmt (g
);
4676 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4679 vec2chain (vec
<tree
, va_gc
> *v
)
4681 tree chain
= NULL_TREE
, t
;
4684 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4686 DECL_CHAIN (t
) = chain
;
4694 /* Remove barriers in REGION->EXIT's block. Note that this is only
4695 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4696 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4697 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4701 remove_exit_barrier (struct omp_region
*region
)
4703 gimple_stmt_iterator gsi
;
4704 basic_block exit_bb
;
4708 int any_addressable_vars
= -1;
4710 exit_bb
= region
->exit
;
4712 /* If the parallel region doesn't return, we don't have REGION->EXIT
4717 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4718 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4719 statements that can appear in between are extremely limited -- no
4720 memory operations at all. Here, we allow nothing at all, so the
4721 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4722 gsi
= gsi_last_bb (exit_bb
);
4723 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4725 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4728 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4730 gsi
= gsi_last_bb (e
->src
);
4731 if (gsi_end_p (gsi
))
4733 stmt
= gsi_stmt (gsi
);
4734 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4735 && !gimple_omp_return_nowait_p (stmt
))
4737 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4738 in many cases. If there could be tasks queued, the barrier
4739 might be needed to let the tasks run before some local
4740 variable of the parallel that the task uses as shared
4741 runs out of scope. The task can be spawned either
4742 from within current function (this would be easy to check)
4743 or from some function it calls and gets passed an address
4744 of such a variable. */
4745 if (any_addressable_vars
< 0)
4747 gomp_parallel
*parallel_stmt
4748 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
4749 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4750 tree local_decls
, block
, decl
;
4753 any_addressable_vars
= 0;
4754 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4755 if (TREE_ADDRESSABLE (decl
))
4757 any_addressable_vars
= 1;
4760 for (block
= gimple_block (stmt
);
4761 !any_addressable_vars
4763 && TREE_CODE (block
) == BLOCK
;
4764 block
= BLOCK_SUPERCONTEXT (block
))
4766 for (local_decls
= BLOCK_VARS (block
);
4768 local_decls
= DECL_CHAIN (local_decls
))
4769 if (TREE_ADDRESSABLE (local_decls
))
4771 any_addressable_vars
= 1;
4774 if (block
== gimple_block (parallel_stmt
))
4778 if (!any_addressable_vars
)
4779 gimple_omp_return_set_nowait (stmt
);
4785 remove_exit_barriers (struct omp_region
*region
)
4787 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4788 remove_exit_barrier (region
);
4792 region
= region
->inner
;
4793 remove_exit_barriers (region
);
4794 while (region
->next
)
4796 region
= region
->next
;
4797 remove_exit_barriers (region
);
4802 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4803 calls. These can't be declared as const functions, but
4804 within one parallel body they are constant, so they can be
4805 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4806 which are declared const. Similarly for task body, except
4807 that in untied task omp_get_thread_num () can change at any task
4808 scheduling point. */
4811 optimize_omp_library_calls (gimple entry_stmt
)
4814 gimple_stmt_iterator gsi
;
4815 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4816 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4817 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4818 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4819 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4820 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4821 OMP_CLAUSE_UNTIED
) != NULL
);
4823 FOR_EACH_BB_FN (bb
, cfun
)
4824 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4826 gimple call
= gsi_stmt (gsi
);
4829 if (is_gimple_call (call
)
4830 && (decl
= gimple_call_fndecl (call
))
4831 && DECL_EXTERNAL (decl
)
4832 && TREE_PUBLIC (decl
)
4833 && DECL_INITIAL (decl
) == NULL
)
4837 if (DECL_NAME (decl
) == thr_num_id
)
4839 /* In #pragma omp task untied omp_get_thread_num () can change
4840 during the execution of the task region. */
4843 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4845 else if (DECL_NAME (decl
) == num_thr_id
)
4846 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4850 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4851 || gimple_call_num_args (call
) != 0)
4854 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4857 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4858 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4859 TREE_TYPE (TREE_TYPE (built_in
))))
4862 gimple_call_set_fndecl (call
, built_in
);
4867 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4871 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4875 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4876 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4879 if (TREE_CODE (t
) == ADDR_EXPR
)
4880 recompute_tree_invariant_for_addr_expr (t
);
4882 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4886 /* Prepend TO = FROM assignment before *GSI_P. */
4889 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4891 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4892 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4893 true, GSI_SAME_STMT
);
4894 gimple stmt
= gimple_build_assign (to
, from
);
4895 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4896 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4897 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4899 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4900 gimple_regimplify_operands (stmt
, &gsi
);
4904 /* Expand the OpenMP parallel or task directive starting at REGION. */
4907 expand_omp_taskreg (struct omp_region
*region
)
4909 basic_block entry_bb
, exit_bb
, new_bb
;
4910 struct function
*child_cfun
;
4911 tree child_fn
, block
, t
;
4912 gimple_stmt_iterator gsi
;
4913 gimple entry_stmt
, stmt
;
4915 vec
<tree
, va_gc
> *ws_args
;
4917 entry_stmt
= last_stmt (region
->entry
);
4918 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4919 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4921 entry_bb
= region
->entry
;
4922 exit_bb
= region
->exit
;
4926 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
4927 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
4928 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
4931 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
4932 and the inner statement contains the name of the built-in function
4934 ws_args
= region
->inner
->ws_args
;
4935 else if (is_combined_parallel (region
))
4936 ws_args
= region
->ws_args
;
4940 if (child_cfun
->cfg
)
4942 /* Due to inlining, it may happen that we have already outlined
4943 the region, in which case all we need to do is make the
4944 sub-graph unreachable and emit the parallel call. */
4945 edge entry_succ_e
, exit_succ_e
;
4947 entry_succ_e
= single_succ_edge (entry_bb
);
4949 gsi
= gsi_last_bb (entry_bb
);
4950 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4951 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4952 gsi_remove (&gsi
, true);
4957 exit_succ_e
= single_succ_edge (exit_bb
);
4958 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4960 remove_edge_and_dominated_blocks (entry_succ_e
);
4964 unsigned srcidx
, dstidx
, num
;
4966 /* If the parallel region needs data sent from the parent
4967 function, then the very first statement (except possible
4968 tree profile counter updates) of the parallel body
4969 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4970 &.OMP_DATA_O is passed as an argument to the child function,
4971 we need to replace it with the argument as seen by the child
4974 In most cases, this will end up being the identity assignment
4975 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4976 a function call that has been inlined, the original PARM_DECL
4977 .OMP_DATA_I may have been converted into a different local
4978 variable. In which case, we need to keep the assignment. */
4979 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4981 basic_block entry_succ_bb
= single_succ (entry_bb
);
4983 gimple parcopy_stmt
= NULL
;
4985 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4989 gcc_assert (!gsi_end_p (gsi
));
4990 stmt
= gsi_stmt (gsi
);
4991 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4994 if (gimple_num_ops (stmt
) == 2)
4996 tree arg
= gimple_assign_rhs1 (stmt
);
4998 /* We're ignore the subcode because we're
4999 effectively doing a STRIP_NOPS. */
5001 if (TREE_CODE (arg
) == ADDR_EXPR
5002 && TREE_OPERAND (arg
, 0)
5003 == gimple_omp_taskreg_data_arg (entry_stmt
))
5005 parcopy_stmt
= stmt
;
5011 gcc_assert (parcopy_stmt
!= NULL
);
5012 arg
= DECL_ARGUMENTS (child_fn
);
5014 if (!gimple_in_ssa_p (cfun
))
5016 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5017 gsi_remove (&gsi
, true);
5020 /* ?? Is setting the subcode really necessary ?? */
5021 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5022 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5027 /* If we are in ssa form, we must load the value from the default
5028 definition of the argument. That should not be defined now,
5029 since the argument is not used uninitialized. */
5030 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5031 narg
= make_ssa_name (arg
, gimple_build_nop ());
5032 set_ssa_default_def (cfun
, arg
, narg
);
5033 /* ?? Is setting the subcode really necessary ?? */
5034 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5035 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5036 update_stmt (parcopy_stmt
);
5040 /* Declare local variables needed in CHILD_CFUN. */
5041 block
= DECL_INITIAL (child_fn
);
5042 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5043 /* The gimplifier could record temporaries in parallel/task block
5044 rather than in containing function's local_decls chain,
5045 which would mean cgraph missed finalizing them. Do it now. */
5046 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5047 if (TREE_CODE (t
) == VAR_DECL
5049 && !DECL_EXTERNAL (t
))
5050 varpool_node::finalize_decl (t
);
5051 DECL_SAVED_TREE (child_fn
) = NULL
;
5052 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5053 gimple_set_body (child_fn
, NULL
);
5054 TREE_USED (block
) = 1;
5056 /* Reset DECL_CONTEXT on function arguments. */
5057 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5058 DECL_CONTEXT (t
) = child_fn
;
5060 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5061 so that it can be moved to the child function. */
5062 gsi
= gsi_last_bb (entry_bb
);
5063 stmt
= gsi_stmt (gsi
);
5064 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5065 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5066 gsi_remove (&gsi
, true);
5067 e
= split_block (entry_bb
, stmt
);
5069 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5071 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5074 gsi
= gsi_last_bb (exit_bb
);
5075 gcc_assert (!gsi_end_p (gsi
)
5076 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5077 stmt
= gimple_build_return (NULL
);
5078 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5079 gsi_remove (&gsi
, true);
5082 /* Move the parallel region into CHILD_CFUN. */
5084 if (gimple_in_ssa_p (cfun
))
5086 init_tree_ssa (child_cfun
);
5087 init_ssa_operands (child_cfun
);
5088 child_cfun
->gimple_df
->in_ssa_p
= true;
5092 block
= gimple_block (entry_stmt
);
5094 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5096 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5097 /* When the OMP expansion process cannot guarantee an up-to-date
5098 loop tree arrange for the child function to fixup loops. */
5099 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5100 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5102 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5103 num
= vec_safe_length (child_cfun
->local_decls
);
5104 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5106 t
= (*child_cfun
->local_decls
)[srcidx
];
5107 if (DECL_CONTEXT (t
) == cfun
->decl
)
5109 if (srcidx
!= dstidx
)
5110 (*child_cfun
->local_decls
)[dstidx
] = t
;
5114 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5116 /* Inform the callgraph about the new function. */
5117 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5118 cgraph_node::add_new_function (child_fn
, true);
5120 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5121 fixed in a following pass. */
5122 push_cfun (child_cfun
);
5124 optimize_omp_library_calls (entry_stmt
);
5125 cgraph_edge::rebuild_edges ();
5127 /* Some EH regions might become dead, see PR34608. If
5128 pass_cleanup_cfg isn't the first pass to happen with the
5129 new child, these dead EH edges might cause problems.
5130 Clean them up now. */
5131 if (flag_exceptions
)
5134 bool changed
= false;
5136 FOR_EACH_BB_FN (bb
, cfun
)
5137 changed
|= gimple_purge_dead_eh_edges (bb
);
5139 cleanup_tree_cfg ();
5141 if (gimple_in_ssa_p (cfun
))
5142 update_ssa (TODO_update_ssa
);
5146 /* Emit a library call to launch the children threads. */
5148 expand_cilk_for_call (new_bb
,
5149 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5150 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5151 expand_parallel_call (region
, new_bb
,
5152 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5154 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5155 if (gimple_in_ssa_p (cfun
))
5156 update_ssa (TODO_update_ssa_only_virtuals
);
5160 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5161 of the combined collapse > 1 loop constructs, generate code like:
5162 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5167 count3 = (adj + N32 - N31) / STEP3;
5168 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5173 count2 = (adj + N22 - N21) / STEP2;
5174 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5179 count1 = (adj + N12 - N11) / STEP1;
5180 count = count1 * count2 * count3;
5181 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5183 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5184 of the combined loop constructs, just initialize COUNTS array
5185 from the _looptemp_ clauses. */
5187 /* NOTE: It *could* be better to moosh all of the BBs together,
5188 creating one larger BB with all the computation and the unexpected
5189 jump at the end. I.e.
5191 bool zero3, zero2, zero1, zero;
5194 count3 = (N32 - N31) /[cl] STEP3;
5196 count2 = (N22 - N21) /[cl] STEP2;
5198 count1 = (N12 - N11) /[cl] STEP1;
5199 zero = zero3 || zero2 || zero1;
5200 count = count1 * count2 * count3;
5201 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5203 After all, we expect the zero=false, and thus we expect to have to
5204 evaluate all of the comparison expressions, so short-circuiting
5205 oughtn't be a win. Since the condition isn't protecting a
5206 denominator, we're not concerned about divide-by-zero, so we can
5207 fully evaluate count even if a numerator turned out to be wrong.
5209 It seems like putting this all together would create much better
5210 scheduling opportunities, and less pressure on the chip's branch
5214 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5215 basic_block
&entry_bb
, tree
*counts
,
5216 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5217 basic_block
&l2_dom_bb
)
5219 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5223 /* Collapsed loops need work for expansion into SSA form. */
5224 gcc_assert (!gimple_in_ssa_p (cfun
));
5226 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5227 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5229 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5230 isn't supposed to be handled, as the inner loop doesn't
5232 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5233 OMP_CLAUSE__LOOPTEMP_
);
5234 gcc_assert (innerc
);
5235 for (i
= 0; i
< fd
->collapse
; i
++)
5237 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5238 OMP_CLAUSE__LOOPTEMP_
);
5239 gcc_assert (innerc
);
5241 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5243 counts
[0] = NULL_TREE
;
5248 for (i
= 0; i
< fd
->collapse
; i
++)
5250 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5252 if (SSA_VAR_P (fd
->loop
.n2
)
5253 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5254 fold_convert (itype
, fd
->loops
[i
].n1
),
5255 fold_convert (itype
, fd
->loops
[i
].n2
)))
5256 == NULL_TREE
|| !integer_onep (t
)))
5260 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5261 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5262 true, GSI_SAME_STMT
);
5263 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5264 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5265 true, GSI_SAME_STMT
);
5266 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5267 NULL_TREE
, NULL_TREE
);
5268 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5269 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5270 expand_omp_regimplify_p
, NULL
, NULL
)
5271 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5272 expand_omp_regimplify_p
, NULL
, NULL
))
5274 *gsi
= gsi_for_stmt (cond_stmt
);
5275 gimple_regimplify_operands (cond_stmt
, gsi
);
5277 e
= split_block (entry_bb
, cond_stmt
);
5278 if (zero_iter_bb
== NULL
)
5280 gassign
*assign_stmt
;
5281 first_zero_iter
= i
;
5282 zero_iter_bb
= create_empty_bb (entry_bb
);
5283 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5284 *gsi
= gsi_after_labels (zero_iter_bb
);
5285 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5286 build_zero_cst (type
));
5287 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5288 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5291 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5292 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5293 e
->flags
= EDGE_TRUE_VALUE
;
5294 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5295 if (l2_dom_bb
== NULL
)
5296 l2_dom_bb
= entry_bb
;
5298 *gsi
= gsi_last_bb (entry_bb
);
5301 if (POINTER_TYPE_P (itype
))
5302 itype
= signed_type_for (itype
);
5303 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5305 t
= fold_build2 (PLUS_EXPR
, itype
,
5306 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5307 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5308 fold_convert (itype
, fd
->loops
[i
].n2
));
5309 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5310 fold_convert (itype
, fd
->loops
[i
].n1
));
5311 /* ?? We could probably use CEIL_DIV_EXPR instead of
5312 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5313 generate the same code in the end because generically we
5314 don't know that the values involved must be negative for
5316 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5317 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5318 fold_build1 (NEGATE_EXPR
, itype
, t
),
5319 fold_build1 (NEGATE_EXPR
, itype
,
5320 fold_convert (itype
,
5321 fd
->loops
[i
].step
)));
5323 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5324 fold_convert (itype
, fd
->loops
[i
].step
));
5325 t
= fold_convert (type
, t
);
5326 if (TREE_CODE (t
) == INTEGER_CST
)
5330 counts
[i
] = create_tmp_reg (type
, ".count");
5331 expand_omp_build_assign (gsi
, counts
[i
], t
);
5333 if (SSA_VAR_P (fd
->loop
.n2
))
5338 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5339 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5345 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5347 V3 = N31 + (T % count3) * STEP3;
5349 V2 = N21 + (T % count2) * STEP2;
5351 V1 = N11 + T * STEP1;
5352 if this loop doesn't have an inner loop construct combined with it.
5353 If it does have an inner loop construct combined with it and the
5354 iteration count isn't known constant, store values from counts array
5355 into its _looptemp_ temporaries instead. */
5358 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5359 tree
*counts
, gimple inner_stmt
, tree startvar
)
5362 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5364 /* If fd->loop.n2 is constant, then no propagation of the counts
5365 is needed, they are constant. */
5366 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5369 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5370 ? gimple_omp_parallel_clauses (inner_stmt
)
5371 : gimple_omp_for_clauses (inner_stmt
);
5372 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5373 isn't supposed to be handled, as the inner loop doesn't
5375 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5376 gcc_assert (innerc
);
5377 for (i
= 0; i
< fd
->collapse
; i
++)
5379 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5380 OMP_CLAUSE__LOOPTEMP_
);
5381 gcc_assert (innerc
);
5384 tree tem
= OMP_CLAUSE_DECL (innerc
);
5385 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5386 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5387 false, GSI_CONTINUE_LINKING
);
5388 gassign
*stmt
= gimple_build_assign (tem
, t
);
5389 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5395 tree type
= TREE_TYPE (fd
->loop
.v
);
5396 tree tem
= create_tmp_reg (type
, ".tem");
5397 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5398 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5400 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5402 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5404 if (POINTER_TYPE_P (vtype
))
5405 itype
= signed_type_for (vtype
);
5407 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5410 t
= fold_convert (itype
, t
);
5411 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5412 fold_convert (itype
, fd
->loops
[i
].step
));
5413 if (POINTER_TYPE_P (vtype
))
5414 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5416 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5417 t
= force_gimple_operand_gsi (gsi
, t
,
5418 DECL_P (fd
->loops
[i
].v
)
5419 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5421 GSI_CONTINUE_LINKING
);
5422 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5423 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5426 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5427 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5428 false, GSI_CONTINUE_LINKING
);
5429 stmt
= gimple_build_assign (tem
, t
);
5430 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5436 /* Helper function for expand_omp_for_*. Generate code like:
5439 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5443 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5450 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5451 basic_block body_bb
)
5453 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5455 gimple_stmt_iterator gsi
;
5461 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5463 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5465 bb
= create_empty_bb (last_bb
);
5466 add_bb_to_loop (bb
, last_bb
->loop_father
);
5467 gsi
= gsi_start_bb (bb
);
5469 if (i
< fd
->collapse
- 1)
5471 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5472 e
->probability
= REG_BR_PROB_BASE
/ 8;
5474 t
= fd
->loops
[i
+ 1].n1
;
5475 t
= force_gimple_operand_gsi (&gsi
, t
,
5476 DECL_P (fd
->loops
[i
+ 1].v
)
5477 && TREE_ADDRESSABLE (fd
->loops
[i
5480 GSI_CONTINUE_LINKING
);
5481 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5482 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5487 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5489 if (POINTER_TYPE_P (vtype
))
5490 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5492 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5493 t
= force_gimple_operand_gsi (&gsi
, t
,
5494 DECL_P (fd
->loops
[i
].v
)
5495 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5496 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5497 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5498 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5502 t
= fd
->loops
[i
].n2
;
5503 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5504 false, GSI_CONTINUE_LINKING
);
5505 tree v
= fd
->loops
[i
].v
;
5506 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5507 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5508 false, GSI_CONTINUE_LINKING
);
5509 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5510 stmt
= gimple_build_cond_empty (t
);
5511 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5512 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5513 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5516 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5524 /* A subroutine of expand_omp_for. Generate code for a parallel
5525 loop with any schedule. Given parameters:
5527 for (V = N1; V cond N2; V += STEP) BODY;
5529 where COND is "<" or ">", we generate pseudocode
5531 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5532 if (more) goto L0; else goto L3;
5539 if (V cond iend) goto L1; else goto L2;
5541 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5544 If this is a combined omp parallel loop, instead of the call to
5545 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5546 If this is gimple_omp_for_combined_p loop, then instead of assigning
5547 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5548 inner GIMPLE_OMP_FOR and V += STEP; and
5549 if (V cond iend) goto L1; else goto L2; are removed.
5551 For collapsed loops, given parameters:
5553 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5554 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5555 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5558 we generate pseudocode
5560 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5565 count3 = (adj + N32 - N31) / STEP3;
5566 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5571 count2 = (adj + N22 - N21) / STEP2;
5572 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5577 count1 = (adj + N12 - N11) / STEP1;
5578 count = count1 * count2 * count3;
5583 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5584 if (more) goto L0; else goto L3;
5588 V3 = N31 + (T % count3) * STEP3;
5590 V2 = N21 + (T % count2) * STEP2;
5592 V1 = N11 + T * STEP1;
5597 if (V < iend) goto L10; else goto L2;
5600 if (V3 cond3 N32) goto L1; else goto L11;
5604 if (V2 cond2 N22) goto L1; else goto L12;
5610 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5616 expand_omp_for_generic (struct omp_region
*region
,
5617 struct omp_for_data
*fd
,
5618 enum built_in_function start_fn
,
5619 enum built_in_function next_fn
,
5622 tree type
, istart0
, iend0
, iend
;
5623 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5624 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5625 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5626 gimple_stmt_iterator gsi
;
5627 gassign
*assign_stmt
;
5628 bool in_combined_parallel
= is_combined_parallel (region
);
5629 bool broken_loop
= region
->cont
== NULL
;
5631 tree
*counts
= NULL
;
5634 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5635 gcc_assert (fd
->iter_type
== long_integer_type_node
5636 || !in_combined_parallel
);
5638 type
= TREE_TYPE (fd
->loop
.v
);
5639 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5640 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5641 TREE_ADDRESSABLE (istart0
) = 1;
5642 TREE_ADDRESSABLE (iend0
) = 1;
5644 /* See if we need to bias by LLONG_MIN. */
5645 if (fd
->iter_type
== long_long_unsigned_type_node
5646 && TREE_CODE (type
) == INTEGER_TYPE
5647 && !TYPE_UNSIGNED (type
))
5651 if (fd
->loop
.cond_code
== LT_EXPR
)
5654 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5658 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5661 if (TREE_CODE (n1
) != INTEGER_CST
5662 || TREE_CODE (n2
) != INTEGER_CST
5663 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5664 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5667 entry_bb
= region
->entry
;
5668 cont_bb
= region
->cont
;
5670 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5671 gcc_assert (broken_loop
5672 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5673 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5674 l1_bb
= single_succ (l0_bb
);
5677 l2_bb
= create_empty_bb (cont_bb
);
5678 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5679 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5683 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5684 exit_bb
= region
->exit
;
5686 gsi
= gsi_last_bb (entry_bb
);
5688 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5689 if (fd
->collapse
> 1)
5691 int first_zero_iter
= -1;
5692 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5694 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5695 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5696 zero_iter_bb
, first_zero_iter
,
5701 /* Some counts[i] vars might be uninitialized if
5702 some loop has zero iterations. But the body shouldn't
5703 be executed in that case, so just avoid uninit warnings. */
5704 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5705 if (SSA_VAR_P (counts
[i
]))
5706 TREE_NO_WARNING (counts
[i
]) = 1;
5708 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5710 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5711 gsi
= gsi_last_bb (entry_bb
);
5712 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5713 get_immediate_dominator (CDI_DOMINATORS
,
5717 if (in_combined_parallel
)
5719 /* In a combined parallel loop, emit a call to
5720 GOMP_loop_foo_next. */
5721 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5722 build_fold_addr_expr (istart0
),
5723 build_fold_addr_expr (iend0
));
5727 tree t0
, t1
, t2
, t3
, t4
;
5728 /* If this is not a combined parallel loop, emit a call to
5729 GOMP_loop_foo_start in ENTRY_BB. */
5730 t4
= build_fold_addr_expr (iend0
);
5731 t3
= build_fold_addr_expr (istart0
);
5732 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5735 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5737 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5738 OMP_CLAUSE__LOOPTEMP_
);
5739 gcc_assert (innerc
);
5740 t0
= OMP_CLAUSE_DECL (innerc
);
5741 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5742 OMP_CLAUSE__LOOPTEMP_
);
5743 gcc_assert (innerc
);
5744 t1
= OMP_CLAUSE_DECL (innerc
);
5746 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5747 && TYPE_PRECISION (TREE_TYPE (t0
))
5748 != TYPE_PRECISION (fd
->iter_type
))
5750 /* Avoid casting pointers to integer of a different size. */
5751 tree itype
= signed_type_for (type
);
5752 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5753 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5757 t1
= fold_convert (fd
->iter_type
, t1
);
5758 t0
= fold_convert (fd
->iter_type
, t0
);
5762 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5763 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5765 if (fd
->iter_type
== long_integer_type_node
)
5769 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5770 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5771 6, t0
, t1
, t2
, t
, t3
, t4
);
5774 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5775 5, t0
, t1
, t2
, t3
, t4
);
5783 /* The GOMP_loop_ull_*start functions have additional boolean
5784 argument, true for < loops and false for > loops.
5785 In Fortran, the C bool type can be different from
5786 boolean_type_node. */
5787 bfn_decl
= builtin_decl_explicit (start_fn
);
5788 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5789 t5
= build_int_cst (c_bool_type
,
5790 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5793 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5794 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5795 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5798 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5799 6, t5
, t0
, t1
, t2
, t3
, t4
);
5802 if (TREE_TYPE (t
) != boolean_type_node
)
5803 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5804 t
, build_int_cst (TREE_TYPE (t
), 0));
5805 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5806 true, GSI_SAME_STMT
);
5807 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5809 /* Remove the GIMPLE_OMP_FOR statement. */
5810 gsi_remove (&gsi
, true);
5812 /* Iteration setup for sequential loop goes in L0_BB. */
5813 tree startvar
= fd
->loop
.v
;
5814 tree endvar
= NULL_TREE
;
5816 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5818 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5819 && gimple_omp_for_kind (inner_stmt
)
5820 == GF_OMP_FOR_KIND_SIMD
);
5821 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5822 OMP_CLAUSE__LOOPTEMP_
);
5823 gcc_assert (innerc
);
5824 startvar
= OMP_CLAUSE_DECL (innerc
);
5825 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5826 OMP_CLAUSE__LOOPTEMP_
);
5827 gcc_assert (innerc
);
5828 endvar
= OMP_CLAUSE_DECL (innerc
);
5831 gsi
= gsi_start_bb (l0_bb
);
5834 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5835 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5836 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5837 t
= fold_convert (TREE_TYPE (startvar
), t
);
5838 t
= force_gimple_operand_gsi (&gsi
, t
,
5840 && TREE_ADDRESSABLE (startvar
),
5841 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5842 assign_stmt
= gimple_build_assign (startvar
, t
);
5843 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5847 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5848 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5849 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5850 t
= fold_convert (TREE_TYPE (startvar
), t
);
5851 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5852 false, GSI_CONTINUE_LINKING
);
5855 assign_stmt
= gimple_build_assign (endvar
, iend
);
5856 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5857 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5858 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5860 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
5861 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5863 if (fd
->collapse
> 1)
5864 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5868 /* Code to control the increment and predicate for the sequential
5869 loop goes in the CONT_BB. */
5870 gsi
= gsi_last_bb (cont_bb
);
5871 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
5872 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
5873 vmain
= gimple_omp_continue_control_use (cont_stmt
);
5874 vback
= gimple_omp_continue_control_def (cont_stmt
);
5876 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5878 if (POINTER_TYPE_P (type
))
5879 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5881 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5882 t
= force_gimple_operand_gsi (&gsi
, t
,
5884 && TREE_ADDRESSABLE (vback
),
5885 NULL_TREE
, true, GSI_SAME_STMT
);
5886 assign_stmt
= gimple_build_assign (vback
, t
);
5887 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
5889 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5890 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5892 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
5893 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
5896 /* Remove GIMPLE_OMP_CONTINUE. */
5897 gsi_remove (&gsi
, true);
5899 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5900 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5902 /* Emit code to get the next parallel iteration in L2_BB. */
5903 gsi
= gsi_start_bb (l2_bb
);
5905 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5906 build_fold_addr_expr (istart0
),
5907 build_fold_addr_expr (iend0
));
5908 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5909 false, GSI_CONTINUE_LINKING
);
5910 if (TREE_TYPE (t
) != boolean_type_node
)
5911 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5912 t
, build_int_cst (TREE_TYPE (t
), 0));
5913 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
5914 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
5917 /* Add the loop cleanup function. */
5918 gsi
= gsi_last_bb (exit_bb
);
5919 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5920 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5921 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5922 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5924 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5925 gcall
*call_stmt
= gimple_build_call (t
, 0);
5926 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5927 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5928 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
5929 gsi_remove (&gsi
, true);
5931 /* Connect the new blocks. */
5932 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5933 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5939 e
= find_edge (cont_bb
, l3_bb
);
5940 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5942 phis
= phi_nodes (l3_bb
);
5943 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5945 gimple phi
= gsi_stmt (gsi
);
5946 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5947 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5951 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5952 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5953 e
= find_edge (cont_bb
, l1_bb
);
5954 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5959 else if (fd
->collapse
> 1)
5962 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5965 e
->flags
= EDGE_TRUE_VALUE
;
5968 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5969 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5973 e
= find_edge (cont_bb
, l2_bb
);
5974 e
->flags
= EDGE_FALLTHRU
;
5976 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5978 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5979 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5980 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5981 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5982 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5983 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5984 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5985 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5987 struct loop
*outer_loop
= alloc_loop ();
5988 outer_loop
->header
= l0_bb
;
5989 outer_loop
->latch
= l2_bb
;
5990 add_loop (outer_loop
, l0_bb
->loop_father
);
5992 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5994 struct loop
*loop
= alloc_loop ();
5995 loop
->header
= l1_bb
;
5996 /* The loop may have multiple latches. */
5997 add_loop (loop
, outer_loop
);
6003 /* A subroutine of expand_omp_for. Generate code for a parallel
6004 loop with static schedule and no specified chunk size. Given
6007 for (V = N1; V cond N2; V += STEP) BODY;
6009 where COND is "<" or ">", we generate pseudocode
6011 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6016 if ((__typeof (V)) -1 > 0 && cond is >)
6017 n = -(adj + N2 - N1) / -STEP;
6019 n = (adj + N2 - N1) / STEP;
6022 if (threadid < tt) goto L3; else goto L4;
6027 s0 = q * threadid + tt;
6030 if (s0 >= e0) goto L2; else goto L0;
6036 if (V cond e) goto L1;
6041 expand_omp_for_static_nochunk (struct omp_region
*region
,
6042 struct omp_for_data
*fd
,
6045 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6046 tree type
, itype
, vmain
, vback
;
6047 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6048 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6050 gimple_stmt_iterator gsi
;
6052 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6053 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6054 bool broken_loop
= region
->cont
== NULL
;
6055 tree
*counts
= NULL
;
6058 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6059 if (POINTER_TYPE_P (type
))
6060 itype
= signed_type_for (type
);
6062 entry_bb
= region
->entry
;
6063 cont_bb
= region
->cont
;
6064 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6065 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6066 gcc_assert (broken_loop
6067 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6068 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6069 body_bb
= single_succ (seq_start_bb
);
6072 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6073 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6075 exit_bb
= region
->exit
;
6077 /* Iteration space partitioning goes in ENTRY_BB. */
6078 gsi
= gsi_last_bb (entry_bb
);
6079 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6081 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6083 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6084 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6087 if (fd
->collapse
> 1)
6089 int first_zero_iter
= -1;
6090 basic_block l2_dom_bb
= NULL
;
6092 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6093 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6094 fin_bb
, first_zero_iter
,
6098 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6099 t
= integer_one_node
;
6101 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6102 fold_convert (type
, fd
->loop
.n1
),
6103 fold_convert (type
, fd
->loop
.n2
));
6104 if (fd
->collapse
== 1
6105 && TYPE_UNSIGNED (type
)
6106 && (t
== NULL_TREE
|| !integer_onep (t
)))
6108 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6109 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6110 true, GSI_SAME_STMT
);
6111 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6112 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6113 true, GSI_SAME_STMT
);
6114 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6115 NULL_TREE
, NULL_TREE
);
6116 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6117 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6118 expand_omp_regimplify_p
, NULL
, NULL
)
6119 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6120 expand_omp_regimplify_p
, NULL
, NULL
))
6122 gsi
= gsi_for_stmt (cond_stmt
);
6123 gimple_regimplify_operands (cond_stmt
, &gsi
);
6125 ep
= split_block (entry_bb
, cond_stmt
);
6126 ep
->flags
= EDGE_TRUE_VALUE
;
6127 entry_bb
= ep
->dest
;
6128 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6129 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6130 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6131 if (gimple_in_ssa_p (cfun
))
6133 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6134 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6135 !gsi_end_p (gpi
); gsi_next (&gpi
))
6137 gphi
*phi
= gpi
.phi ();
6138 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6139 ep
, UNKNOWN_LOCATION
);
6142 gsi
= gsi_last_bb (entry_bb
);
6145 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6146 t
= fold_convert (itype
, t
);
6147 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6148 true, GSI_SAME_STMT
);
6150 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6151 t
= fold_convert (itype
, t
);
6152 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6153 true, GSI_SAME_STMT
);
6157 step
= fd
->loop
.step
;
6158 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6160 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6161 OMP_CLAUSE__LOOPTEMP_
);
6162 gcc_assert (innerc
);
6163 n1
= OMP_CLAUSE_DECL (innerc
);
6164 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6165 OMP_CLAUSE__LOOPTEMP_
);
6166 gcc_assert (innerc
);
6167 n2
= OMP_CLAUSE_DECL (innerc
);
6169 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6170 true, NULL_TREE
, true, GSI_SAME_STMT
);
6171 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6172 true, NULL_TREE
, true, GSI_SAME_STMT
);
6173 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6174 true, NULL_TREE
, true, GSI_SAME_STMT
);
6176 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6177 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6178 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6179 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6180 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6181 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6182 fold_build1 (NEGATE_EXPR
, itype
, t
),
6183 fold_build1 (NEGATE_EXPR
, itype
, step
));
6185 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6186 t
= fold_convert (itype
, t
);
6187 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6189 q
= create_tmp_reg (itype
, "q");
6190 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6191 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6192 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6194 tt
= create_tmp_reg (itype
, "tt");
6195 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6196 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6197 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6199 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6200 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6201 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6203 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6204 gsi
= gsi_last_bb (second_bb
);
6205 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6207 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6209 gassign
*assign_stmt
6210 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6211 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6213 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6214 gsi
= gsi_last_bb (third_bb
);
6215 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6217 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6218 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6219 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6221 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6222 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6224 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6225 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6227 /* Remove the GIMPLE_OMP_FOR statement. */
6228 gsi_remove (&gsi
, true);
6230 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6231 gsi
= gsi_start_bb (seq_start_bb
);
6233 tree startvar
= fd
->loop
.v
;
6234 tree endvar
= NULL_TREE
;
6236 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6238 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6239 ? gimple_omp_parallel_clauses (inner_stmt
)
6240 : gimple_omp_for_clauses (inner_stmt
);
6241 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6242 gcc_assert (innerc
);
6243 startvar
= OMP_CLAUSE_DECL (innerc
);
6244 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6245 OMP_CLAUSE__LOOPTEMP_
);
6246 gcc_assert (innerc
);
6247 endvar
= OMP_CLAUSE_DECL (innerc
);
6249 t
= fold_convert (itype
, s0
);
6250 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6251 if (POINTER_TYPE_P (type
))
6252 t
= fold_build_pointer_plus (n1
, t
);
6254 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6255 t
= fold_convert (TREE_TYPE (startvar
), t
);
6256 t
= force_gimple_operand_gsi (&gsi
, t
,
6258 && TREE_ADDRESSABLE (startvar
),
6259 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6260 assign_stmt
= gimple_build_assign (startvar
, t
);
6261 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6263 t
= fold_convert (itype
, e0
);
6264 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6265 if (POINTER_TYPE_P (type
))
6266 t
= fold_build_pointer_plus (n1
, t
);
6268 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6269 t
= fold_convert (TREE_TYPE (startvar
), t
);
6270 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6271 false, GSI_CONTINUE_LINKING
);
6274 assign_stmt
= gimple_build_assign (endvar
, e
);
6275 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6276 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6277 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6279 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6280 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6282 if (fd
->collapse
> 1)
6283 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6287 /* The code controlling the sequential loop replaces the
6288 GIMPLE_OMP_CONTINUE. */
6289 gsi
= gsi_last_bb (cont_bb
);
6290 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6291 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6292 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6293 vback
= gimple_omp_continue_control_def (cont_stmt
);
6295 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6297 if (POINTER_TYPE_P (type
))
6298 t
= fold_build_pointer_plus (vmain
, step
);
6300 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6301 t
= force_gimple_operand_gsi (&gsi
, t
,
6303 && TREE_ADDRESSABLE (vback
),
6304 NULL_TREE
, true, GSI_SAME_STMT
);
6305 assign_stmt
= gimple_build_assign (vback
, t
);
6306 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6308 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6309 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6311 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6314 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6315 gsi_remove (&gsi
, true);
6317 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6318 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6321 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6322 gsi
= gsi_last_bb (exit_bb
);
6323 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6325 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6326 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6328 gsi_remove (&gsi
, true);
6330 /* Connect all the blocks. */
6331 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6332 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6333 ep
= find_edge (entry_bb
, second_bb
);
6334 ep
->flags
= EDGE_TRUE_VALUE
;
6335 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6336 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6337 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6341 ep
= find_edge (cont_bb
, body_bb
);
6342 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6347 else if (fd
->collapse
> 1)
6350 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6353 ep
->flags
= EDGE_TRUE_VALUE
;
6354 find_edge (cont_bb
, fin_bb
)->flags
6355 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6358 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6359 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6360 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6362 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6363 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6364 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6365 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6367 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6369 struct loop
*loop
= alloc_loop ();
6370 loop
->header
= body_bb
;
6371 if (collapse_bb
== NULL
)
6372 loop
->latch
= cont_bb
;
6373 add_loop (loop
, body_bb
->loop_father
);
6378 /* A subroutine of expand_omp_for. Generate code for a parallel
6379 loop with static schedule and a specified chunk size. Given
6382 for (V = N1; V cond N2; V += STEP) BODY;
6384 where COND is "<" or ">", we generate pseudocode
6386 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6391 if ((__typeof (V)) -1 > 0 && cond is >)
6392 n = -(adj + N2 - N1) / -STEP;
6394 n = (adj + N2 - N1) / STEP;
6396 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6397 here so that V is defined
6398 if the loop is not entered
6400 s0 = (trip * nthreads + threadid) * CHUNK;
6401 e0 = min(s0 + CHUNK, n);
6402 if (s0 < n) goto L1; else goto L4;
6409 if (V cond e) goto L2; else goto L3;
6417 expand_omp_for_static_chunk (struct omp_region
*region
,
6418 struct omp_for_data
*fd
, gimple inner_stmt
)
6420 tree n
, s0
, e0
, e
, t
;
6421 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6422 tree type
, itype
, vmain
, vback
, vextra
;
6423 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6424 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6425 gimple_stmt_iterator gsi
;
6427 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6428 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6429 bool broken_loop
= region
->cont
== NULL
;
6430 tree
*counts
= NULL
;
6433 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6434 if (POINTER_TYPE_P (type
))
6435 itype
= signed_type_for (type
);
6437 entry_bb
= region
->entry
;
6438 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6440 iter_part_bb
= se
->dest
;
6441 cont_bb
= region
->cont
;
6442 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6443 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6444 gcc_assert (broken_loop
6445 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6446 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6447 body_bb
= single_succ (seq_start_bb
);
6450 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6451 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6452 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6454 exit_bb
= region
->exit
;
6456 /* Trip and adjustment setup goes in ENTRY_BB. */
6457 gsi
= gsi_last_bb (entry_bb
);
6458 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6460 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6462 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6463 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6466 if (fd
->collapse
> 1)
6468 int first_zero_iter
= -1;
6469 basic_block l2_dom_bb
= NULL
;
6471 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6472 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6473 fin_bb
, first_zero_iter
,
6477 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6478 t
= integer_one_node
;
6480 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6481 fold_convert (type
, fd
->loop
.n1
),
6482 fold_convert (type
, fd
->loop
.n2
));
6483 if (fd
->collapse
== 1
6484 && TYPE_UNSIGNED (type
)
6485 && (t
== NULL_TREE
|| !integer_onep (t
)))
6487 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6488 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6489 true, GSI_SAME_STMT
);
6490 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6491 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6492 true, GSI_SAME_STMT
);
6493 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6494 NULL_TREE
, NULL_TREE
);
6495 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6496 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6497 expand_omp_regimplify_p
, NULL
, NULL
)
6498 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6499 expand_omp_regimplify_p
, NULL
, NULL
))
6501 gsi
= gsi_for_stmt (cond_stmt
);
6502 gimple_regimplify_operands (cond_stmt
, &gsi
);
6504 se
= split_block (entry_bb
, cond_stmt
);
6505 se
->flags
= EDGE_TRUE_VALUE
;
6506 entry_bb
= se
->dest
;
6507 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6508 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6509 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6510 if (gimple_in_ssa_p (cfun
))
6512 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6513 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6514 !gsi_end_p (gpi
); gsi_next (&gpi
))
6516 gphi
*phi
= gpi
.phi ();
6517 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6518 se
, UNKNOWN_LOCATION
);
6521 gsi
= gsi_last_bb (entry_bb
);
6524 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6525 t
= fold_convert (itype
, t
);
6526 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6527 true, GSI_SAME_STMT
);
6529 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6530 t
= fold_convert (itype
, t
);
6531 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6532 true, GSI_SAME_STMT
);
6536 step
= fd
->loop
.step
;
6537 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6539 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6540 OMP_CLAUSE__LOOPTEMP_
);
6541 gcc_assert (innerc
);
6542 n1
= OMP_CLAUSE_DECL (innerc
);
6543 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6544 OMP_CLAUSE__LOOPTEMP_
);
6545 gcc_assert (innerc
);
6546 n2
= OMP_CLAUSE_DECL (innerc
);
6548 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6549 true, NULL_TREE
, true, GSI_SAME_STMT
);
6550 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6551 true, NULL_TREE
, true, GSI_SAME_STMT
);
6552 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6553 true, NULL_TREE
, true, GSI_SAME_STMT
);
6555 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6556 true, NULL_TREE
, true, GSI_SAME_STMT
);
6558 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6559 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6560 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6561 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6562 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6563 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6564 fold_build1 (NEGATE_EXPR
, itype
, t
),
6565 fold_build1 (NEGATE_EXPR
, itype
, step
));
6567 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6568 t
= fold_convert (itype
, t
);
6569 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6570 true, GSI_SAME_STMT
);
6572 trip_var
= create_tmp_reg (itype
, ".trip");
6573 if (gimple_in_ssa_p (cfun
))
6575 trip_init
= make_ssa_name (trip_var
);
6576 trip_main
= make_ssa_name (trip_var
);
6577 trip_back
= make_ssa_name (trip_var
);
6581 trip_init
= trip_var
;
6582 trip_main
= trip_var
;
6583 trip_back
= trip_var
;
6586 gassign
*assign_stmt
6587 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6588 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6590 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6591 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6592 if (POINTER_TYPE_P (type
))
6593 t
= fold_build_pointer_plus (n1
, t
);
6595 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6596 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6597 true, GSI_SAME_STMT
);
6599 /* Remove the GIMPLE_OMP_FOR. */
6600 gsi_remove (&gsi
, true);
6602 /* Iteration space partitioning goes in ITER_PART_BB. */
6603 gsi
= gsi_last_bb (iter_part_bb
);
6605 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6606 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6607 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6608 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6609 false, GSI_CONTINUE_LINKING
);
6611 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6612 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6613 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6614 false, GSI_CONTINUE_LINKING
);
6616 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6617 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6619 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6620 gsi
= gsi_start_bb (seq_start_bb
);
6622 tree startvar
= fd
->loop
.v
;
6623 tree endvar
= NULL_TREE
;
6625 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6627 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6628 ? gimple_omp_parallel_clauses (inner_stmt
)
6629 : gimple_omp_for_clauses (inner_stmt
);
6630 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6631 gcc_assert (innerc
);
6632 startvar
= OMP_CLAUSE_DECL (innerc
);
6633 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6634 OMP_CLAUSE__LOOPTEMP_
);
6635 gcc_assert (innerc
);
6636 endvar
= OMP_CLAUSE_DECL (innerc
);
6639 t
= fold_convert (itype
, s0
);
6640 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6641 if (POINTER_TYPE_P (type
))
6642 t
= fold_build_pointer_plus (n1
, t
);
6644 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6645 t
= fold_convert (TREE_TYPE (startvar
), t
);
6646 t
= force_gimple_operand_gsi (&gsi
, t
,
6648 && TREE_ADDRESSABLE (startvar
),
6649 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6650 assign_stmt
= gimple_build_assign (startvar
, t
);
6651 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6653 t
= fold_convert (itype
, e0
);
6654 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6655 if (POINTER_TYPE_P (type
))
6656 t
= fold_build_pointer_plus (n1
, t
);
6658 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6659 t
= fold_convert (TREE_TYPE (startvar
), t
);
6660 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6661 false, GSI_CONTINUE_LINKING
);
6664 assign_stmt
= gimple_build_assign (endvar
, e
);
6665 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6666 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6667 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6669 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6670 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6672 if (fd
->collapse
> 1)
6673 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6677 /* The code controlling the sequential loop goes in CONT_BB,
6678 replacing the GIMPLE_OMP_CONTINUE. */
6679 gsi
= gsi_last_bb (cont_bb
);
6680 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6681 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6682 vback
= gimple_omp_continue_control_def (cont_stmt
);
6684 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6686 if (POINTER_TYPE_P (type
))
6687 t
= fold_build_pointer_plus (vmain
, step
);
6689 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6690 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6691 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6692 true, GSI_SAME_STMT
);
6693 assign_stmt
= gimple_build_assign (vback
, t
);
6694 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6696 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6697 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6699 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6702 /* Remove GIMPLE_OMP_CONTINUE. */
6703 gsi_remove (&gsi
, true);
6705 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6706 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6708 /* Trip update code goes into TRIP_UPDATE_BB. */
6709 gsi
= gsi_start_bb (trip_update_bb
);
6711 t
= build_int_cst (itype
, 1);
6712 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6713 assign_stmt
= gimple_build_assign (trip_back
, t
);
6714 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6717 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6718 gsi
= gsi_last_bb (exit_bb
);
6719 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6721 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6722 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6724 gsi_remove (&gsi
, true);
6726 /* Connect the new blocks. */
6727 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6728 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6732 se
= find_edge (cont_bb
, body_bb
);
6733 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6738 else if (fd
->collapse
> 1)
6741 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6744 se
->flags
= EDGE_TRUE_VALUE
;
6745 find_edge (cont_bb
, trip_update_bb
)->flags
6746 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6748 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6751 if (gimple_in_ssa_p (cfun
))
6759 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6761 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6762 remove arguments of the phi nodes in fin_bb. We need to create
6763 appropriate phi nodes in iter_part_bb instead. */
6764 se
= single_pred_edge (fin_bb
);
6765 re
= single_succ_edge (trip_update_bb
);
6766 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
6767 ene
= single_succ_edge (entry_bb
);
6769 psi
= gsi_start_phis (fin_bb
);
6770 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6771 gsi_next (&psi
), ++i
)
6774 source_location locus
;
6777 t
= gimple_phi_result (phi
);
6778 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6779 nphi
= create_phi_node (t
, iter_part_bb
);
6781 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6782 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6784 /* A special case -- fd->loop.v is not yet computed in
6785 iter_part_bb, we need to use vextra instead. */
6786 if (t
== fd
->loop
.v
)
6788 add_phi_arg (nphi
, t
, ene
, locus
);
6789 locus
= redirect_edge_var_map_location (vm
);
6790 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6792 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6793 redirect_edge_var_map_clear (re
);
6796 psi
= gsi_start_phis (fin_bb
);
6797 if (gsi_end_p (psi
))
6799 remove_phi_node (&psi
, false);
6802 /* Make phi node for trip. */
6803 phi
= create_phi_node (trip_main
, iter_part_bb
);
6804 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6806 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6811 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6812 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6813 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6814 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6815 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6816 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6817 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6818 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6819 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6823 struct loop
*trip_loop
= alloc_loop ();
6824 trip_loop
->header
= iter_part_bb
;
6825 trip_loop
->latch
= trip_update_bb
;
6826 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6828 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6830 struct loop
*loop
= alloc_loop ();
6831 loop
->header
= body_bb
;
6832 if (collapse_bb
== NULL
)
6833 loop
->latch
= cont_bb
;
6834 add_loop (loop
, trip_loop
);
6839 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
6841 for (V = N1; V cond N2; V += STEP) BODY;
6843 where COND is "<" or ">" or "!=", we generate pseudocode
6845 for (ind_var = low; ind_var < high; ind_var++)
6847 V = n1 + (ind_var * STEP)
6852 In the above pseudocode, low and high are function parameters of the
6853 child function. In the function below, we are inserting a temp.
6854 variable that will be making a call to two OMP functions that will not be
6855 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
6856 with _Cilk_for). These functions are replaced with low and high
6857 by the function that handles taskreg. */
6861 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
6863 bool broken_loop
= region
->cont
== NULL
;
6864 basic_block entry_bb
= region
->entry
;
6865 basic_block cont_bb
= region
->cont
;
6867 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6868 gcc_assert (broken_loop
6869 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6870 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6871 basic_block l1_bb
, l2_bb
;
6875 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6876 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6877 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6878 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6882 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6883 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6884 l2_bb
= single_succ (l1_bb
);
6886 basic_block exit_bb
= region
->exit
;
6887 basic_block l2_dom_bb
= NULL
;
6889 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
6891 /* Below statements until the "tree high_val = ..." are pseudo statements
6892 used to pass information to be used by expand_omp_taskreg.
6893 low_val and high_val will be replaced by the __low and __high
6894 parameter from the child function.
6896 The call_exprs part is a place-holder, it is mainly used
6897 to distinctly identify to the top-level part that this is
6898 where we should put low and high (reasoning given in header
6902 = gimple_omp_parallel_child_fn (
6903 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
6904 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
6905 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
6907 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
6909 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
6912 gcc_assert (low_val
&& high_val
);
6914 tree type
= TREE_TYPE (low_val
);
6915 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
6916 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6918 /* Not needed in SSA form right now. */
6919 gcc_assert (!gimple_in_ssa_p (cfun
));
6920 if (l2_dom_bb
== NULL
)
6926 gimple stmt
= gimple_build_assign (ind_var
, n1
);
6928 /* Replace the GIMPLE_OMP_FOR statement. */
6929 gsi_replace (&gsi
, stmt
, true);
6933 /* Code to control the increment goes in the CONT_BB. */
6934 gsi
= gsi_last_bb (cont_bb
);
6935 stmt
= gsi_stmt (gsi
);
6936 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6937 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
6938 build_one_cst (type
));
6940 /* Replace GIMPLE_OMP_CONTINUE. */
6941 gsi_replace (&gsi
, stmt
, true);
6944 /* Emit the condition in L1_BB. */
6945 gsi
= gsi_after_labels (l1_bb
);
6946 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
6947 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
6949 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
6950 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6951 fd
->loop
.n1
, fold_convert (sizetype
, t
));
6953 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6954 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
6955 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
6956 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6958 /* The condition is always '<' since the runtime will fill in the low
6960 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
6961 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6963 /* Remove GIMPLE_OMP_RETURN. */
6964 gsi
= gsi_last_bb (exit_bb
);
6965 gsi_remove (&gsi
, true);
6967 /* Connect the new blocks. */
6968 remove_edge (FALLTHRU_EDGE (entry_bb
));
6973 remove_edge (BRANCH_EDGE (entry_bb
));
6974 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6976 e
= BRANCH_EDGE (l1_bb
);
6977 ne
= FALLTHRU_EDGE (l1_bb
);
6978 e
->flags
= EDGE_TRUE_VALUE
;
6982 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6984 ne
= single_succ_edge (l1_bb
);
6985 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6988 ne
->flags
= EDGE_FALSE_VALUE
;
6989 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6990 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6992 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6993 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6994 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6998 struct loop
*loop
= alloc_loop ();
6999 loop
->header
= l1_bb
;
7000 loop
->latch
= cont_bb
;
7001 add_loop (loop
, l1_bb
->loop_father
);
7002 loop
->safelen
= INT_MAX
;
7005 /* Pick the correct library function based on the precision of the
7006 induction variable type. */
7007 tree lib_fun
= NULL_TREE
;
7008 if (TYPE_PRECISION (type
) == 32)
7009 lib_fun
= cilk_for_32_fndecl
;
7010 else if (TYPE_PRECISION (type
) == 64)
7011 lib_fun
= cilk_for_64_fndecl
;
7015 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7017 /* WS_ARGS contains the library function flavor to call:
7018 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7019 user-defined grain value. If the user does not define one, then zero
7020 is passed in by the parser. */
7021 vec_alloc (region
->ws_args
, 2);
7022 region
->ws_args
->quick_push (lib_fun
);
7023 region
->ws_args
->quick_push (fd
->chunk_size
);
7026 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7027 loop. Given parameters:
7029 for (V = N1; V cond N2; V += STEP) BODY;
7031 where COND is "<" or ">", we generate pseudocode
7039 if (V cond N2) goto L0; else goto L2;
7042 For collapsed loops, given parameters:
7044 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7045 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7046 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7049 we generate pseudocode
7055 count3 = (adj + N32 - N31) / STEP3;
7060 count2 = (adj + N22 - N21) / STEP2;
7065 count1 = (adj + N12 - N11) / STEP1;
7066 count = count1 * count2 * count3;
7076 V2 += (V3 cond3 N32) ? 0 : STEP2;
7077 V3 = (V3 cond3 N32) ? V3 : N31;
7078 V1 += (V2 cond2 N22) ? 0 : STEP1;
7079 V2 = (V2 cond2 N22) ? V2 : N21;
7081 if (V < count) goto L0; else goto L2;
7087 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7090 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7091 gimple_stmt_iterator gsi
;
7094 bool broken_loop
= region
->cont
== NULL
;
7096 tree
*counts
= NULL
;
7098 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7099 OMP_CLAUSE_SAFELEN
);
7100 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7101 OMP_CLAUSE__SIMDUID_
);
7104 type
= TREE_TYPE (fd
->loop
.v
);
7105 entry_bb
= region
->entry
;
7106 cont_bb
= region
->cont
;
7107 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7108 gcc_assert (broken_loop
7109 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7110 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7113 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7114 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7115 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7116 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7120 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7121 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7122 l2_bb
= single_succ (l1_bb
);
7124 exit_bb
= region
->exit
;
7127 gsi
= gsi_last_bb (entry_bb
);
7129 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7130 /* Not needed in SSA form right now. */
7131 gcc_assert (!gimple_in_ssa_p (cfun
));
7132 if (fd
->collapse
> 1)
7134 int first_zero_iter
= -1;
7135 basic_block zero_iter_bb
= l2_bb
;
7137 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7138 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7139 zero_iter_bb
, first_zero_iter
,
7142 if (l2_dom_bb
== NULL
)
7147 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7149 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7150 OMP_CLAUSE__LOOPTEMP_
);
7151 gcc_assert (innerc
);
7152 n1
= OMP_CLAUSE_DECL (innerc
);
7153 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7154 OMP_CLAUSE__LOOPTEMP_
);
7155 gcc_assert (innerc
);
7156 n2
= OMP_CLAUSE_DECL (innerc
);
7157 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7158 fold_convert (type
, n1
));
7159 if (fd
->collapse
> 1)
7162 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7168 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7169 fold_convert (type
, fd
->loop
.n1
));
7170 if (fd
->collapse
> 1)
7171 for (i
= 0; i
< fd
->collapse
; i
++)
7173 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7174 if (POINTER_TYPE_P (itype
))
7175 itype
= signed_type_for (itype
);
7176 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7177 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7181 /* Remove the GIMPLE_OMP_FOR statement. */
7182 gsi_remove (&gsi
, true);
7186 /* Code to control the increment goes in the CONT_BB. */
7187 gsi
= gsi_last_bb (cont_bb
);
7188 stmt
= gsi_stmt (gsi
);
7189 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7191 if (POINTER_TYPE_P (type
))
7192 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7194 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7195 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7197 if (fd
->collapse
> 1)
7199 i
= fd
->collapse
- 1;
7200 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7202 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7203 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7207 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7209 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7212 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7214 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7216 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7217 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7218 if (POINTER_TYPE_P (itype2
))
7219 itype2
= signed_type_for (itype2
);
7220 t
= build3 (COND_EXPR
, itype2
,
7221 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7223 fold_convert (itype
, fd
->loops
[i
].n2
)),
7224 build_int_cst (itype2
, 0),
7225 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7226 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7227 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7229 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7230 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7232 t
= build3 (COND_EXPR
, itype
,
7233 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7235 fold_convert (itype
, fd
->loops
[i
].n2
)),
7237 fold_convert (itype
, fd
->loops
[i
].n1
));
7238 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7242 /* Remove GIMPLE_OMP_CONTINUE. */
7243 gsi_remove (&gsi
, true);
7246 /* Emit the condition in L1_BB. */
7247 gsi
= gsi_start_bb (l1_bb
);
7249 t
= fold_convert (type
, n2
);
7250 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7251 false, GSI_CONTINUE_LINKING
);
7252 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7253 cond_stmt
= gimple_build_cond_empty (t
);
7254 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7255 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7257 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7260 gsi
= gsi_for_stmt (cond_stmt
);
7261 gimple_regimplify_operands (cond_stmt
, &gsi
);
7264 /* Remove GIMPLE_OMP_RETURN. */
7265 gsi
= gsi_last_bb (exit_bb
);
7266 gsi_remove (&gsi
, true);
7268 /* Connect the new blocks. */
7269 remove_edge (FALLTHRU_EDGE (entry_bb
));
7273 remove_edge (BRANCH_EDGE (entry_bb
));
7274 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7276 e
= BRANCH_EDGE (l1_bb
);
7277 ne
= FALLTHRU_EDGE (l1_bb
);
7278 e
->flags
= EDGE_TRUE_VALUE
;
7282 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7284 ne
= single_succ_edge (l1_bb
);
7285 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7288 ne
->flags
= EDGE_FALSE_VALUE
;
7289 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7290 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7292 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7293 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7294 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7298 struct loop
*loop
= alloc_loop ();
7299 loop
->header
= l1_bb
;
7300 loop
->latch
= cont_bb
;
7301 add_loop (loop
, l1_bb
->loop_father
);
7302 if (safelen
== NULL_TREE
)
7303 loop
->safelen
= INT_MAX
;
7306 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7307 if (TREE_CODE (safelen
) != INTEGER_CST
)
7309 else if (!tree_fits_uhwi_p (safelen
)
7310 || tree_to_uhwi (safelen
) > INT_MAX
)
7311 loop
->safelen
= INT_MAX
;
7313 loop
->safelen
= tree_to_uhwi (safelen
);
7314 if (loop
->safelen
== 1)
7319 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7320 cfun
->has_simduid_loops
= true;
7322 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7324 if ((flag_tree_loop_vectorize
7325 || (!global_options_set
.x_flag_tree_loop_vectorize
7326 && !global_options_set
.x_flag_tree_vectorize
))
7327 && flag_tree_loop_optimize
7328 && loop
->safelen
> 1)
7330 loop
->force_vectorize
= true;
7331 cfun
->has_force_vectorize_loops
= true;
7337 /* Expand the OpenMP loop defined by REGION. */
7340 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7342 struct omp_for_data fd
;
7343 struct omp_for_data_loop
*loops
;
7346 = (struct omp_for_data_loop
*)
7347 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7348 * sizeof (struct omp_for_data_loop
));
7349 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7351 region
->sched_kind
= fd
.sched_kind
;
7353 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7354 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7355 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7358 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7359 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7360 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7363 /* If there isn't a continue then this is a degerate case where
7364 the introduction of abnormal edges during lowering will prevent
7365 original loops from being detected. Fix that up. */
7366 loops_state_set (LOOPS_NEED_FIXUP
);
7368 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7369 expand_omp_simd (region
, &fd
);
7370 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7371 expand_cilk_for (region
, &fd
);
7372 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7373 && !fd
.have_ordered
)
7375 if (fd
.chunk_size
== NULL
)
7376 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7378 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7382 int fn_index
, start_ix
, next_ix
;
7384 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7385 == GF_OMP_FOR_KIND_FOR
);
7386 if (fd
.chunk_size
== NULL
7387 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7388 fd
.chunk_size
= integer_zero_node
;
7389 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7390 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7391 ? 3 : fd
.sched_kind
;
7392 fn_index
+= fd
.have_ordered
* 4;
7393 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7394 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7395 if (fd
.iter_type
== long_long_unsigned_type_node
)
7397 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7398 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7399 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7400 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7402 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7403 (enum built_in_function
) next_ix
, inner_stmt
);
7406 if (gimple_in_ssa_p (cfun
))
7407 update_ssa (TODO_update_ssa_only_virtuals
);
7411 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7413 v = GOMP_sections_start (n);
7430 v = GOMP_sections_next ();
7435 If this is a combined parallel sections, replace the call to
7436 GOMP_sections_start with call to GOMP_sections_next. */
7439 expand_omp_sections (struct omp_region
*region
)
7441 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7443 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7444 gimple_stmt_iterator si
, switch_si
;
7445 gomp_sections
*sections_stmt
;
7447 gomp_continue
*cont
;
7450 struct omp_region
*inner
;
7452 bool exit_reachable
= region
->cont
!= NULL
;
7454 gcc_assert (region
->exit
!= NULL
);
7455 entry_bb
= region
->entry
;
7456 l0_bb
= single_succ (entry_bb
);
7457 l1_bb
= region
->cont
;
7458 l2_bb
= region
->exit
;
7459 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7460 l2
= gimple_block_label (l2_bb
);
7463 /* This can happen if there are reductions. */
7464 len
= EDGE_COUNT (l0_bb
->succs
);
7465 gcc_assert (len
> 0);
7466 e
= EDGE_SUCC (l0_bb
, len
- 1);
7467 si
= gsi_last_bb (e
->dest
);
7470 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7471 l2
= gimple_block_label (e
->dest
);
7473 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7475 si
= gsi_last_bb (e
->dest
);
7477 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7479 l2
= gimple_block_label (e
->dest
);
7485 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7487 default_bb
= create_empty_bb (l0_bb
);
7489 /* We will build a switch() with enough cases for all the
7490 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7491 and a default case to abort if something goes wrong. */
7492 len
= EDGE_COUNT (l0_bb
->succs
);
7494 /* Use vec::quick_push on label_vec throughout, since we know the size
7496 auto_vec
<tree
> label_vec (len
);
7498 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7499 GIMPLE_OMP_SECTIONS statement. */
7500 si
= gsi_last_bb (entry_bb
);
7501 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7502 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7503 vin
= gimple_omp_sections_control (sections_stmt
);
7504 if (!is_combined_parallel (region
))
7506 /* If we are not inside a combined parallel+sections region,
7507 call GOMP_sections_start. */
7508 t
= build_int_cst (unsigned_type_node
, len
- 1);
7509 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7510 stmt
= gimple_build_call (u
, 1, t
);
7514 /* Otherwise, call GOMP_sections_next. */
7515 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7516 stmt
= gimple_build_call (u
, 0);
7518 gimple_call_set_lhs (stmt
, vin
);
7519 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7520 gsi_remove (&si
, true);
7522 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7524 switch_si
= gsi_last_bb (l0_bb
);
7525 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7528 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
7529 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7530 vmain
= gimple_omp_continue_control_use (cont
);
7531 vnext
= gimple_omp_continue_control_def (cont
);
7539 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7540 label_vec
.quick_push (t
);
7543 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7544 for (inner
= region
->inner
, casei
= 1;
7546 inner
= inner
->next
, i
++, casei
++)
7548 basic_block s_entry_bb
, s_exit_bb
;
7550 /* Skip optional reduction region. */
7551 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7558 s_entry_bb
= inner
->entry
;
7559 s_exit_bb
= inner
->exit
;
7561 t
= gimple_block_label (s_entry_bb
);
7562 u
= build_int_cst (unsigned_type_node
, casei
);
7563 u
= build_case_label (u
, NULL
, t
);
7564 label_vec
.quick_push (u
);
7566 si
= gsi_last_bb (s_entry_bb
);
7567 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7568 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7569 gsi_remove (&si
, true);
7570 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7572 if (s_exit_bb
== NULL
)
7575 si
= gsi_last_bb (s_exit_bb
);
7576 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7577 gsi_remove (&si
, true);
7579 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7582 /* Error handling code goes in DEFAULT_BB. */
7583 t
= gimple_block_label (default_bb
);
7584 u
= build_case_label (NULL
, NULL
, t
);
7585 make_edge (l0_bb
, default_bb
, 0);
7586 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7588 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7589 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7590 gsi_remove (&switch_si
, true);
7592 si
= gsi_start_bb (default_bb
);
7593 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7594 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7600 /* Code to get the next section goes in L1_BB. */
7601 si
= gsi_last_bb (l1_bb
);
7602 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7604 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7605 stmt
= gimple_build_call (bfn_decl
, 0);
7606 gimple_call_set_lhs (stmt
, vnext
);
7607 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7608 gsi_remove (&si
, true);
7610 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7613 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7614 si
= gsi_last_bb (l2_bb
);
7615 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7616 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7617 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7618 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7620 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7621 stmt
= gimple_build_call (t
, 0);
7622 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7623 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7624 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7625 gsi_remove (&si
, true);
7627 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7631 /* Expand code for an OpenMP single directive. We've already expanded
7632 much of the code, here we simply place the GOMP_barrier call. */
7635 expand_omp_single (struct omp_region
*region
)
7637 basic_block entry_bb
, exit_bb
;
7638 gimple_stmt_iterator si
;
7640 entry_bb
= region
->entry
;
7641 exit_bb
= region
->exit
;
7643 si
= gsi_last_bb (entry_bb
);
7644 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7645 gsi_remove (&si
, true);
7646 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7648 si
= gsi_last_bb (exit_bb
);
7649 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7651 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7652 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7654 gsi_remove (&si
, true);
7655 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7659 /* Generic expansion for OpenMP synchronization directives: master,
7660 ordered and critical. All we need to do here is remove the entry
7661 and exit markers for REGION. */
7664 expand_omp_synch (struct omp_region
*region
)
7666 basic_block entry_bb
, exit_bb
;
7667 gimple_stmt_iterator si
;
7669 entry_bb
= region
->entry
;
7670 exit_bb
= region
->exit
;
7672 si
= gsi_last_bb (entry_bb
);
7673 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7674 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7675 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7676 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7677 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7678 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7679 gsi_remove (&si
, true);
7680 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7684 si
= gsi_last_bb (exit_bb
);
7685 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7686 gsi_remove (&si
, true);
7687 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7691 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7692 operation as a normal volatile load. */
7695 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7696 tree loaded_val
, int index
)
7698 enum built_in_function tmpbase
;
7699 gimple_stmt_iterator gsi
;
7700 basic_block store_bb
;
7703 tree decl
, call
, type
, itype
;
7705 gsi
= gsi_last_bb (load_bb
);
7706 stmt
= gsi_stmt (gsi
);
7707 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7708 loc
= gimple_location (stmt
);
7710 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7711 is smaller than word size, then expand_atomic_load assumes that the load
7712 is atomic. We could avoid the builtin entirely in this case. */
7714 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7715 decl
= builtin_decl_explicit (tmpbase
);
7716 if (decl
== NULL_TREE
)
7719 type
= TREE_TYPE (loaded_val
);
7720 itype
= TREE_TYPE (TREE_TYPE (decl
));
7722 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7723 build_int_cst (NULL
,
7724 gimple_omp_atomic_seq_cst_p (stmt
)
7726 : MEMMODEL_RELAXED
));
7727 if (!useless_type_conversion_p (type
, itype
))
7728 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7729 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7731 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7732 gsi_remove (&gsi
, true);
7734 store_bb
= single_succ (load_bb
);
7735 gsi
= gsi_last_bb (store_bb
);
7736 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7737 gsi_remove (&gsi
, true);
7739 if (gimple_in_ssa_p (cfun
))
7740 update_ssa (TODO_update_ssa_no_phi
);
7745 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7746 operation as a normal volatile store. */
7749 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7750 tree loaded_val
, tree stored_val
, int index
)
7752 enum built_in_function tmpbase
;
7753 gimple_stmt_iterator gsi
;
7754 basic_block store_bb
= single_succ (load_bb
);
7757 tree decl
, call
, type
, itype
;
7761 gsi
= gsi_last_bb (load_bb
);
7762 stmt
= gsi_stmt (gsi
);
7763 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7765 /* If the load value is needed, then this isn't a store but an exchange. */
7766 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7768 gsi
= gsi_last_bb (store_bb
);
7769 stmt
= gsi_stmt (gsi
);
7770 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7771 loc
= gimple_location (stmt
);
7773 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7774 is smaller than word size, then expand_atomic_store assumes that the store
7775 is atomic. We could avoid the builtin entirely in this case. */
7777 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7778 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7779 decl
= builtin_decl_explicit (tmpbase
);
7780 if (decl
== NULL_TREE
)
7783 type
= TREE_TYPE (stored_val
);
7785 /* Dig out the type of the function's second argument. */
7786 itype
= TREE_TYPE (decl
);
7787 itype
= TYPE_ARG_TYPES (itype
);
7788 itype
= TREE_CHAIN (itype
);
7789 itype
= TREE_VALUE (itype
);
7790 imode
= TYPE_MODE (itype
);
7792 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7795 if (!useless_type_conversion_p (itype
, type
))
7796 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7797 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7798 build_int_cst (NULL
,
7799 gimple_omp_atomic_seq_cst_p (stmt
)
7801 : MEMMODEL_RELAXED
));
7804 if (!useless_type_conversion_p (type
, itype
))
7805 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7806 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7809 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7810 gsi_remove (&gsi
, true);
7812 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7813 gsi
= gsi_last_bb (load_bb
);
7814 gsi_remove (&gsi
, true);
7816 if (gimple_in_ssa_p (cfun
))
7817 update_ssa (TODO_update_ssa_no_phi
);
7822 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7823 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7824 size of the data type, and thus usable to find the index of the builtin
7825 decl. Returns false if the expression is not of the proper form. */
7828 expand_omp_atomic_fetch_op (basic_block load_bb
,
7829 tree addr
, tree loaded_val
,
7830 tree stored_val
, int index
)
7832 enum built_in_function oldbase
, newbase
, tmpbase
;
7833 tree decl
, itype
, call
;
7835 basic_block store_bb
= single_succ (load_bb
);
7836 gimple_stmt_iterator gsi
;
7839 enum tree_code code
;
7840 bool need_old
, need_new
;
7844 /* We expect to find the following sequences:
7847 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7850 val = tmp OP something; (or: something OP tmp)
7851 GIMPLE_OMP_STORE (val)
7853 ???FIXME: Allow a more flexible sequence.
7854 Perhaps use data flow to pick the statements.
7858 gsi
= gsi_after_labels (store_bb
);
7859 stmt
= gsi_stmt (gsi
);
7860 loc
= gimple_location (stmt
);
7861 if (!is_gimple_assign (stmt
))
7864 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7866 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7867 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7868 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7869 gcc_checking_assert (!need_old
|| !need_new
);
7871 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7874 /* Check for one of the supported fetch-op operations. */
7875 code
= gimple_assign_rhs_code (stmt
);
7879 case POINTER_PLUS_EXPR
:
7880 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7881 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7884 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7885 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7888 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7889 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7892 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7893 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7896 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7897 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7903 /* Make sure the expression is of the proper form. */
7904 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7905 rhs
= gimple_assign_rhs2 (stmt
);
7906 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7907 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7908 rhs
= gimple_assign_rhs1 (stmt
);
7912 tmpbase
= ((enum built_in_function
)
7913 ((need_new
? newbase
: oldbase
) + index
+ 1));
7914 decl
= builtin_decl_explicit (tmpbase
);
7915 if (decl
== NULL_TREE
)
7917 itype
= TREE_TYPE (TREE_TYPE (decl
));
7918 imode
= TYPE_MODE (itype
);
7920 /* We could test all of the various optabs involved, but the fact of the
7921 matter is that (with the exception of i486 vs i586 and xadd) all targets
7922 that support any atomic operaton optab also implements compare-and-swap.
7923 Let optabs.c take care of expanding any compare-and-swap loop. */
7924 if (!can_compare_and_swap_p (imode
, true))
7927 gsi
= gsi_last_bb (load_bb
);
7928 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7930 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7931 It only requires that the operation happen atomically. Thus we can
7932 use the RELAXED memory model. */
7933 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7934 fold_convert_loc (loc
, itype
, rhs
),
7935 build_int_cst (NULL
,
7936 seq_cst
? MEMMODEL_SEQ_CST
7937 : MEMMODEL_RELAXED
));
7939 if (need_old
|| need_new
)
7941 lhs
= need_old
? loaded_val
: stored_val
;
7942 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7943 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7946 call
= fold_convert_loc (loc
, void_type_node
, call
);
7947 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7948 gsi_remove (&gsi
, true);
7950 gsi
= gsi_last_bb (store_bb
);
7951 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7952 gsi_remove (&gsi
, true);
7953 gsi
= gsi_last_bb (store_bb
);
7954 gsi_remove (&gsi
, true);
7956 if (gimple_in_ssa_p (cfun
))
7957 update_ssa (TODO_update_ssa_no_phi
);
7962 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7966 newval = rhs; // with oldval replacing *addr in rhs
7967 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7968 if (oldval != newval)
7971 INDEX is log2 of the size of the data type, and thus usable to find the
7972 index of the builtin decl. */
7975 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7976 tree addr
, tree loaded_val
, tree stored_val
,
7979 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7980 tree type
, itype
, cmpxchg
, iaddr
;
7981 gimple_stmt_iterator si
;
7982 basic_block loop_header
= single_succ (load_bb
);
7985 enum built_in_function fncode
;
7987 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7988 order to use the RELAXED memory model effectively. */
7989 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7991 cmpxchg
= builtin_decl_explicit (fncode
);
7992 if (cmpxchg
== NULL_TREE
)
7994 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7995 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7997 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8000 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8001 si
= gsi_last_bb (load_bb
);
8002 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8004 /* For floating-point values, we'll need to view-convert them to integers
8005 so that we can perform the atomic compare and swap. Simplify the
8006 following code by always setting up the "i"ntegral variables. */
8007 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8011 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8014 = force_gimple_operand_gsi (&si
,
8015 fold_convert (TREE_TYPE (iaddr
), addr
),
8016 false, NULL_TREE
, true, GSI_SAME_STMT
);
8017 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8018 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8019 loadedi
= create_tmp_var (itype
);
8020 if (gimple_in_ssa_p (cfun
))
8021 loadedi
= make_ssa_name (loadedi
);
8026 loadedi
= loaded_val
;
8029 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8030 tree loaddecl
= builtin_decl_explicit (fncode
);
8033 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8034 build_call_expr (loaddecl
, 2, iaddr
,
8035 build_int_cst (NULL_TREE
,
8036 MEMMODEL_RELAXED
)));
8038 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8039 build_int_cst (TREE_TYPE (iaddr
), 0));
8042 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8045 /* Move the value to the LOADEDI temporary. */
8046 if (gimple_in_ssa_p (cfun
))
8048 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8049 phi
= create_phi_node (loadedi
, loop_header
);
8050 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8054 gsi_insert_before (&si
,
8055 gimple_build_assign (loadedi
, initial
),
8057 if (loadedi
!= loaded_val
)
8059 gimple_stmt_iterator gsi2
;
8062 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8063 gsi2
= gsi_start_bb (loop_header
);
8064 if (gimple_in_ssa_p (cfun
))
8067 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8068 true, GSI_SAME_STMT
);
8069 stmt
= gimple_build_assign (loaded_val
, x
);
8070 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8074 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8075 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8076 true, GSI_SAME_STMT
);
8079 gsi_remove (&si
, true);
8081 si
= gsi_last_bb (store_bb
);
8082 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8085 storedi
= stored_val
;
8088 force_gimple_operand_gsi (&si
,
8089 build1 (VIEW_CONVERT_EXPR
, itype
,
8090 stored_val
), true, NULL_TREE
, true,
8093 /* Build the compare&swap statement. */
8094 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8095 new_storedi
= force_gimple_operand_gsi (&si
,
8096 fold_convert (TREE_TYPE (loadedi
),
8099 true, GSI_SAME_STMT
);
8101 if (gimple_in_ssa_p (cfun
))
8105 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8106 stmt
= gimple_build_assign (old_vali
, loadedi
);
8107 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8109 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8110 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8113 /* Note that we always perform the comparison as an integer, even for
8114 floating point. This allows the atomic operation to properly
8115 succeed even with NaNs and -0.0. */
8116 stmt
= gimple_build_cond_empty
8117 (build2 (NE_EXPR
, boolean_type_node
,
8118 new_storedi
, old_vali
));
8119 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8122 e
= single_succ_edge (store_bb
);
8123 e
->flags
&= ~EDGE_FALLTHRU
;
8124 e
->flags
|= EDGE_FALSE_VALUE
;
8126 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8128 /* Copy the new value to loadedi (we already did that before the condition
8129 if we are not in SSA). */
8130 if (gimple_in_ssa_p (cfun
))
8132 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8133 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8136 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8137 gsi_remove (&si
, true);
8139 struct loop
*loop
= alloc_loop ();
8140 loop
->header
= loop_header
;
8141 loop
->latch
= store_bb
;
8142 add_loop (loop
, loop_header
->loop_father
);
8144 if (gimple_in_ssa_p (cfun
))
8145 update_ssa (TODO_update_ssa_no_phi
);
8150 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8152 GOMP_atomic_start ();
8156 The result is not globally atomic, but works so long as all parallel
8157 references are within #pragma omp atomic directives. According to
8158 responses received from omp@openmp.org, appears to be within spec.
8159 Which makes sense, since that's how several other compilers handle
8160 this situation as well.
8161 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8162 expanding. STORED_VAL is the operand of the matching
8163 GIMPLE_OMP_ATOMIC_STORE.
8166 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8170 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8175 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8176 tree addr
, tree loaded_val
, tree stored_val
)
8178 gimple_stmt_iterator si
;
8182 si
= gsi_last_bb (load_bb
);
8183 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8185 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8186 t
= build_call_expr (t
, 0);
8187 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8189 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8190 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8191 gsi_remove (&si
, true);
8193 si
= gsi_last_bb (store_bb
);
8194 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8196 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8198 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8200 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8201 t
= build_call_expr (t
, 0);
8202 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8203 gsi_remove (&si
, true);
8205 if (gimple_in_ssa_p (cfun
))
8206 update_ssa (TODO_update_ssa_no_phi
);
8210 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8211 using expand_omp_atomic_fetch_op. If it failed, we try to
8212 call expand_omp_atomic_pipeline, and if it fails too, the
8213 ultimate fallback is wrapping the operation in a mutex
8214 (expand_omp_atomic_mutex). REGION is the atomic region built
8215 by build_omp_regions_1(). */
8218 expand_omp_atomic (struct omp_region
*region
)
8220 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8221 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8222 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8223 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8224 tree addr
= gimple_omp_atomic_load_rhs (load
);
8225 tree stored_val
= gimple_omp_atomic_store_val (store
);
8226 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8227 HOST_WIDE_INT index
;
8229 /* Make sure the type is one of the supported sizes. */
8230 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8231 index
= exact_log2 (index
);
8232 if (index
>= 0 && index
<= 4)
8234 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8236 /* __sync builtins require strict data alignment. */
8237 if (exact_log2 (align
) >= index
)
8240 if (loaded_val
== stored_val
8241 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8242 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8243 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8244 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8248 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8249 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8250 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8251 && store_bb
== single_succ (load_bb
)
8252 && first_stmt (store_bb
) == store
8253 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8257 /* When possible, use specialized atomic update functions. */
8258 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8259 && store_bb
== single_succ (load_bb
)
8260 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8261 loaded_val
, stored_val
, index
))
8264 /* If we don't have specialized __sync builtins, try and implement
8265 as a compare and swap loop. */
8266 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8267 loaded_val
, stored_val
, index
))
8272 /* The ultimate fallback is wrapping the operation in a mutex. */
8273 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8277 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
8280 expand_omp_target (struct omp_region
*region
)
8282 basic_block entry_bb
, exit_bb
, new_bb
;
8283 struct function
*child_cfun
= NULL
;
8284 tree child_fn
= NULL_TREE
, block
, t
;
8285 gimple_stmt_iterator gsi
;
8286 gomp_target
*entry_stmt
;
8290 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8291 new_bb
= region
->entry
;
8292 int kind
= gimple_omp_target_kind (entry_stmt
);
8293 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8295 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8296 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8299 entry_bb
= region
->entry
;
8300 exit_bb
= region
->exit
;
8302 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8304 unsigned srcidx
, dstidx
, num
;
8306 /* If the target region needs data sent from the parent
8307 function, then the very first statement (except possible
8308 tree profile counter updates) of the parallel body
8309 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8310 &.OMP_DATA_O is passed as an argument to the child function,
8311 we need to replace it with the argument as seen by the child
8314 In most cases, this will end up being the identity assignment
8315 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
8316 a function call that has been inlined, the original PARM_DECL
8317 .OMP_DATA_I may have been converted into a different local
8318 variable. In which case, we need to keep the assignment. */
8319 if (gimple_omp_target_data_arg (entry_stmt
))
8321 basic_block entry_succ_bb
= single_succ (entry_bb
);
8322 gimple_stmt_iterator gsi
;
8324 gimple tgtcopy_stmt
= NULL
;
8326 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
8328 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8330 gcc_assert (!gsi_end_p (gsi
));
8331 stmt
= gsi_stmt (gsi
);
8332 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8335 if (gimple_num_ops (stmt
) == 2)
8337 tree arg
= gimple_assign_rhs1 (stmt
);
8339 /* We're ignoring the subcode because we're
8340 effectively doing a STRIP_NOPS. */
8342 if (TREE_CODE (arg
) == ADDR_EXPR
8343 && TREE_OPERAND (arg
, 0) == sender
)
8345 tgtcopy_stmt
= stmt
;
8351 gcc_assert (tgtcopy_stmt
!= NULL
);
8352 arg
= DECL_ARGUMENTS (child_fn
);
8354 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8355 gsi_remove (&gsi
, true);
8358 /* Declare local variables needed in CHILD_CFUN. */
8359 block
= DECL_INITIAL (child_fn
);
8360 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8361 /* The gimplifier could record temporaries in target block
8362 rather than in containing function's local_decls chain,
8363 which would mean cgraph missed finalizing them. Do it now. */
8364 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8365 if (TREE_CODE (t
) == VAR_DECL
8367 && !DECL_EXTERNAL (t
))
8368 varpool_node::finalize_decl (t
);
8369 DECL_SAVED_TREE (child_fn
) = NULL
;
8370 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8371 gimple_set_body (child_fn
, NULL
);
8372 TREE_USED (block
) = 1;
8374 /* Reset DECL_CONTEXT on function arguments. */
8375 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8376 DECL_CONTEXT (t
) = child_fn
;
8378 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
8379 so that it can be moved to the child function. */
8380 gsi
= gsi_last_bb (entry_bb
);
8381 stmt
= gsi_stmt (gsi
);
8382 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
8383 && gimple_omp_target_kind (stmt
)
8384 == GF_OMP_TARGET_KIND_REGION
);
8385 gsi_remove (&gsi
, true);
8386 e
= split_block (entry_bb
, stmt
);
8388 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8390 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8393 gsi
= gsi_last_bb (exit_bb
);
8394 gcc_assert (!gsi_end_p (gsi
)
8395 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8396 stmt
= gimple_build_return (NULL
);
8397 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8398 gsi_remove (&gsi
, true);
8401 /* Move the target region into CHILD_CFUN. */
8403 block
= gimple_block (entry_stmt
);
8405 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8407 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8408 /* When the OMP expansion process cannot guarantee an up-to-date
8409 loop tree arrange for the child function to fixup loops. */
8410 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8411 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8413 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8414 num
= vec_safe_length (child_cfun
->local_decls
);
8415 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8417 t
= (*child_cfun
->local_decls
)[srcidx
];
8418 if (DECL_CONTEXT (t
) == cfun
->decl
)
8420 if (srcidx
!= dstidx
)
8421 (*child_cfun
->local_decls
)[dstidx
] = t
;
8425 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8427 /* Inform the callgraph about the new function. */
8428 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8429 cgraph_node::add_new_function (child_fn
, true);
8431 #ifdef ENABLE_OFFLOADING
8432 /* Add the new function to the offload table. */
8433 vec_safe_push (offload_funcs
, child_fn
);
8436 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8437 fixed in a following pass. */
8438 push_cfun (child_cfun
);
8439 cgraph_edge::rebuild_edges ();
8441 #ifdef ENABLE_OFFLOADING
8442 /* Prevent IPA from removing child_fn as unreachable, since there are no
8443 refs from the parent function to child_fn in offload LTO mode. */
8444 struct cgraph_node
*node
= cgraph_node::get (child_fn
);
8445 node
->mark_force_output ();
8448 /* Some EH regions might become dead, see PR34608. If
8449 pass_cleanup_cfg isn't the first pass to happen with the
8450 new child, these dead EH edges might cause problems.
8451 Clean them up now. */
8452 if (flag_exceptions
)
8455 bool changed
= false;
8457 FOR_EACH_BB_FN (bb
, cfun
)
8458 changed
|= gimple_purge_dead_eh_edges (bb
);
8460 cleanup_tree_cfg ();
8465 /* Emit a library call to launch the target region, or do data
8467 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8468 enum built_in_function start_ix
;
8469 location_t clause_loc
;
8471 clauses
= gimple_omp_target_clauses (entry_stmt
);
8473 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8474 start_ix
= BUILT_IN_GOMP_TARGET
;
8475 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8476 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8478 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8480 /* By default, the value of DEVICE is -1 (let runtime library choose)
8481 and there is no conditional. */
8483 device
= build_int_cst (integer_type_node
, -1);
8485 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8487 cond
= OMP_CLAUSE_IF_EXPR (c
);
8489 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8492 device
= OMP_CLAUSE_DEVICE_ID (c
);
8493 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8496 clause_loc
= gimple_location (entry_stmt
);
8498 /* Ensure 'device' is of the correct type. */
8499 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8501 /* If we found the clause 'if (cond)', build
8502 (cond ? device : -2). */
8505 cond
= gimple_boolify (cond
);
8507 basic_block cond_bb
, then_bb
, else_bb
;
8511 tmp_var
= create_tmp_var (TREE_TYPE (device
));
8512 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8514 gsi
= gsi_last_bb (new_bb
);
8516 e
= split_block (new_bb
, gsi_stmt (gsi
));
8519 e
= split_block (new_bb
, NULL
);
8524 then_bb
= create_empty_bb (cond_bb
);
8525 else_bb
= create_empty_bb (then_bb
);
8526 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8527 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8529 stmt
= gimple_build_cond_empty (cond
);
8530 gsi
= gsi_last_bb (cond_bb
);
8531 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8533 gsi
= gsi_start_bb (then_bb
);
8534 stmt
= gimple_build_assign (tmp_var
, device
);
8535 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8537 gsi
= gsi_start_bb (else_bb
);
8538 stmt
= gimple_build_assign (tmp_var
,
8539 build_int_cst (integer_type_node
, -2));
8540 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8542 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8543 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8544 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8545 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8546 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8547 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8552 gsi
= gsi_last_bb (new_bb
);
8553 t
= gimple_omp_target_data_arg (entry_stmt
);
8556 t1
= size_zero_node
;
8557 t2
= build_zero_cst (ptr_type_node
);
8563 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8564 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8565 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8566 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8567 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8571 /* FIXME: This will be address of
8572 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8573 symbol, as soon as the linker plugin is able to create it for us. */
8574 tree openmp_target
= build_zero_cst (ptr_type_node
);
8575 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8577 tree fnaddr
= build_fold_addr_expr (child_fn
);
8578 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8579 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8582 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8583 device
, openmp_target
, t1
, t2
, t3
, t4
);
8584 gimple_set_location (g
, gimple_location (entry_stmt
));
8585 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8586 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8589 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8590 gsi_remove (&gsi
, true);
8592 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8594 gsi
= gsi_last_bb (region
->exit
);
8596 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8597 gsi_remove (&gsi
, true);
8602 /* Expand the parallel region tree rooted at REGION. Expansion
8603 proceeds in depth-first order. Innermost regions are expanded
8604 first. This way, parallel regions that require a new function to
8605 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8606 internal dependencies in their body. */
8609 expand_omp (struct omp_region
*region
)
8613 location_t saved_location
;
8614 gimple inner_stmt
= NULL
;
8616 /* First, determine whether this is a combined parallel+workshare
8618 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8619 determine_parallel_type (region
);
8621 if (region
->type
== GIMPLE_OMP_FOR
8622 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8623 inner_stmt
= last_stmt (region
->inner
->entry
);
8626 expand_omp (region
->inner
);
8628 saved_location
= input_location
;
8629 if (gimple_has_location (last_stmt (region
->entry
)))
8630 input_location
= gimple_location (last_stmt (region
->entry
));
8632 switch (region
->type
)
8634 case GIMPLE_OMP_PARALLEL
:
8635 case GIMPLE_OMP_TASK
:
8636 expand_omp_taskreg (region
);
8639 case GIMPLE_OMP_FOR
:
8640 expand_omp_for (region
, inner_stmt
);
8643 case GIMPLE_OMP_SECTIONS
:
8644 expand_omp_sections (region
);
8647 case GIMPLE_OMP_SECTION
:
8648 /* Individual omp sections are handled together with their
8649 parent GIMPLE_OMP_SECTIONS region. */
8652 case GIMPLE_OMP_SINGLE
:
8653 expand_omp_single (region
);
8656 case GIMPLE_OMP_MASTER
:
8657 case GIMPLE_OMP_TASKGROUP
:
8658 case GIMPLE_OMP_ORDERED
:
8659 case GIMPLE_OMP_CRITICAL
:
8660 case GIMPLE_OMP_TEAMS
:
8661 expand_omp_synch (region
);
8664 case GIMPLE_OMP_ATOMIC_LOAD
:
8665 expand_omp_atomic (region
);
8668 case GIMPLE_OMP_TARGET
:
8669 expand_omp_target (region
);
8676 input_location
= saved_location
;
8677 region
= region
->next
;
8682 /* Helper for build_omp_regions. Scan the dominator tree starting at
8683 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8684 true, the function ends once a single tree is built (otherwise, whole
8685 forest of OMP constructs may be built). */
8688 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8691 gimple_stmt_iterator gsi
;
8695 gsi
= gsi_last_bb (bb
);
8696 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8698 struct omp_region
*region
;
8699 enum gimple_code code
;
8701 stmt
= gsi_stmt (gsi
);
8702 code
= gimple_code (stmt
);
8703 if (code
== GIMPLE_OMP_RETURN
)
8705 /* STMT is the return point out of region PARENT. Mark it
8706 as the exit point and make PARENT the immediately
8707 enclosing region. */
8708 gcc_assert (parent
);
8711 parent
= parent
->outer
;
8713 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8715 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8716 GIMPLE_OMP_RETURN, but matches with
8717 GIMPLE_OMP_ATOMIC_LOAD. */
8718 gcc_assert (parent
);
8719 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8722 parent
= parent
->outer
;
8725 else if (code
== GIMPLE_OMP_CONTINUE
)
8727 gcc_assert (parent
);
8730 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8732 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8733 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8736 else if (code
== GIMPLE_OMP_TARGET
8737 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8738 new_omp_region (bb
, code
, parent
);
8741 /* Otherwise, this directive becomes the parent for a new
8743 region
= new_omp_region (bb
, code
, parent
);
8748 if (single_tree
&& !parent
)
8751 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8753 son
= next_dom_son (CDI_DOMINATORS
, son
))
8754 build_omp_regions_1 (son
, parent
, single_tree
);
8757 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8761 build_omp_regions_root (basic_block root
)
8763 gcc_assert (root_omp_region
== NULL
);
8764 build_omp_regions_1 (root
, NULL
, true);
8765 gcc_assert (root_omp_region
!= NULL
);
8768 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8771 omp_expand_local (basic_block head
)
8773 build_omp_regions_root (head
);
8774 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8776 fprintf (dump_file
, "\nOMP region tree\n\n");
8777 dump_omp_region (dump_file
, root_omp_region
, 0);
8778 fprintf (dump_file
, "\n");
8781 remove_exit_barriers (root_omp_region
);
8782 expand_omp (root_omp_region
);
8784 free_omp_regions ();
8787 /* Scan the CFG and build a tree of OMP regions. Return the root of
8788 the OMP region tree. */
8791 build_omp_regions (void)
8793 gcc_assert (root_omp_region
== NULL
);
8794 calculate_dominance_info (CDI_DOMINATORS
);
8795 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8798 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8801 execute_expand_omp (void)
8803 build_omp_regions ();
8805 if (!root_omp_region
)
8810 fprintf (dump_file
, "\nOMP region tree\n\n");
8811 dump_omp_region (dump_file
, root_omp_region
, 0);
8812 fprintf (dump_file
, "\n");
8815 remove_exit_barriers (root_omp_region
);
8817 expand_omp (root_omp_region
);
8819 cleanup_tree_cfg ();
8821 free_omp_regions ();
8826 /* OMP expansion -- the default pass, run before creation of SSA form. */
8830 const pass_data pass_data_expand_omp
=
8832 GIMPLE_PASS
, /* type */
8833 "ompexp", /* name */
8834 OPTGROUP_NONE
, /* optinfo_flags */
8835 TV_NONE
, /* tv_id */
8836 PROP_gimple_any
, /* properties_required */
8837 PROP_gimple_eomp
, /* properties_provided */
8838 0, /* properties_destroyed */
8839 0, /* todo_flags_start */
8840 0, /* todo_flags_finish */
8843 class pass_expand_omp
: public gimple_opt_pass
8846 pass_expand_omp (gcc::context
*ctxt
)
8847 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8850 /* opt_pass methods: */
8851 virtual unsigned int execute (function
*)
8853 bool gate
= ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8854 || flag_cilkplus
!= 0) && !seen_error ());
8856 /* This pass always runs, to provide PROP_gimple_eomp.
8857 But there is nothing to do unless -fopenmp is given. */
8861 return execute_expand_omp ();
8864 }; // class pass_expand_omp
8869 make_pass_expand_omp (gcc::context
*ctxt
)
8871 return new pass_expand_omp (ctxt
);
8876 const pass_data pass_data_expand_omp_ssa
=
8878 GIMPLE_PASS
, /* type */
8879 "ompexpssa", /* name */
8880 OPTGROUP_NONE
, /* optinfo_flags */
8881 TV_NONE
, /* tv_id */
8882 PROP_cfg
| PROP_ssa
, /* properties_required */
8883 PROP_gimple_eomp
, /* properties_provided */
8884 0, /* properties_destroyed */
8885 0, /* todo_flags_start */
8886 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
8889 class pass_expand_omp_ssa
: public gimple_opt_pass
8892 pass_expand_omp_ssa (gcc::context
*ctxt
)
8893 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
8896 /* opt_pass methods: */
8897 virtual bool gate (function
*fun
)
8899 return !(fun
->curr_properties
& PROP_gimple_eomp
);
8901 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8903 }; // class pass_expand_omp_ssa
8908 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
8910 return new pass_expand_omp_ssa (ctxt
);
8913 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8915 /* If ctx is a worksharing context inside of a cancellable parallel
8916 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8917 and conditional branch to parallel's cancel_label to handle
8918 cancellation in the implicit barrier. */
8921 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8923 gimple omp_return
= gimple_seq_last_stmt (*body
);
8924 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8925 if (gimple_omp_return_nowait_p (omp_return
))
8928 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8929 && ctx
->outer
->cancellable
)
8931 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8932 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8933 tree lhs
= create_tmp_var (c_bool_type
);
8934 gimple_omp_return_set_lhs (omp_return
, lhs
);
8935 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8936 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8937 fold_convert (c_bool_type
,
8938 boolean_false_node
),
8939 ctx
->outer
->cancel_label
, fallthru_label
);
8940 gimple_seq_add_stmt (body
, g
);
8941 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8945 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8946 CTX is the enclosing OMP context for the current statement. */
8949 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8951 tree block
, control
;
8952 gimple_stmt_iterator tgsi
;
8953 gomp_sections
*stmt
;
8955 gbind
*new_stmt
, *bind
;
8956 gimple_seq ilist
, dlist
, olist
, new_body
;
8958 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
8960 push_gimplify_context ();
8964 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8965 &ilist
, &dlist
, ctx
, NULL
);
8967 new_body
= gimple_omp_body (stmt
);
8968 gimple_omp_set_body (stmt
, NULL
);
8969 tgsi
= gsi_start (new_body
);
8970 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8975 sec_start
= gsi_stmt (tgsi
);
8976 sctx
= maybe_lookup_ctx (sec_start
);
8979 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8980 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8981 GSI_CONTINUE_LINKING
);
8982 gimple_omp_set_body (sec_start
, NULL
);
8984 if (gsi_one_before_end_p (tgsi
))
8986 gimple_seq l
= NULL
;
8987 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8989 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8990 gimple_omp_section_set_last (sec_start
);
8993 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8994 GSI_CONTINUE_LINKING
);
8997 block
= make_node (BLOCK
);
8998 bind
= gimple_build_bind (NULL
, new_body
, block
);
9001 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
9003 block
= make_node (BLOCK
);
9004 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9005 gsi_replace (gsi_p
, new_stmt
, true);
9007 pop_gimplify_context (new_stmt
);
9008 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9009 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9010 if (BLOCK_VARS (block
))
9011 TREE_USED (block
) = 1;
9014 gimple_seq_add_seq (&new_body
, ilist
);
9015 gimple_seq_add_stmt (&new_body
, stmt
);
9016 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
9017 gimple_seq_add_stmt (&new_body
, bind
);
9019 control
= create_tmp_var (unsigned_type_node
, ".section");
9020 t
= gimple_build_omp_continue (control
, control
);
9021 gimple_omp_sections_set_control (stmt
, control
);
9022 gimple_seq_add_stmt (&new_body
, t
);
9024 gimple_seq_add_seq (&new_body
, olist
);
9025 if (ctx
->cancellable
)
9026 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9027 gimple_seq_add_seq (&new_body
, dlist
);
9029 new_body
= maybe_catch_exception (new_body
);
9031 t
= gimple_build_omp_return
9032 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
9033 OMP_CLAUSE_NOWAIT
));
9034 gimple_seq_add_stmt (&new_body
, t
);
9035 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
9037 gimple_bind_set_body (new_stmt
, new_body
);
9041 /* A subroutine of lower_omp_single. Expand the simple form of
9042 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
9044 if (GOMP_single_start ())
9046 [ GOMP_barrier (); ] -> unless 'nowait' is present.
9048 FIXME. It may be better to delay expanding the logic of this until
9049 pass_expand_omp. The expanded logic may make the job more difficult
9050 to a synchronization analysis pass. */
9053 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
9055 location_t loc
= gimple_location (single_stmt
);
9056 tree tlabel
= create_artificial_label (loc
);
9057 tree flabel
= create_artificial_label (loc
);
9061 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
9062 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
9063 call
= gimple_build_call (decl
, 0);
9064 gimple_call_set_lhs (call
, lhs
);
9065 gimple_seq_add_stmt (pre_p
, call
);
9067 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
9068 fold_convert_loc (loc
, TREE_TYPE (lhs
),
9071 gimple_seq_add_stmt (pre_p
, cond
);
9072 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
9073 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
9074 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
9078 /* A subroutine of lower_omp_single. Expand the simple form of
9079 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
9081 #pragma omp single copyprivate (a, b, c)
9083 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
9086 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
9092 GOMP_single_copy_end (©out);
9103 FIXME. It may be better to delay expanding the logic of this until
9104 pass_expand_omp. The expanded logic may make the job more difficult
9105 to a synchronization analysis pass. */
9108 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
9111 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
9112 gimple_seq copyin_seq
;
9113 location_t loc
= gimple_location (single_stmt
);
9115 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
9117 ptr_type
= build_pointer_type (ctx
->record_type
);
9118 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
9120 l0
= create_artificial_label (loc
);
9121 l1
= create_artificial_label (loc
);
9122 l2
= create_artificial_label (loc
);
9124 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
9125 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
9126 t
= fold_convert_loc (loc
, ptr_type
, t
);
9127 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
9129 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
9130 build_int_cst (ptr_type
, 0));
9131 t
= build3 (COND_EXPR
, void_type_node
, t
,
9132 build_and_jump (&l0
), build_and_jump (&l1
));
9133 gimplify_and_add (t
, pre_p
);
9135 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
9137 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
9140 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
9143 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9144 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
9145 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
9146 gimplify_and_add (t
, pre_p
);
9148 t
= build_and_jump (&l2
);
9149 gimplify_and_add (t
, pre_p
);
9151 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
9153 gimple_seq_add_seq (pre_p
, copyin_seq
);
9155 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
9159 /* Expand code for an OpenMP single directive. */
9162 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9166 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
9168 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
9170 push_gimplify_context ();
9172 block
= make_node (BLOCK
);
9173 bind
= gimple_build_bind (NULL
, NULL
, block
);
9174 gsi_replace (gsi_p
, bind
, true);
9177 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
9178 &bind_body
, &dlist
, ctx
, NULL
);
9179 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
9181 gimple_seq_add_stmt (&bind_body
, single_stmt
);
9183 if (ctx
->record_type
)
9184 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
9186 lower_omp_single_simple (single_stmt
, &bind_body
);
9188 gimple_omp_set_body (single_stmt
, NULL
);
9190 gimple_seq_add_seq (&bind_body
, dlist
);
9192 bind_body
= maybe_catch_exception (bind_body
);
9194 t
= gimple_build_omp_return
9195 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
9196 OMP_CLAUSE_NOWAIT
));
9197 gimple_seq_add_stmt (&bind_body_tail
, t
);
9198 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
9199 if (ctx
->record_type
)
9201 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
9202 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9203 TREE_THIS_VOLATILE (clobber
) = 1;
9204 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
9205 clobber
), GSI_SAME_STMT
);
9207 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
9208 gimple_bind_set_body (bind
, bind_body
);
9210 pop_gimplify_context (bind
);
9212 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9213 BLOCK_VARS (block
) = ctx
->block_vars
;
9214 if (BLOCK_VARS (block
))
9215 TREE_USED (block
) = 1;
9219 /* Expand code for an OpenMP master directive. */
9222 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9224 tree block
, lab
= NULL
, x
, bfn_decl
;
9225 gimple stmt
= gsi_stmt (*gsi_p
);
9227 location_t loc
= gimple_location (stmt
);
9230 push_gimplify_context ();
9232 block
= make_node (BLOCK
);
9233 bind
= gimple_build_bind (NULL
, NULL
, block
);
9234 gsi_replace (gsi_p
, bind
, true);
9235 gimple_bind_add_stmt (bind
, stmt
);
9237 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9238 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
9239 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
9240 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
9242 gimplify_and_add (x
, &tseq
);
9243 gimple_bind_add_seq (bind
, tseq
);
9245 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9246 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9247 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9248 gimple_omp_set_body (stmt
, NULL
);
9250 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
9252 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9254 pop_gimplify_context (bind
);
9256 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9257 BLOCK_VARS (block
) = ctx
->block_vars
;
9261 /* Expand code for an OpenMP taskgroup directive. */
9264 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9266 gimple stmt
= gsi_stmt (*gsi_p
);
9269 tree block
= make_node (BLOCK
);
9271 bind
= gimple_build_bind (NULL
, NULL
, block
);
9272 gsi_replace (gsi_p
, bind
, true);
9273 gimple_bind_add_stmt (bind
, stmt
);
9275 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
9277 gimple_bind_add_stmt (bind
, x
);
9279 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9280 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9281 gimple_omp_set_body (stmt
, NULL
);
9283 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9285 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9286 BLOCK_VARS (block
) = ctx
->block_vars
;
9290 /* Expand code for an OpenMP ordered directive. */
9293 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9296 gimple stmt
= gsi_stmt (*gsi_p
);
9300 push_gimplify_context ();
9302 block
= make_node (BLOCK
);
9303 bind
= gimple_build_bind (NULL
, NULL
, block
);
9304 gsi_replace (gsi_p
, bind
, true);
9305 gimple_bind_add_stmt (bind
, stmt
);
9307 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
9309 gimple_bind_add_stmt (bind
, x
);
9311 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9312 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9313 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9314 gimple_omp_set_body (stmt
, NULL
);
9316 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
9317 gimple_bind_add_stmt (bind
, x
);
9319 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9321 pop_gimplify_context (bind
);
9323 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9324 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9328 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
9329 substitution of a couple of function calls. But in the NAMED case,
9330 requires that languages coordinate a symbol name. It is therefore
9331 best put here in common code. */
9333 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
9336 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9339 tree name
, lock
, unlock
;
9340 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
9342 location_t loc
= gimple_location (stmt
);
9345 name
= gimple_omp_critical_name (stmt
);
9350 if (!critical_name_mutexes
)
9351 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
9353 tree
*n
= critical_name_mutexes
->get (name
);
9358 decl
= create_tmp_var_raw (ptr_type_node
);
9360 new_str
= ACONCAT ((".gomp_critical_user_",
9361 IDENTIFIER_POINTER (name
), NULL
));
9362 DECL_NAME (decl
) = get_identifier (new_str
);
9363 TREE_PUBLIC (decl
) = 1;
9364 TREE_STATIC (decl
) = 1;
9365 DECL_COMMON (decl
) = 1;
9366 DECL_ARTIFICIAL (decl
) = 1;
9367 DECL_IGNORED_P (decl
) = 1;
9369 varpool_node::finalize_decl (decl
);
9371 critical_name_mutexes
->put (name
, decl
);
9376 /* If '#pragma omp critical' is inside target region or
9377 inside function marked as offloadable, the symbol must be
9378 marked as offloadable too. */
9380 if (cgraph_node::get (current_function_decl
)->offloadable
)
9381 varpool_node::get_create (decl
)->offloadable
= 1;
9383 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
9384 if (is_targetreg_ctx (octx
))
9386 varpool_node::get_create (decl
)->offloadable
= 1;
9390 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
9391 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
9393 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
9394 unlock
= build_call_expr_loc (loc
, unlock
, 1,
9395 build_fold_addr_expr_loc (loc
, decl
));
9399 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9400 lock
= build_call_expr_loc (loc
, lock
, 0);
9402 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9403 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9406 push_gimplify_context ();
9408 block
= make_node (BLOCK
);
9409 bind
= gimple_build_bind (NULL
, NULL
, block
);
9410 gsi_replace (gsi_p
, bind
, true);
9411 gimple_bind_add_stmt (bind
, stmt
);
9413 tbody
= gimple_bind_body (bind
);
9414 gimplify_and_add (lock
, &tbody
);
9415 gimple_bind_set_body (bind
, tbody
);
9417 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9418 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9419 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9420 gimple_omp_set_body (stmt
, NULL
);
9422 tbody
= gimple_bind_body (bind
);
9423 gimplify_and_add (unlock
, &tbody
);
9424 gimple_bind_set_body (bind
, tbody
);
9426 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9428 pop_gimplify_context (bind
);
9429 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9430 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9434 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9435 for a lastprivate clause. Given a loop control predicate of (V
9436 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9437 is appended to *DLIST, iterator initialization is appended to
9441 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9442 gimple_seq
*dlist
, struct omp_context
*ctx
)
9444 tree clauses
, cond
, vinit
;
9445 enum tree_code cond_code
;
9448 cond_code
= fd
->loop
.cond_code
;
9449 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9451 /* When possible, use a strict equality expression. This can let VRP
9452 type optimizations deduce the value and remove a copy. */
9453 if (tree_fits_shwi_p (fd
->loop
.step
))
9455 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9456 if (step
== 1 || step
== -1)
9457 cond_code
= EQ_EXPR
;
9460 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
9462 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9464 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
9465 if (!gimple_seq_empty_p (stmts
))
9467 gimple_seq_add_seq (&stmts
, *dlist
);
9470 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9471 vinit
= fd
->loop
.n1
;
9472 if (cond_code
== EQ_EXPR
9473 && tree_fits_shwi_p (fd
->loop
.n2
)
9474 && ! integer_zerop (fd
->loop
.n2
))
9475 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9477 vinit
= unshare_expr (vinit
);
9479 /* Initialize the iterator variable, so that threads that don't execute
9480 any iterations don't execute the lastprivate clauses by accident. */
9481 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9486 /* Lower code for an OpenMP loop directive. */
9489 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9492 struct omp_for_data fd
, *fdp
= NULL
;
9493 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
9495 gimple_seq omp_for_body
, body
, dlist
;
9498 push_gimplify_context ();
9500 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9502 block
= make_node (BLOCK
);
9503 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9504 /* Replace at gsi right away, so that 'stmt' is no member
9505 of a sequence anymore as we're going to add to to a different
9507 gsi_replace (gsi_p
, new_stmt
, true);
9509 /* Move declaration of temporaries in the loop body before we make
9511 omp_for_body
= gimple_omp_body (stmt
);
9512 if (!gimple_seq_empty_p (omp_for_body
)
9513 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9516 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
9517 tree vars
= gimple_bind_vars (inner_bind
);
9518 gimple_bind_append_vars (new_stmt
, vars
);
9519 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9520 keep them on the inner_bind and it's block. */
9521 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9522 if (gimple_bind_block (inner_bind
))
9523 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9526 if (gimple_omp_for_combined_into_p (stmt
))
9528 extract_omp_for_data (stmt
, &fd
, NULL
);
9531 /* We need two temporaries with fd.loop.v type (istart/iend)
9532 and then (fd.collapse - 1) temporaries with the same
9533 type for count2 ... countN-1 vars if not constant. */
9535 tree type
= fd
.iter_type
;
9537 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9538 count
+= fd
.collapse
- 1;
9539 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9540 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9544 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9545 OMP_CLAUSE__LOOPTEMP_
);
9546 for (i
= 0; i
< count
; i
++)
9551 gcc_assert (outerc
);
9552 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9553 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9554 OMP_CLAUSE__LOOPTEMP_
);
9558 temp
= create_tmp_var (type
);
9559 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9561 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9562 OMP_CLAUSE_DECL (*pc
) = temp
;
9563 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9568 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9571 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9573 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9575 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9577 /* Lower the header expressions. At this point, we can assume that
9578 the header is of the form:
9580 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9582 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9583 using the .omp_data_s mapping, if needed. */
9584 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9586 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9587 if (!is_gimple_min_invariant (*rhs_p
))
9588 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9590 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9591 if (!is_gimple_min_invariant (*rhs_p
))
9592 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9594 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9595 if (!is_gimple_min_invariant (*rhs_p
))
9596 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9599 /* Once lowered, extract the bounds and clauses. */
9600 extract_omp_for_data (stmt
, &fd
, NULL
);
9602 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9604 gimple_seq_add_stmt (&body
, stmt
);
9605 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9607 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9610 /* After the loop, add exit clauses. */
9611 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9613 if (ctx
->cancellable
)
9614 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9616 gimple_seq_add_seq (&body
, dlist
);
9618 body
= maybe_catch_exception (body
);
9620 /* Region exit marker goes at the end of the loop body. */
9621 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9622 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9623 pop_gimplify_context (new_stmt
);
9625 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9626 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9627 if (BLOCK_VARS (block
))
9628 TREE_USED (block
) = 1;
9630 gimple_bind_set_body (new_stmt
, body
);
9631 gimple_omp_set_body (stmt
, NULL
);
9632 gimple_omp_for_set_pre_body (stmt
, NULL
);
9635 /* Callback for walk_stmts. Check if the current statement only contains
9636 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9639 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9640 bool *handled_ops_p
,
9641 struct walk_stmt_info
*wi
)
9643 int *info
= (int *) wi
->info
;
9644 gimple stmt
= gsi_stmt (*gsi_p
);
9646 *handled_ops_p
= true;
9647 switch (gimple_code (stmt
))
9651 case GIMPLE_OMP_FOR
:
9652 case GIMPLE_OMP_SECTIONS
:
9653 *info
= *info
== 0 ? 1 : -1;
9662 struct omp_taskcopy_context
9664 /* This field must be at the beginning, as we do "inheritance": Some
9665 callback functions for tree-inline.c (e.g., omp_copy_decl)
9666 receive a copy_body_data pointer that is up-casted to an
9667 omp_context pointer. */
9673 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9675 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9677 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9678 return create_tmp_var (TREE_TYPE (var
));
9684 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9686 tree name
, new_fields
= NULL
, type
, f
;
9688 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9689 name
= DECL_NAME (TYPE_NAME (orig_type
));
9690 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9691 TYPE_DECL
, name
, type
);
9692 TYPE_NAME (type
) = name
;
9694 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9696 tree new_f
= copy_node (f
);
9697 DECL_CONTEXT (new_f
) = type
;
9698 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9699 TREE_CHAIN (new_f
) = new_fields
;
9700 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9701 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9702 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9705 tcctx
->cb
.decl_map
->put (f
, new_f
);
9707 TYPE_FIELDS (type
) = nreverse (new_fields
);
9712 /* Create task copyfn. */
9715 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
9717 struct function
*child_cfun
;
9718 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9719 tree record_type
, srecord_type
, bind
, list
;
9720 bool record_needs_remap
= false, srecord_needs_remap
= false;
9722 struct omp_taskcopy_context tcctx
;
9723 location_t loc
= gimple_location (task_stmt
);
9725 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9726 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9727 gcc_assert (child_cfun
->cfg
== NULL
);
9728 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9730 /* Reset DECL_CONTEXT on function arguments. */
9731 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9732 DECL_CONTEXT (t
) = child_fn
;
9734 /* Populate the function. */
9735 push_gimplify_context ();
9736 push_cfun (child_cfun
);
9738 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9739 TREE_SIDE_EFFECTS (bind
) = 1;
9741 DECL_SAVED_TREE (child_fn
) = bind
;
9742 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9744 /* Remap src and dst argument types if needed. */
9745 record_type
= ctx
->record_type
;
9746 srecord_type
= ctx
->srecord_type
;
9747 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9748 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9750 record_needs_remap
= true;
9753 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9754 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9756 srecord_needs_remap
= true;
9760 if (record_needs_remap
|| srecord_needs_remap
)
9762 memset (&tcctx
, '\0', sizeof (tcctx
));
9763 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9764 tcctx
.cb
.dst_fn
= child_fn
;
9765 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
9766 gcc_checking_assert (tcctx
.cb
.src_node
);
9767 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9768 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9769 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9770 tcctx
.cb
.eh_lp_nr
= 0;
9771 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9772 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
9775 if (record_needs_remap
)
9776 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9777 if (srecord_needs_remap
)
9778 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9781 tcctx
.cb
.decl_map
= NULL
;
9783 arg
= DECL_ARGUMENTS (child_fn
);
9784 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9785 sarg
= DECL_CHAIN (arg
);
9786 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9788 /* First pass: initialize temporaries used in record_type and srecord_type
9789 sizes and field offsets. */
9790 if (tcctx
.cb
.decl_map
)
9791 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9792 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9796 decl
= OMP_CLAUSE_DECL (c
);
9797 p
= tcctx
.cb
.decl_map
->get (decl
);
9800 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9801 sf
= (tree
) n
->value
;
9802 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9803 src
= build_simple_mem_ref_loc (loc
, sarg
);
9804 src
= omp_build_component_ref (src
, sf
);
9805 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9806 append_to_statement_list (t
, &list
);
9809 /* Second pass: copy shared var pointers and copy construct non-VLA
9810 firstprivate vars. */
9811 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9812 switch (OMP_CLAUSE_CODE (c
))
9814 case OMP_CLAUSE_SHARED
:
9815 decl
= OMP_CLAUSE_DECL (c
);
9816 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9819 f
= (tree
) n
->value
;
9820 if (tcctx
.cb
.decl_map
)
9821 f
= *tcctx
.cb
.decl_map
->get (f
);
9822 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9823 sf
= (tree
) n
->value
;
9824 if (tcctx
.cb
.decl_map
)
9825 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9826 src
= build_simple_mem_ref_loc (loc
, sarg
);
9827 src
= omp_build_component_ref (src
, sf
);
9828 dst
= build_simple_mem_ref_loc (loc
, arg
);
9829 dst
= omp_build_component_ref (dst
, f
);
9830 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9831 append_to_statement_list (t
, &list
);
9833 case OMP_CLAUSE_FIRSTPRIVATE
:
9834 decl
= OMP_CLAUSE_DECL (c
);
9835 if (is_variable_sized (decl
))
9837 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9840 f
= (tree
) n
->value
;
9841 if (tcctx
.cb
.decl_map
)
9842 f
= *tcctx
.cb
.decl_map
->get (f
);
9843 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9846 sf
= (tree
) n
->value
;
9847 if (tcctx
.cb
.decl_map
)
9848 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9849 src
= build_simple_mem_ref_loc (loc
, sarg
);
9850 src
= omp_build_component_ref (src
, sf
);
9851 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9852 src
= build_simple_mem_ref_loc (loc
, src
);
9856 dst
= build_simple_mem_ref_loc (loc
, arg
);
9857 dst
= omp_build_component_ref (dst
, f
);
9858 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9859 append_to_statement_list (t
, &list
);
9861 case OMP_CLAUSE_PRIVATE
:
9862 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9864 decl
= OMP_CLAUSE_DECL (c
);
9865 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9866 f
= (tree
) n
->value
;
9867 if (tcctx
.cb
.decl_map
)
9868 f
= *tcctx
.cb
.decl_map
->get (f
);
9869 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9872 sf
= (tree
) n
->value
;
9873 if (tcctx
.cb
.decl_map
)
9874 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9875 src
= build_simple_mem_ref_loc (loc
, sarg
);
9876 src
= omp_build_component_ref (src
, sf
);
9877 if (use_pointer_for_field (decl
, NULL
))
9878 src
= build_simple_mem_ref_loc (loc
, src
);
9882 dst
= build_simple_mem_ref_loc (loc
, arg
);
9883 dst
= omp_build_component_ref (dst
, f
);
9884 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9885 append_to_statement_list (t
, &list
);
9891 /* Last pass: handle VLA firstprivates. */
9892 if (tcctx
.cb
.decl_map
)
9893 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9894 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9898 decl
= OMP_CLAUSE_DECL (c
);
9899 if (!is_variable_sized (decl
))
9901 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9904 f
= (tree
) n
->value
;
9905 f
= *tcctx
.cb
.decl_map
->get (f
);
9906 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9907 ind
= DECL_VALUE_EXPR (decl
);
9908 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9909 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9910 n
= splay_tree_lookup (ctx
->sfield_map
,
9911 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9912 sf
= (tree
) n
->value
;
9913 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9914 src
= build_simple_mem_ref_loc (loc
, sarg
);
9915 src
= omp_build_component_ref (src
, sf
);
9916 src
= build_simple_mem_ref_loc (loc
, src
);
9917 dst
= build_simple_mem_ref_loc (loc
, arg
);
9918 dst
= omp_build_component_ref (dst
, f
);
9919 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9920 append_to_statement_list (t
, &list
);
9921 n
= splay_tree_lookup (ctx
->field_map
,
9922 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9923 df
= (tree
) n
->value
;
9924 df
= *tcctx
.cb
.decl_map
->get (df
);
9925 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9926 ptr
= omp_build_component_ref (ptr
, df
);
9927 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9928 build_fold_addr_expr_loc (loc
, dst
));
9929 append_to_statement_list (t
, &list
);
9932 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9933 append_to_statement_list (t
, &list
);
9935 if (tcctx
.cb
.decl_map
)
9936 delete tcctx
.cb
.decl_map
;
9937 pop_gimplify_context (NULL
);
9938 BIND_EXPR_BODY (bind
) = list
;
9943 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9947 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9949 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9951 gcc_assert (clauses
);
9952 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9953 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9954 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9956 case OMP_CLAUSE_DEPEND_IN
:
9959 case OMP_CLAUSE_DEPEND_OUT
:
9960 case OMP_CLAUSE_DEPEND_INOUT
:
9966 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9967 tree array
= create_tmp_var (type
);
9968 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9970 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9971 gimple_seq_add_stmt (iseq
, g
);
9972 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9974 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9975 gimple_seq_add_stmt (iseq
, g
);
9976 for (i
= 0; i
< 2; i
++)
9978 if ((i
? n_in
: n_out
) == 0)
9980 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9981 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9982 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9984 tree t
= OMP_CLAUSE_DECL (c
);
9985 t
= fold_convert (ptr_type_node
, t
);
9986 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9987 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9988 NULL_TREE
, NULL_TREE
);
9989 g
= gimple_build_assign (r
, t
);
9990 gimple_seq_add_stmt (iseq
, g
);
9993 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9994 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9995 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9996 OMP_CLAUSE_CHAIN (c
) = *p
;
9998 tree clobber
= build_constructor (type
, NULL
);
9999 TREE_THIS_VOLATILE (clobber
) = 1;
10000 g
= gimple_build_assign (array
, clobber
);
10001 gimple_seq_add_stmt (oseq
, g
);
10004 /* Lower the OpenMP parallel or task directive in the current statement
10005 in GSI_P. CTX holds context information for the directive. */
10008 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10012 gimple stmt
= gsi_stmt (*gsi_p
);
10013 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
10014 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
10015 location_t loc
= gimple_location (stmt
);
10017 clauses
= gimple_omp_taskreg_clauses (stmt
);
10019 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
10020 par_body
= gimple_bind_body (par_bind
);
10021 child_fn
= ctx
->cb
.dst_fn
;
10022 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
10023 && !gimple_omp_parallel_combined_p (stmt
))
10025 struct walk_stmt_info wi
;
10028 memset (&wi
, 0, sizeof (wi
));
10030 wi
.val_only
= true;
10031 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
10033 gimple_omp_parallel_set_combined_p (stmt
, true);
10035 gimple_seq dep_ilist
= NULL
;
10036 gimple_seq dep_olist
= NULL
;
10037 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
10038 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
10040 push_gimplify_context ();
10041 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
10042 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
10045 if (ctx
->srecord_type
)
10046 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
10048 push_gimplify_context ();
10053 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
10054 lower_omp (&par_body
, ctx
);
10055 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
10056 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
10058 /* Declare all the variables created by mapping and the variables
10059 declared in the scope of the parallel body. */
10060 record_vars_into (ctx
->block_vars
, child_fn
);
10061 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
10063 if (ctx
->record_type
)
10066 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
10067 : ctx
->record_type
, ".omp_data_o");
10068 DECL_NAMELESS (ctx
->sender_decl
) = 1;
10069 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
10070 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
10075 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
10076 lower_send_shared_vars (&ilist
, &olist
, ctx
);
10078 if (ctx
->record_type
)
10080 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
10081 TREE_THIS_VOLATILE (clobber
) = 1;
10082 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
10086 /* Once all the expansions are done, sequence all the different
10087 fragments inside gimple_omp_body. */
10091 if (ctx
->record_type
)
10093 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10094 /* fixup_child_record_type might have changed receiver_decl's type. */
10095 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10096 gimple_seq_add_stmt (&new_body
,
10097 gimple_build_assign (ctx
->receiver_decl
, t
));
10100 gimple_seq_add_seq (&new_body
, par_ilist
);
10101 gimple_seq_add_seq (&new_body
, par_body
);
10102 gimple_seq_add_seq (&new_body
, par_rlist
);
10103 if (ctx
->cancellable
)
10104 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10105 gimple_seq_add_seq (&new_body
, par_olist
);
10106 new_body
= maybe_catch_exception (new_body
);
10107 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10108 gimple_omp_set_body (stmt
, new_body
);
10110 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
10111 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
10112 gimple_bind_add_seq (bind
, ilist
);
10113 gimple_bind_add_stmt (bind
, stmt
);
10114 gimple_bind_add_seq (bind
, olist
);
10116 pop_gimplify_context (NULL
);
10120 gimple_bind_add_seq (dep_bind
, dep_ilist
);
10121 gimple_bind_add_stmt (dep_bind
, bind
);
10122 gimple_bind_add_seq (dep_bind
, dep_olist
);
10123 pop_gimplify_context (dep_bind
);
10127 /* Lower the OpenMP target directive in the current statement
10128 in GSI_P. CTX holds context information for the directive. */
10131 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10134 tree child_fn
, t
, c
;
10135 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
10136 gbind
*tgt_bind
= NULL
, *bind
;
10137 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
10138 location_t loc
= gimple_location (stmt
);
10139 int kind
= gimple_omp_target_kind (stmt
);
10140 unsigned int map_cnt
= 0;
10142 clauses
= gimple_omp_target_clauses (stmt
);
10143 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10145 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
10146 tgt_body
= gimple_bind_body (tgt_bind
);
10148 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10149 tgt_body
= gimple_omp_body (stmt
);
10150 child_fn
= ctx
->cb
.dst_fn
;
10152 push_gimplify_context ();
10154 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10155 switch (OMP_CLAUSE_CODE (c
))
10161 case OMP_CLAUSE_MAP
:
10162 case OMP_CLAUSE_TO
:
10163 case OMP_CLAUSE_FROM
:
10164 var
= OMP_CLAUSE_DECL (c
);
10167 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
10168 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10173 if (DECL_SIZE (var
)
10174 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
10176 tree var2
= DECL_VALUE_EXPR (var
);
10177 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
10178 var2
= TREE_OPERAND (var2
, 0);
10179 gcc_assert (DECL_P (var2
));
10183 if (!maybe_lookup_field (var
, ctx
))
10186 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10188 x
= build_receiver_ref (var
, true, ctx
);
10189 tree new_var
= lookup_decl (var
, ctx
);
10190 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10191 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10192 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10193 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
10194 x
= build_simple_mem_ref (x
);
10195 SET_DECL_VALUE_EXPR (new_var
, x
);
10196 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
10201 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10203 target_nesting_level
++;
10204 lower_omp (&tgt_body
, ctx
);
10205 target_nesting_level
--;
10207 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10208 lower_omp (&tgt_body
, ctx
);
10210 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10212 /* Declare all the variables created by mapping and the variables
10213 declared in the scope of the target body. */
10214 record_vars_into (ctx
->block_vars
, child_fn
);
10215 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
10220 if (ctx
->record_type
)
10223 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
10224 DECL_NAMELESS (ctx
->sender_decl
) = 1;
10225 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
10226 t
= make_tree_vec (3);
10227 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
10228 TREE_VEC_ELT (t
, 1)
10229 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
10230 ".omp_data_sizes");
10231 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
10232 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
10233 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
10234 TREE_VEC_ELT (t
, 2)
10235 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
10237 ".omp_data_kinds");
10238 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
10239 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
10240 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
10241 gimple_omp_target_set_data_arg (stmt
, t
);
10243 vec
<constructor_elt
, va_gc
> *vsize
;
10244 vec
<constructor_elt
, va_gc
> *vkind
;
10245 vec_alloc (vsize
, map_cnt
);
10246 vec_alloc (vkind
, map_cnt
);
10247 unsigned int map_idx
= 0;
10249 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10250 switch (OMP_CLAUSE_CODE (c
))
10256 case OMP_CLAUSE_MAP
:
10257 case OMP_CLAUSE_TO
:
10258 case OMP_CLAUSE_FROM
:
10260 ovar
= OMP_CLAUSE_DECL (c
);
10261 if (!DECL_P (ovar
))
10263 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10264 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10266 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
10267 == get_base_address (ovar
));
10268 nc
= OMP_CLAUSE_CHAIN (c
);
10269 ovar
= OMP_CLAUSE_DECL (nc
);
10273 tree x
= build_sender_ref (ovar
, ctx
);
10275 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
10276 gimplify_assign (x
, v
, &ilist
);
10282 if (DECL_SIZE (ovar
)
10283 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
10285 tree ovar2
= DECL_VALUE_EXPR (ovar
);
10286 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
10287 ovar2
= TREE_OPERAND (ovar2
, 0);
10288 gcc_assert (DECL_P (ovar2
));
10291 if (!maybe_lookup_field (ovar
, ctx
))
10295 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
10296 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
10297 talign
= DECL_ALIGN_UNIT (ovar
);
10300 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
10301 tree x
= build_sender_ref (ovar
, ctx
);
10302 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10303 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10304 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10305 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
10307 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10309 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
10310 mark_addressable (avar
);
10311 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
10312 talign
= DECL_ALIGN_UNIT (avar
);
10313 avar
= build_fold_addr_expr (avar
);
10314 gimplify_assign (x
, avar
, &ilist
);
10316 else if (is_gimple_reg (var
))
10318 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10319 tree avar
= create_tmp_var (TREE_TYPE (var
));
10320 mark_addressable (avar
);
10321 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
10322 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
10323 gimplify_assign (avar
, var
, &ilist
);
10324 avar
= build_fold_addr_expr (avar
);
10325 gimplify_assign (x
, avar
, &ilist
);
10326 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
10327 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
10328 && !TYPE_READONLY (TREE_TYPE (var
)))
10330 x
= build_sender_ref (ovar
, ctx
);
10331 x
= build_simple_mem_ref (x
);
10332 gimplify_assign (var
, x
, &olist
);
10337 var
= build_fold_addr_expr (var
);
10338 gimplify_assign (x
, var
, &ilist
);
10341 tree s
= OMP_CLAUSE_SIZE (c
);
10342 if (s
== NULL_TREE
)
10343 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
10344 s
= fold_convert (size_type_node
, s
);
10345 tree purpose
= size_int (map_idx
++);
10346 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
10347 if (TREE_CODE (s
) != INTEGER_CST
)
10348 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
10350 unsigned char tkind
= 0;
10351 switch (OMP_CLAUSE_CODE (c
))
10353 case OMP_CLAUSE_MAP
:
10354 tkind
= OMP_CLAUSE_MAP_KIND (c
);
10356 case OMP_CLAUSE_TO
:
10357 tkind
= OMP_CLAUSE_MAP_TO
;
10359 case OMP_CLAUSE_FROM
:
10360 tkind
= OMP_CLAUSE_MAP_FROM
;
10363 gcc_unreachable ();
10365 talign
= ceil_log2 (talign
);
10366 tkind
|= talign
<< 3;
10367 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
10368 build_int_cst (unsigned_char_type_node
,
10374 gcc_assert (map_idx
== map_cnt
);
10376 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
10377 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
10378 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
10379 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
10380 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
10382 gimple_seq initlist
= NULL
;
10383 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
10384 TREE_VEC_ELT (t
, 1)),
10385 &initlist
, true, NULL_TREE
);
10386 gimple_seq_add_seq (&ilist
, initlist
);
10388 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
10390 TREE_THIS_VOLATILE (clobber
) = 1;
10391 gimple_seq_add_stmt (&olist
,
10392 gimple_build_assign (TREE_VEC_ELT (t
, 1),
10396 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10397 TREE_THIS_VOLATILE (clobber
) = 1;
10398 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
10402 /* Once all the expansions are done, sequence all the different
10403 fragments inside gimple_omp_body. */
10407 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
10409 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10410 /* fixup_child_record_type might have changed receiver_decl's type. */
10411 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10412 gimple_seq_add_stmt (&new_body
,
10413 gimple_build_assign (ctx
->receiver_decl
, t
));
10416 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10418 gimple_seq_add_seq (&new_body
, tgt_body
);
10419 new_body
= maybe_catch_exception (new_body
);
10421 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10422 new_body
= tgt_body
;
10423 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
10425 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10426 gimple_omp_set_body (stmt
, new_body
);
10429 bind
= gimple_build_bind (NULL
, NULL
,
10430 tgt_bind
? gimple_bind_block (tgt_bind
)
10432 gsi_replace (gsi_p
, bind
, true);
10433 gimple_bind_add_seq (bind
, ilist
);
10434 gimple_bind_add_stmt (bind
, stmt
);
10435 gimple_bind_add_seq (bind
, olist
);
10437 pop_gimplify_context (NULL
);
10440 /* Expand code for an OpenMP teams directive. */
10443 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10445 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
10446 push_gimplify_context ();
10448 tree block
= make_node (BLOCK
);
10449 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
10450 gsi_replace (gsi_p
, bind
, true);
10451 gimple_seq bind_body
= NULL
;
10452 gimple_seq dlist
= NULL
;
10453 gimple_seq olist
= NULL
;
10455 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10456 OMP_CLAUSE_NUM_TEAMS
);
10457 if (num_teams
== NULL_TREE
)
10458 num_teams
= build_int_cst (unsigned_type_node
, 0);
10461 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
10462 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
10463 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
10465 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10466 OMP_CLAUSE_THREAD_LIMIT
);
10467 if (thread_limit
== NULL_TREE
)
10468 thread_limit
= build_int_cst (unsigned_type_node
, 0);
10471 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
10472 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
10473 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
10477 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
10478 &bind_body
, &dlist
, ctx
, NULL
);
10479 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
10480 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
10481 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
10483 location_t loc
= gimple_location (teams_stmt
);
10484 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
10485 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
10486 gimple_set_location (call
, loc
);
10487 gimple_seq_add_stmt (&bind_body
, call
);
10489 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10490 gimple_omp_set_body (teams_stmt
, NULL
);
10491 gimple_seq_add_seq (&bind_body
, olist
);
10492 gimple_seq_add_seq (&bind_body
, dlist
);
10493 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10494 gimple_bind_set_body (bind
, bind_body
);
10496 pop_gimplify_context (bind
);
10498 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10499 BLOCK_VARS (block
) = ctx
->block_vars
;
10500 if (BLOCK_VARS (block
))
10501 TREE_USED (block
) = 1;
10505 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10506 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10507 of OpenMP context, but with task_shared_vars set. */
10510 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10515 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10516 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10519 if (task_shared_vars
10521 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10524 /* If a global variable has been privatized, TREE_CONSTANT on
10525 ADDR_EXPR might be wrong. */
10526 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10527 recompute_tree_invariant_for_addr_expr (t
);
10529 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10534 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10536 gimple stmt
= gsi_stmt (*gsi_p
);
10537 struct walk_stmt_info wi
;
10540 if (gimple_has_location (stmt
))
10541 input_location
= gimple_location (stmt
);
10543 if (task_shared_vars
)
10544 memset (&wi
, '\0', sizeof (wi
));
10546 /* If we have issued syntax errors, avoid doing any heavy lifting.
10547 Just replace the OpenMP directives with a NOP to avoid
10548 confusing RTL expansion. */
10549 if (seen_error () && is_gimple_omp (stmt
))
10551 gsi_replace (gsi_p
, gimple_build_nop (), true);
10555 switch (gimple_code (stmt
))
10559 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
10560 if ((ctx
|| task_shared_vars
)
10561 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
10562 lower_omp_regimplify_p
,
10563 ctx
? NULL
: &wi
, NULL
)
10564 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
10565 lower_omp_regimplify_p
,
10566 ctx
? NULL
: &wi
, NULL
)))
10567 gimple_regimplify_operands (cond_stmt
, gsi_p
);
10571 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
10573 case GIMPLE_EH_FILTER
:
10574 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10577 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10578 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10580 case GIMPLE_TRANSACTION
:
10581 lower_omp (gimple_transaction_body_ptr (
10582 as_a
<gtransaction
*> (stmt
)),
10586 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
10588 case GIMPLE_OMP_PARALLEL
:
10589 case GIMPLE_OMP_TASK
:
10590 ctx
= maybe_lookup_ctx (stmt
);
10592 if (ctx
->cancellable
)
10593 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10594 lower_omp_taskreg (gsi_p
, ctx
);
10596 case GIMPLE_OMP_FOR
:
10597 ctx
= maybe_lookup_ctx (stmt
);
10599 if (ctx
->cancellable
)
10600 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10601 lower_omp_for (gsi_p
, ctx
);
10603 case GIMPLE_OMP_SECTIONS
:
10604 ctx
= maybe_lookup_ctx (stmt
);
10606 if (ctx
->cancellable
)
10607 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10608 lower_omp_sections (gsi_p
, ctx
);
10610 case GIMPLE_OMP_SINGLE
:
10611 ctx
= maybe_lookup_ctx (stmt
);
10613 lower_omp_single (gsi_p
, ctx
);
10615 case GIMPLE_OMP_MASTER
:
10616 ctx
= maybe_lookup_ctx (stmt
);
10618 lower_omp_master (gsi_p
, ctx
);
10620 case GIMPLE_OMP_TASKGROUP
:
10621 ctx
= maybe_lookup_ctx (stmt
);
10623 lower_omp_taskgroup (gsi_p
, ctx
);
10625 case GIMPLE_OMP_ORDERED
:
10626 ctx
= maybe_lookup_ctx (stmt
);
10628 lower_omp_ordered (gsi_p
, ctx
);
10630 case GIMPLE_OMP_CRITICAL
:
10631 ctx
= maybe_lookup_ctx (stmt
);
10633 lower_omp_critical (gsi_p
, ctx
);
10635 case GIMPLE_OMP_ATOMIC_LOAD
:
10636 if ((ctx
|| task_shared_vars
)
10637 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
10638 as_a
<gomp_atomic_load
*> (stmt
)),
10639 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10640 gimple_regimplify_operands (stmt
, gsi_p
);
10642 case GIMPLE_OMP_TARGET
:
10643 ctx
= maybe_lookup_ctx (stmt
);
10645 lower_omp_target (gsi_p
, ctx
);
10647 case GIMPLE_OMP_TEAMS
:
10648 ctx
= maybe_lookup_ctx (stmt
);
10650 lower_omp_teams (gsi_p
, ctx
);
10654 call_stmt
= as_a
<gcall
*> (stmt
);
10655 fndecl
= gimple_call_fndecl (call_stmt
);
10657 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10658 switch (DECL_FUNCTION_CODE (fndecl
))
10660 case BUILT_IN_GOMP_BARRIER
:
10664 case BUILT_IN_GOMP_CANCEL
:
10665 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10668 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10669 cctx
= cctx
->outer
;
10670 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
10671 if (!cctx
->cancellable
)
10673 if (DECL_FUNCTION_CODE (fndecl
)
10674 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10676 stmt
= gimple_build_nop ();
10677 gsi_replace (gsi_p
, stmt
, false);
10681 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10683 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10684 gimple_call_set_fndecl (call_stmt
, fndecl
);
10685 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
10688 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
10689 gimple_call_set_lhs (call_stmt
, lhs
);
10690 tree fallthru_label
;
10691 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10693 g
= gimple_build_label (fallthru_label
);
10694 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10695 g
= gimple_build_cond (NE_EXPR
, lhs
,
10696 fold_convert (TREE_TYPE (lhs
),
10697 boolean_false_node
),
10698 cctx
->cancel_label
, fallthru_label
);
10699 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10706 if ((ctx
|| task_shared_vars
)
10707 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10710 /* Just remove clobbers, this should happen only if we have
10711 "privatized" local addressable variables in SIMD regions,
10712 the clobber isn't needed in that case and gimplifying address
10713 of the ARRAY_REF into a pointer and creating MEM_REF based
10714 clobber would create worse code than we get with the clobber
10716 if (gimple_clobber_p (stmt
))
10718 gsi_replace (gsi_p
, gimple_build_nop (), true);
10721 gimple_regimplify_operands (stmt
, gsi_p
);
10728 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10730 location_t saved_location
= input_location
;
10731 gimple_stmt_iterator gsi
;
10732 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10733 lower_omp_1 (&gsi
, ctx
);
10734 /* During gimplification, we have not always invoked fold_stmt
10735 (gimplify.c:maybe_fold_stmt); call it now. */
10736 if (target_nesting_level
)
10737 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10739 input_location
= saved_location
;
10742 /* Main entry point. */
10744 static unsigned int
10745 execute_lower_omp (void)
10751 /* This pass always runs, to provide PROP_gimple_lomp.
10752 But there is nothing to do unless -fopenmp is given. */
10753 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10756 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10757 delete_omp_context
);
10759 body
= gimple_body (current_function_decl
);
10760 scan_omp (&body
, NULL
);
10761 gcc_assert (taskreg_nesting_level
== 0);
10762 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
10763 finish_taskreg_scan (ctx
);
10764 taskreg_contexts
.release ();
10766 if (all_contexts
->root
)
10768 if (task_shared_vars
)
10769 push_gimplify_context ();
10770 lower_omp (&body
, NULL
);
10771 if (task_shared_vars
)
10772 pop_gimplify_context (NULL
);
10777 splay_tree_delete (all_contexts
);
10778 all_contexts
= NULL
;
10780 BITMAP_FREE (task_shared_vars
);
10786 const pass_data pass_data_lower_omp
=
10788 GIMPLE_PASS
, /* type */
10789 "omplower", /* name */
10790 OPTGROUP_NONE
, /* optinfo_flags */
10791 TV_NONE
, /* tv_id */
10792 PROP_gimple_any
, /* properties_required */
10793 PROP_gimple_lomp
, /* properties_provided */
10794 0, /* properties_destroyed */
10795 0, /* todo_flags_start */
10796 0, /* todo_flags_finish */
10799 class pass_lower_omp
: public gimple_opt_pass
10802 pass_lower_omp (gcc::context
*ctxt
)
10803 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10806 /* opt_pass methods: */
10807 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10809 }; // class pass_lower_omp
10811 } // anon namespace
10814 make_pass_lower_omp (gcc::context
*ctxt
)
10816 return new pass_lower_omp (ctxt
);
10819 /* The following is a utility to diagnose OpenMP structured block violations.
10820 It is not part of the "omplower" pass, as that's invoked too late. It
10821 should be invoked by the respective front ends after gimplification. */
10823 static splay_tree all_labels
;
10825 /* Check for mismatched contexts and generate an error if needed. Return
10826 true if an error is detected. */
10829 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10830 gimple branch_ctx
, gimple label_ctx
)
10832 if (label_ctx
== branch_ctx
)
10837 Previously we kept track of the label's entire context in diagnose_sb_[12]
10838 so we could traverse it and issue a correct "exit" or "enter" error
10839 message upon a structured block violation.
10841 We built the context by building a list with tree_cons'ing, but there is
10842 no easy counterpart in gimple tuples. It seems like far too much work
10843 for issuing exit/enter error messages. If someone really misses the
10844 distinct error message... patches welcome.
10848 /* Try to avoid confusing the user by producing and error message
10849 with correct "exit" or "enter" verbiage. We prefer "exit"
10850 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10851 if (branch_ctx
== NULL
)
10857 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10862 label_ctx
= TREE_CHAIN (label_ctx
);
10867 error ("invalid exit from OpenMP structured block");
10869 error ("invalid entry to OpenMP structured block");
10872 bool cilkplus_block
= false;
10876 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10877 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10879 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10880 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10881 cilkplus_block
= true;
10884 /* If it's obvious we have an invalid entry, be specific about the error. */
10885 if (branch_ctx
== NULL
)
10887 if (cilkplus_block
)
10888 error ("invalid entry to Cilk Plus structured block");
10890 error ("invalid entry to OpenMP structured block");
10894 /* Otherwise, be vague and lazy, but efficient. */
10895 if (cilkplus_block
)
10896 error ("invalid branch to/from a Cilk Plus structured block");
10898 error ("invalid branch to/from an OpenMP structured block");
10901 gsi_replace (gsi_p
, gimple_build_nop (), false);
10905 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10906 where each label is found. */
10909 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10910 struct walk_stmt_info
*wi
)
10912 gimple context
= (gimple
) wi
->info
;
10913 gimple inner_context
;
10914 gimple stmt
= gsi_stmt (*gsi_p
);
10916 *handled_ops_p
= true;
10918 switch (gimple_code (stmt
))
10922 case GIMPLE_OMP_PARALLEL
:
10923 case GIMPLE_OMP_TASK
:
10924 case GIMPLE_OMP_SECTIONS
:
10925 case GIMPLE_OMP_SINGLE
:
10926 case GIMPLE_OMP_SECTION
:
10927 case GIMPLE_OMP_MASTER
:
10928 case GIMPLE_OMP_ORDERED
:
10929 case GIMPLE_OMP_CRITICAL
:
10930 case GIMPLE_OMP_TARGET
:
10931 case GIMPLE_OMP_TEAMS
:
10932 case GIMPLE_OMP_TASKGROUP
:
10933 /* The minimal context here is just the current OMP construct. */
10934 inner_context
= stmt
;
10935 wi
->info
= inner_context
;
10936 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10937 wi
->info
= context
;
10940 case GIMPLE_OMP_FOR
:
10941 inner_context
= stmt
;
10942 wi
->info
= inner_context
;
10943 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10945 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10946 diagnose_sb_1
, NULL
, wi
);
10947 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10948 wi
->info
= context
;
10952 splay_tree_insert (all_labels
,
10953 (splay_tree_key
) gimple_label_label (
10954 as_a
<glabel
*> (stmt
)),
10955 (splay_tree_value
) context
);
10965 /* Pass 2: Check each branch and see if its context differs from that of
10966 the destination label's context. */
10969 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10970 struct walk_stmt_info
*wi
)
10972 gimple context
= (gimple
) wi
->info
;
10974 gimple stmt
= gsi_stmt (*gsi_p
);
10976 *handled_ops_p
= true;
10978 switch (gimple_code (stmt
))
10982 case GIMPLE_OMP_PARALLEL
:
10983 case GIMPLE_OMP_TASK
:
10984 case GIMPLE_OMP_SECTIONS
:
10985 case GIMPLE_OMP_SINGLE
:
10986 case GIMPLE_OMP_SECTION
:
10987 case GIMPLE_OMP_MASTER
:
10988 case GIMPLE_OMP_ORDERED
:
10989 case GIMPLE_OMP_CRITICAL
:
10990 case GIMPLE_OMP_TARGET
:
10991 case GIMPLE_OMP_TEAMS
:
10992 case GIMPLE_OMP_TASKGROUP
:
10994 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10995 wi
->info
= context
;
10998 case GIMPLE_OMP_FOR
:
11000 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
11002 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
11003 diagnose_sb_2
, NULL
, wi
);
11004 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
11005 wi
->info
= context
;
11010 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11011 tree lab
= gimple_cond_true_label (cond_stmt
);
11014 n
= splay_tree_lookup (all_labels
,
11015 (splay_tree_key
) lab
);
11016 diagnose_sb_0 (gsi_p
, context
,
11017 n
? (gimple
) n
->value
: NULL
);
11019 lab
= gimple_cond_false_label (cond_stmt
);
11022 n
= splay_tree_lookup (all_labels
,
11023 (splay_tree_key
) lab
);
11024 diagnose_sb_0 (gsi_p
, context
,
11025 n
? (gimple
) n
->value
: NULL
);
11032 tree lab
= gimple_goto_dest (stmt
);
11033 if (TREE_CODE (lab
) != LABEL_DECL
)
11036 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
11037 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
11041 case GIMPLE_SWITCH
:
11043 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
11045 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
11047 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
11048 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
11049 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
11055 case GIMPLE_RETURN
:
11056 diagnose_sb_0 (gsi_p
, context
, NULL
);
11066 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
11069 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
11072 gimple last
= last_stmt (bb
);
11073 enum gimple_code code
= gimple_code (last
);
11074 struct omp_region
*cur_region
= *region
;
11075 bool fallthru
= false;
11079 case GIMPLE_OMP_PARALLEL
:
11080 case GIMPLE_OMP_TASK
:
11081 case GIMPLE_OMP_FOR
:
11082 case GIMPLE_OMP_SINGLE
:
11083 case GIMPLE_OMP_TEAMS
:
11084 case GIMPLE_OMP_MASTER
:
11085 case GIMPLE_OMP_TASKGROUP
:
11086 case GIMPLE_OMP_ORDERED
:
11087 case GIMPLE_OMP_CRITICAL
:
11088 case GIMPLE_OMP_SECTION
:
11089 cur_region
= new_omp_region (bb
, code
, cur_region
);
11093 case GIMPLE_OMP_TARGET
:
11094 cur_region
= new_omp_region (bb
, code
, cur_region
);
11096 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
11097 cur_region
= cur_region
->outer
;
11100 case GIMPLE_OMP_SECTIONS
:
11101 cur_region
= new_omp_region (bb
, code
, cur_region
);
11105 case GIMPLE_OMP_SECTIONS_SWITCH
:
11109 case GIMPLE_OMP_ATOMIC_LOAD
:
11110 case GIMPLE_OMP_ATOMIC_STORE
:
11114 case GIMPLE_OMP_RETURN
:
11115 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
11116 somewhere other than the next block. This will be
11118 cur_region
->exit
= bb
;
11119 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
11120 cur_region
= cur_region
->outer
;
11123 case GIMPLE_OMP_CONTINUE
:
11124 cur_region
->cont
= bb
;
11125 switch (cur_region
->type
)
11127 case GIMPLE_OMP_FOR
:
11128 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
11129 succs edges as abnormal to prevent splitting
11131 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
11132 /* Make the loopback edge. */
11133 make_edge (bb
, single_succ (cur_region
->entry
),
11136 /* Create an edge from GIMPLE_OMP_FOR to exit, which
11137 corresponds to the case that the body of the loop
11138 is not executed at all. */
11139 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
11140 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
11144 case GIMPLE_OMP_SECTIONS
:
11145 /* Wire up the edges into and out of the nested sections. */
11147 basic_block switch_bb
= single_succ (cur_region
->entry
);
11149 struct omp_region
*i
;
11150 for (i
= cur_region
->inner
; i
; i
= i
->next
)
11152 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
11153 make_edge (switch_bb
, i
->entry
, 0);
11154 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
11157 /* Make the loopback edge to the block with
11158 GIMPLE_OMP_SECTIONS_SWITCH. */
11159 make_edge (bb
, switch_bb
, 0);
11161 /* Make the edge from the switch to exit. */
11162 make_edge (switch_bb
, bb
->next_bb
, 0);
11168 gcc_unreachable ();
11173 gcc_unreachable ();
11176 if (*region
!= cur_region
)
11178 *region
= cur_region
;
11180 *region_idx
= cur_region
->entry
->index
;
11188 static unsigned int
11189 diagnose_omp_structured_block_errors (void)
11191 struct walk_stmt_info wi
;
11192 gimple_seq body
= gimple_body (current_function_decl
);
11194 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
11196 memset (&wi
, 0, sizeof (wi
));
11197 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
11199 memset (&wi
, 0, sizeof (wi
));
11200 wi
.want_locations
= true;
11201 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
11203 gimple_set_body (current_function_decl
, body
);
11205 splay_tree_delete (all_labels
);
11213 const pass_data pass_data_diagnose_omp_blocks
=
11215 GIMPLE_PASS
, /* type */
11216 "*diagnose_omp_blocks", /* name */
11217 OPTGROUP_NONE
, /* optinfo_flags */
11218 TV_NONE
, /* tv_id */
11219 PROP_gimple_any
, /* properties_required */
11220 0, /* properties_provided */
11221 0, /* properties_destroyed */
11222 0, /* todo_flags_start */
11223 0, /* todo_flags_finish */
11226 class pass_diagnose_omp_blocks
: public gimple_opt_pass
11229 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11230 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
11233 /* opt_pass methods: */
11234 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
11235 virtual unsigned int execute (function
*)
11237 return diagnose_omp_structured_block_errors ();
11240 }; // class pass_diagnose_omp_blocks
11242 } // anon namespace
11245 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11247 return new pass_diagnose_omp_blocks (ctxt
);
11250 /* SIMD clone supporting code. */
11252 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
11253 of arguments to reserve space for. */
11255 static struct cgraph_simd_clone
*
11256 simd_clone_struct_alloc (int nargs
)
11258 struct cgraph_simd_clone
*clone_info
;
11259 size_t len
= (sizeof (struct cgraph_simd_clone
)
11260 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
11261 clone_info
= (struct cgraph_simd_clone
*)
11262 ggc_internal_cleared_alloc (len
);
11266 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
11269 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
11270 struct cgraph_simd_clone
*from
)
11272 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
11273 + ((from
->nargs
- from
->inbranch
)
11274 * sizeof (struct cgraph_simd_clone_arg
))));
11277 /* Return vector of parameter types of function FNDECL. This uses
11278 TYPE_ARG_TYPES if available, otherwise falls back to types of
11279 DECL_ARGUMENTS types. */
11282 simd_clone_vector_of_formal_parm_types (tree fndecl
)
11284 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
11285 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
11286 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
11289 FOR_EACH_VEC_ELT (args
, i
, arg
)
11290 args
[i
] = TREE_TYPE (args
[i
]);
11294 /* Given a simd function in NODE, extract the simd specific
11295 information from the OMP clauses passed in CLAUSES, and return
11296 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
11297 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
11298 otherwise set to FALSE. */
11300 static struct cgraph_simd_clone
*
11301 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
11302 bool *inbranch_specified
)
11304 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11307 *inbranch_specified
= false;
11309 n
= args
.length ();
11310 if (n
> 0 && args
.last () == void_type_node
)
11313 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
11314 be cloned have a distinctive artificial label in addition to "omp
11318 && lookup_attribute ("cilk simd function",
11319 DECL_ATTRIBUTES (node
->decl
)));
11321 /* Allocate one more than needed just in case this is an in-branch
11322 clone which will require a mask argument. */
11323 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
11324 clone_info
->nargs
= n
;
11325 clone_info
->cilk_elemental
= cilk_clone
;
11332 clauses
= TREE_VALUE (clauses
);
11333 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
11336 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
11338 switch (OMP_CLAUSE_CODE (t
))
11340 case OMP_CLAUSE_INBRANCH
:
11341 clone_info
->inbranch
= 1;
11342 *inbranch_specified
= true;
11344 case OMP_CLAUSE_NOTINBRANCH
:
11345 clone_info
->inbranch
= 0;
11346 *inbranch_specified
= true;
11348 case OMP_CLAUSE_SIMDLEN
:
11349 clone_info
->simdlen
11350 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
11352 case OMP_CLAUSE_LINEAR
:
11354 tree decl
= OMP_CLAUSE_DECL (t
);
11355 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
11356 int argno
= TREE_INT_CST_LOW (decl
);
11357 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
11359 clone_info
->args
[argno
].arg_type
11360 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
11361 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11362 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
11363 && clone_info
->args
[argno
].linear_step
< n
);
11367 if (POINTER_TYPE_P (args
[argno
]))
11368 step
= fold_convert (ssizetype
, step
);
11369 if (!tree_fits_shwi_p (step
))
11371 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11372 "ignoring large linear step");
11376 else if (integer_zerop (step
))
11378 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11379 "ignoring zero linear step");
11385 clone_info
->args
[argno
].arg_type
11386 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
11387 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11392 case OMP_CLAUSE_UNIFORM
:
11394 tree decl
= OMP_CLAUSE_DECL (t
);
11395 int argno
= tree_to_uhwi (decl
);
11396 clone_info
->args
[argno
].arg_type
11397 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
11400 case OMP_CLAUSE_ALIGNED
:
11402 tree decl
= OMP_CLAUSE_DECL (t
);
11403 int argno
= tree_to_uhwi (decl
);
11404 clone_info
->args
[argno
].alignment
11405 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
11416 /* Given a SIMD clone in NODE, calculate the characteristic data
11417 type and return the coresponding type. The characteristic data
11418 type is computed as described in the Intel Vector ABI. */
11421 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
11422 struct cgraph_simd_clone
*clone_info
)
11424 tree type
= integer_type_node
;
11425 tree fndecl
= node
->decl
;
11427 /* a) For non-void function, the characteristic data type is the
11429 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
11430 type
= TREE_TYPE (TREE_TYPE (fndecl
));
11432 /* b) If the function has any non-uniform, non-linear parameters,
11433 then the characteristic data type is the type of the first
11437 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
11438 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
11439 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
11447 /* c) If the characteristic data type determined by a) or b) above
11448 is struct, union, or class type which is pass-by-value (except
11449 for the type that maps to the built-in complex data type), the
11450 characteristic data type is int. */
11451 if (RECORD_OR_UNION_TYPE_P (type
)
11452 && !aggregate_value_p (type
, NULL
)
11453 && TREE_CODE (type
) != COMPLEX_TYPE
)
11454 return integer_type_node
;
11456 /* d) If none of the above three classes is applicable, the
11457 characteristic data type is int. */
11461 /* e) For Intel Xeon Phi native and offload compilation, if the
11462 resulting characteristic data type is 8-bit or 16-bit integer
11463 data type, the characteristic data type is int. */
11464 /* Well, we don't handle Xeon Phi yet. */
11468 simd_clone_mangle (struct cgraph_node
*node
,
11469 struct cgraph_simd_clone
*clone_info
)
11471 char vecsize_mangle
= clone_info
->vecsize_mangle
;
11472 char mask
= clone_info
->inbranch
? 'M' : 'N';
11473 unsigned int simdlen
= clone_info
->simdlen
;
11477 gcc_assert (vecsize_mangle
&& simdlen
);
11479 pp_string (&pp
, "_ZGV");
11480 pp_character (&pp
, vecsize_mangle
);
11481 pp_character (&pp
, mask
);
11482 pp_decimal_int (&pp
, simdlen
);
11484 for (n
= 0; n
< clone_info
->nargs
; ++n
)
11486 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
11488 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
11489 pp_character (&pp
, 'u');
11490 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11492 gcc_assert (arg
.linear_step
!= 0);
11493 pp_character (&pp
, 'l');
11494 if (arg
.linear_step
> 1)
11495 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11496 else if (arg
.linear_step
< 0)
11498 pp_character (&pp
, 'n');
11499 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
11503 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11505 pp_character (&pp
, 's');
11506 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11509 pp_character (&pp
, 'v');
11512 pp_character (&pp
, 'a');
11513 pp_decimal_int (&pp
, arg
.alignment
);
11517 pp_underscore (&pp
);
11519 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11520 const char *str
= pp_formatted_text (&pp
);
11522 /* If there already is a SIMD clone with the same mangled name, don't
11523 add another one. This can happen e.g. for
11524 #pragma omp declare simd
11525 #pragma omp declare simd simdlen(8)
11526 int foo (int, int);
11527 if the simdlen is assumed to be 8 for the first one, etc. */
11528 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11529 clone
= clone
->simdclone
->next_clone
)
11530 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11534 return get_identifier (str
);
11537 /* Create a simd clone of OLD_NODE and return it. */
11539 static struct cgraph_node
*
11540 simd_clone_create (struct cgraph_node
*old_node
)
11542 struct cgraph_node
*new_node
;
11543 if (old_node
->definition
)
11545 if (!old_node
->has_gimple_body_p ())
11547 old_node
->get_body ();
11548 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
11554 tree old_decl
= old_node
->decl
;
11555 tree new_decl
= copy_node (old_node
->decl
);
11556 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11557 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11558 SET_DECL_RTL (new_decl
, NULL
);
11559 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11560 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11561 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
11562 symtab
->call_cgraph_insertion_hooks (new_node
);
11564 if (new_node
== NULL
)
11567 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11569 /* The function cgraph_function_versioning () will force the new
11570 symbol local. Undo this, and inherit external visability from
11572 new_node
->local
.local
= old_node
->local
.local
;
11573 new_node
->externally_visible
= old_node
->externally_visible
;
11578 /* Adjust the return type of the given function to its appropriate
11579 vector counterpart. Returns a simd array to be used throughout the
11580 function as a return value. */
11583 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11585 tree fndecl
= node
->decl
;
11586 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11587 unsigned int veclen
;
11590 /* Adjust the function return type. */
11591 if (orig_rettype
== void_type_node
)
11593 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11594 t
= TREE_TYPE (TREE_TYPE (fndecl
));
11595 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
11596 veclen
= node
->simdclone
->vecsize_int
;
11598 veclen
= node
->simdclone
->vecsize_float
;
11599 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
11600 if (veclen
> node
->simdclone
->simdlen
)
11601 veclen
= node
->simdclone
->simdlen
;
11602 if (POINTER_TYPE_P (t
))
11603 t
= pointer_sized_int_node
;
11604 if (veclen
== node
->simdclone
->simdlen
)
11605 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
11608 t
= build_vector_type (t
, veclen
);
11609 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11611 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11612 if (!node
->definition
)
11615 t
= DECL_RESULT (fndecl
);
11616 /* Adjust the DECL_RESULT. */
11617 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11618 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11621 tree atype
= build_array_type_nelts (orig_rettype
,
11622 node
->simdclone
->simdlen
);
11623 if (veclen
!= node
->simdclone
->simdlen
)
11624 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11626 /* Set up a SIMD array to use as the return value. */
11627 tree retval
= create_tmp_var_raw (atype
, "retval");
11628 gimple_add_tmp_var (retval
);
11632 /* Each vector argument has a corresponding array to be used locally
11633 as part of the eventual loop. Create such temporary array and
11636 PREFIX is the prefix to be used for the temporary.
11638 TYPE is the inner element type.
11640 SIMDLEN is the number of elements. */
11643 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11645 tree atype
= build_array_type_nelts (type
, simdlen
);
11646 tree avar
= create_tmp_var_raw (atype
, prefix
);
11647 gimple_add_tmp_var (avar
);
11651 /* Modify the function argument types to their corresponding vector
11652 counterparts if appropriate. Also, create one array for each simd
11653 argument to be used locally when using the function arguments as
11656 NODE is the function whose arguments are to be adjusted.
11658 Returns an adjustment vector that will be filled describing how the
11659 argument types will be adjusted. */
11661 static ipa_parm_adjustment_vec
11662 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11665 ipa_parm_adjustment_vec adjustments
;
11667 if (node
->definition
)
11668 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11670 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11671 adjustments
.create (args
.length ());
11672 unsigned i
, j
, veclen
;
11673 struct ipa_parm_adjustment adj
;
11674 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11676 memset (&adj
, 0, sizeof (adj
));
11677 tree parm
= args
[i
];
11678 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11679 adj
.base_index
= i
;
11682 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11683 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11685 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11687 /* No adjustment necessary for scalar arguments. */
11688 adj
.op
= IPA_PARM_OP_COPY
;
11692 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11693 veclen
= node
->simdclone
->vecsize_int
;
11695 veclen
= node
->simdclone
->vecsize_float
;
11696 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11697 if (veclen
> node
->simdclone
->simdlen
)
11698 veclen
= node
->simdclone
->simdlen
;
11699 adj
.arg_prefix
= "simd";
11700 if (POINTER_TYPE_P (parm_type
))
11701 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
11703 adj
.type
= build_vector_type (parm_type
, veclen
);
11704 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11705 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11707 adjustments
.safe_push (adj
);
11710 memset (&adj
, 0, sizeof (adj
));
11711 adj
.op
= IPA_PARM_OP_NEW
;
11712 adj
.arg_prefix
= "simd";
11713 adj
.base_index
= i
;
11714 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11718 if (node
->definition
)
11719 node
->simdclone
->args
[i
].simd_array
11720 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11721 parm_type
, node
->simdclone
->simdlen
);
11723 adjustments
.safe_push (adj
);
11726 if (node
->simdclone
->inbranch
)
11729 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11732 memset (&adj
, 0, sizeof (adj
));
11733 adj
.op
= IPA_PARM_OP_NEW
;
11734 adj
.arg_prefix
= "mask";
11736 adj
.base_index
= i
;
11737 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11738 veclen
= node
->simdclone
->vecsize_int
;
11740 veclen
= node
->simdclone
->vecsize_float
;
11741 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11742 if (veclen
> node
->simdclone
->simdlen
)
11743 veclen
= node
->simdclone
->simdlen
;
11744 if (POINTER_TYPE_P (base_type
))
11745 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
11747 adj
.type
= build_vector_type (base_type
, veclen
);
11748 adjustments
.safe_push (adj
);
11750 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11751 adjustments
.safe_push (adj
);
11753 /* We have previously allocated one extra entry for the mask. Use
11755 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11757 if (node
->definition
)
11759 sc
->args
[i
].orig_arg
11760 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11761 sc
->args
[i
].simd_array
11762 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11764 sc
->args
[i
].orig_type
= base_type
;
11765 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11768 if (node
->definition
)
11769 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11772 tree new_arg_types
= NULL_TREE
, new_reversed
;
11773 bool last_parm_void
= false;
11774 if (args
.length () > 0 && args
.last () == void_type_node
)
11775 last_parm_void
= true;
11777 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11778 j
= adjustments
.length ();
11779 for (i
= 0; i
< j
; i
++)
11781 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11783 if (adj
->op
== IPA_PARM_OP_COPY
)
11784 ptype
= args
[adj
->base_index
];
11787 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11789 new_reversed
= nreverse (new_arg_types
);
11790 if (last_parm_void
)
11793 TREE_CHAIN (new_arg_types
) = void_list_node
;
11795 new_reversed
= void_list_node
;
11798 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11799 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11800 TREE_TYPE (node
->decl
) = new_type
;
11802 adjustments
.release ();
11805 return adjustments
;
11808 /* Initialize and copy the function arguments in NODE to their
11809 corresponding local simd arrays. Returns a fresh gimple_seq with
11810 the instruction sequence generated. */
11813 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11814 ipa_parm_adjustment_vec adjustments
)
11816 gimple_seq seq
= NULL
;
11817 unsigned i
= 0, j
= 0, k
;
11819 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11821 arg
= DECL_CHAIN (arg
), i
++, j
++)
11823 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11826 node
->simdclone
->args
[i
].vector_arg
= arg
;
11828 tree array
= node
->simdclone
->args
[i
].simd_array
;
11829 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11831 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11832 tree ptr
= build_fold_addr_expr (array
);
11833 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11834 build_int_cst (ptype
, 0));
11835 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11836 gimplify_and_add (t
, &seq
);
11840 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11841 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11842 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11844 tree ptr
= build_fold_addr_expr (array
);
11848 arg
= DECL_CHAIN (arg
);
11852 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11853 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11854 build_int_cst (ptype
, k
* elemsize
));
11855 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11856 gimplify_and_add (t
, &seq
);
11863 /* Callback info for ipa_simd_modify_stmt_ops below. */
11865 struct modify_stmt_info
{
11866 ipa_parm_adjustment_vec adjustments
;
11868 /* True if the parent statement was modified by
11869 ipa_simd_modify_stmt_ops. */
11873 /* Callback for walk_gimple_op.
11875 Adjust operands from a given statement as specified in the
11876 adjustments vector in the callback data. */
11879 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11881 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11882 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11883 tree
*orig_tp
= tp
;
11884 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11885 tp
= &TREE_OPERAND (*tp
, 0);
11886 struct ipa_parm_adjustment
*cand
= NULL
;
11887 if (TREE_CODE (*tp
) == PARM_DECL
)
11888 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11892 *walk_subtrees
= 0;
11895 tree repl
= NULL_TREE
;
11897 repl
= unshare_expr (cand
->new_decl
);
11902 *walk_subtrees
= 0;
11903 bool modified
= info
->modified
;
11904 info
->modified
= false;
11905 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11906 if (!info
->modified
)
11908 info
->modified
= modified
;
11911 info
->modified
= modified
;
11920 repl
= build_fold_addr_expr (repl
);
11922 if (is_gimple_debug (info
->stmt
))
11924 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
11925 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
11926 DECL_ARTIFICIAL (vexpr
) = 1;
11927 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
11928 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
11933 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
11934 repl
= gimple_assign_lhs (stmt
);
11936 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11937 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11940 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11942 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11948 info
->modified
= true;
11952 /* Traverse the function body and perform all modifications as
11953 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11954 modified such that the replacement/reduction value will now be an
11955 offset into the corresponding simd_array.
11957 This function will replace all function argument uses with their
11958 corresponding simd array elements, and ajust the return values
11962 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11963 ipa_parm_adjustment_vec adjustments
,
11964 tree retval_array
, tree iter
)
11967 unsigned int i
, j
, l
;
11969 /* Re-use the adjustments array, but this time use it to replace
11970 every function argument use to an offset into the corresponding
11972 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11974 if (!node
->simdclone
->args
[i
].vector_arg
)
11977 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11978 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11979 adjustments
[j
].new_decl
11980 = build4 (ARRAY_REF
,
11982 node
->simdclone
->args
[i
].simd_array
,
11984 NULL_TREE
, NULL_TREE
);
11985 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11986 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11987 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11990 l
= adjustments
.length ();
11991 for (i
= 1; i
< num_ssa_names
; i
++)
11993 tree name
= ssa_name (i
);
11995 && SSA_NAME_VAR (name
)
11996 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11998 for (j
= 0; j
< l
; j
++)
11999 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
12000 && adjustments
[j
].new_decl
)
12003 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
12006 = copy_var_decl (adjustments
[j
].base
,
12007 DECL_NAME (adjustments
[j
].base
),
12008 TREE_TYPE (adjustments
[j
].base
));
12009 adjustments
[j
].new_ssa_base
= base_var
;
12012 base_var
= adjustments
[j
].new_ssa_base
;
12013 if (SSA_NAME_IS_DEFAULT_DEF (name
))
12015 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
12016 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
12017 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
12018 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
12019 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
12020 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
12021 gimple stmt
= gimple_build_assign (name
, new_decl
);
12022 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
12025 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
12030 struct modify_stmt_info info
;
12031 info
.adjustments
= adjustments
;
12033 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
12035 gimple_stmt_iterator gsi
;
12037 gsi
= gsi_start_bb (bb
);
12038 while (!gsi_end_p (gsi
))
12040 gimple stmt
= gsi_stmt (gsi
);
12042 struct walk_stmt_info wi
;
12044 memset (&wi
, 0, sizeof (wi
));
12045 info
.modified
= false;
12047 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
12049 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
12051 tree retval
= gimple_return_retval (return_stmt
);
12054 gsi_remove (&gsi
, true);
12058 /* Replace `return foo' with `retval_array[iter] = foo'. */
12059 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
12060 retval_array
, iter
, NULL
, NULL
);
12061 stmt
= gimple_build_assign (ref
, retval
);
12062 gsi_replace (&gsi
, stmt
, true);
12063 info
.modified
= true;
12068 update_stmt (stmt
);
12069 if (maybe_clean_eh_stmt (stmt
))
12070 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
12077 /* Adjust the argument types in NODE to their appropriate vector
12081 simd_clone_adjust (struct cgraph_node
*node
)
12083 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
12085 targetm
.simd_clone
.adjust (node
);
12087 tree retval
= simd_clone_adjust_return_type (node
);
12088 ipa_parm_adjustment_vec adjustments
12089 = simd_clone_adjust_argument_types (node
);
12091 push_gimplify_context ();
12093 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
12095 /* Adjust all uses of vector arguments accordingly. Adjust all
12096 return values accordingly. */
12097 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
12098 tree iter1
= make_ssa_name (iter
);
12099 tree iter2
= make_ssa_name (iter
);
12100 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
12102 /* Initialize the iteration variable. */
12103 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
12104 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
12105 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
12106 /* Insert the SIMD array and iv initialization at function
12108 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
12110 pop_gimplify_context (NULL
);
12112 /* Create a new BB right before the original exit BB, to hold the
12113 iteration increment and the condition/branch. */
12114 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
12115 basic_block incr_bb
= create_empty_bb (orig_exit
);
12116 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
12117 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
12118 flag. Set it now to be a FALLTHRU_EDGE. */
12119 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
12120 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
12121 for (unsigned i
= 0;
12122 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
12124 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
12125 redirect_edge_succ (e
, incr_bb
);
12127 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
12128 e
->probability
= REG_BR_PROB_BASE
;
12129 gsi
= gsi_last_bb (incr_bb
);
12130 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
12131 build_int_cst (unsigned_type_node
, 1));
12132 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12134 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
12135 struct loop
*loop
= alloc_loop ();
12136 cfun
->has_force_vectorize_loops
= true;
12137 loop
->safelen
= node
->simdclone
->simdlen
;
12138 loop
->force_vectorize
= true;
12139 loop
->header
= body_bb
;
12141 /* Branch around the body if the mask applies. */
12142 if (node
->simdclone
->inbranch
)
12144 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
12146 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
12147 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
12148 tree aref
= build4 (ARRAY_REF
,
12149 TREE_TYPE (TREE_TYPE (mask_array
)),
12152 g
= gimple_build_assign (mask
, aref
);
12153 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12154 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
12155 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
12157 aref
= build1 (VIEW_CONVERT_EXPR
,
12158 build_nonstandard_integer_type (bitsize
, 0), mask
);
12159 mask
= make_ssa_name (TREE_TYPE (aref
));
12160 g
= gimple_build_assign (mask
, aref
);
12161 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12164 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
12166 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12167 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
12168 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
12171 /* Generate the condition. */
12172 g
= gimple_build_cond (LT_EXPR
,
12174 build_int_cst (unsigned_type_node
,
12175 node
->simdclone
->simdlen
),
12177 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12178 e
= split_block (incr_bb
, gsi_stmt (gsi
));
12179 basic_block latch_bb
= e
->dest
;
12180 basic_block new_exit_bb
;
12181 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
12182 loop
->latch
= latch_bb
;
12184 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
12186 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
12187 /* The successor of incr_bb is already pointing to latch_bb; just
12189 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
12190 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
12192 gphi
*phi
= create_phi_node (iter1
, body_bb
);
12193 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
12194 edge latch_edge
= single_succ_edge (latch_bb
);
12195 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
12197 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12199 /* Generate the new return. */
12200 gsi
= gsi_last_bb (new_exit_bb
);
12202 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
12203 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
12204 retval
= TREE_OPERAND (retval
, 0);
12207 retval
= build1 (VIEW_CONVERT_EXPR
,
12208 TREE_TYPE (TREE_TYPE (node
->decl
)),
12210 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
12211 false, GSI_CONTINUE_LINKING
);
12213 g
= gimple_build_return (retval
);
12214 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12216 /* Handle aligned clauses by replacing default defs of the aligned
12217 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
12218 lhs. Handle linear by adding PHIs. */
12219 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
12220 if (node
->simdclone
->args
[i
].alignment
12221 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
12222 && (node
->simdclone
->args
[i
].alignment
12223 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
12224 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
12227 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
12228 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12229 tree def
= ssa_default_def (cfun
, orig_arg
);
12230 if (def
&& !has_zero_uses (def
))
12232 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
12233 gimple_seq seq
= NULL
;
12234 bool need_cvt
= false;
12236 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
12238 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
12241 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
12242 gimple_call_set_lhs (g
, t
);
12243 gimple_seq_add_stmt_without_update (&seq
, g
);
12246 t
= make_ssa_name (orig_arg
);
12247 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
12248 gimple_seq_add_stmt_without_update (&seq
, g
);
12250 gsi_insert_seq_on_edge_immediate
12251 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
12253 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
12254 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
12256 node
->create_edge (cgraph_node::get_create (fn
),
12257 call
, entry_bb
->count
, freq
);
12259 imm_use_iterator iter
;
12260 use_operand_p use_p
;
12262 tree repl
= gimple_get_lhs (g
);
12263 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12264 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
12267 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12268 SET_USE (use_p
, repl
);
12271 else if (node
->simdclone
->args
[i
].arg_type
12272 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12274 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12275 tree def
= ssa_default_def (cfun
, orig_arg
);
12276 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12277 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
12278 if (def
&& !has_zero_uses (def
))
12280 iter1
= make_ssa_name (orig_arg
);
12281 iter2
= make_ssa_name (orig_arg
);
12282 phi
= create_phi_node (iter1
, body_bb
);
12283 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
12284 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12285 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12286 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
12287 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12288 ? TREE_TYPE (orig_arg
) : sizetype
;
12290 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
12291 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
12292 gsi
= gsi_last_bb (incr_bb
);
12293 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12295 imm_use_iterator iter
;
12296 use_operand_p use_p
;
12298 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12299 if (use_stmt
== phi
)
12302 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12303 SET_USE (use_p
, iter1
);
12307 calculate_dominance_info (CDI_DOMINATORS
);
12308 add_loop (loop
, loop
->header
->loop_father
);
12309 update_ssa (TODO_update_ssa
);
12314 /* If the function in NODE is tagged as an elemental SIMD function,
12315 create the appropriate SIMD clones. */
12318 expand_simd_clones (struct cgraph_node
*node
)
12320 tree attr
= lookup_attribute ("omp declare simd",
12321 DECL_ATTRIBUTES (node
->decl
));
12322 if (attr
== NULL_TREE
12323 || node
->global
.inlined_to
12324 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
12328 #pragma omp declare simd
12330 in C, there we don't know the argument types at all. */
12331 if (!node
->definition
12332 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
12337 /* Start with parsing the "omp declare simd" attribute(s). */
12338 bool inbranch_clause_specified
;
12339 struct cgraph_simd_clone
*clone_info
12340 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
12341 &inbranch_clause_specified
);
12342 if (clone_info
== NULL
)
12345 int orig_simdlen
= clone_info
->simdlen
;
12346 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
12347 /* The target can return 0 (no simd clones should be created),
12348 1 (just one ISA of simd clones should be created) or higher
12349 count of ISA variants. In that case, clone_info is initialized
12350 for the first ISA variant. */
12352 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
12357 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
12358 also create one inbranch and one !inbranch clone of it. */
12359 for (int i
= 0; i
< count
* 2; i
++)
12361 struct cgraph_simd_clone
*clone
= clone_info
;
12362 if (inbranch_clause_specified
&& (i
& 1) != 0)
12367 clone
= simd_clone_struct_alloc (clone_info
->nargs
12369 simd_clone_struct_copy (clone
, clone_info
);
12370 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
12371 and simd_clone_adjust_argument_types did to the first
12373 clone
->nargs
-= clone_info
->inbranch
;
12374 clone
->simdlen
= orig_simdlen
;
12375 /* And call the target hook again to get the right ISA. */
12376 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
12380 clone
->inbranch
= 1;
12383 /* simd_clone_mangle might fail if such a clone has been created
12385 tree id
= simd_clone_mangle (node
, clone
);
12386 if (id
== NULL_TREE
)
12389 /* Only when we are sure we want to create the clone actually
12390 clone the function (or definitions) or create another
12391 extern FUNCTION_DECL (for prototypes without definitions). */
12392 struct cgraph_node
*n
= simd_clone_create (node
);
12396 n
->simdclone
= clone
;
12397 clone
->origin
= node
;
12398 clone
->next_clone
= NULL
;
12399 if (node
->simd_clones
== NULL
)
12401 clone
->prev_clone
= n
;
12402 node
->simd_clones
= n
;
12406 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
12407 clone
->prev_clone
->simdclone
->next_clone
= n
;
12408 node
->simd_clones
->simdclone
->prev_clone
= n
;
12410 symtab
->change_decl_assembler_name (n
->decl
, id
);
12411 /* And finally adjust the return type, parameters and for
12412 definitions also function body. */
12413 if (node
->definition
)
12414 simd_clone_adjust (n
);
12417 simd_clone_adjust_return_type (n
);
12418 simd_clone_adjust_argument_types (n
);
12422 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
12425 /* Entry point for IPA simd clone creation pass. */
12427 static unsigned int
12428 ipa_omp_simd_clone (void)
12430 struct cgraph_node
*node
;
12431 FOR_EACH_FUNCTION (node
)
12432 expand_simd_clones (node
);
12438 const pass_data pass_data_omp_simd_clone
=
12440 SIMPLE_IPA_PASS
, /* type */
12441 "simdclone", /* name */
12442 OPTGROUP_NONE
, /* optinfo_flags */
12443 TV_NONE
, /* tv_id */
12444 ( PROP_ssa
| PROP_cfg
), /* properties_required */
12445 0, /* properties_provided */
12446 0, /* properties_destroyed */
12447 0, /* todo_flags_start */
12448 0, /* todo_flags_finish */
12451 class pass_omp_simd_clone
: public simple_ipa_opt_pass
12454 pass_omp_simd_clone(gcc::context
*ctxt
)
12455 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
12458 /* opt_pass methods: */
12459 virtual bool gate (function
*);
12460 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
12464 pass_omp_simd_clone::gate (function
*)
12466 return ((flag_openmp
|| flag_openmp_simd
12468 || (in_lto_p
&& !flag_wpa
))
12469 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
12472 } // anon namespace
12474 simple_ipa_opt_pass
*
12475 make_pass_omp_simd_clone (gcc::context
*ctxt
)
12477 return new pass_omp_simd_clone (ctxt
);
12480 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
12481 adds their addresses and sizes to constructor-vector V_CTOR. */
12483 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
12484 vec
<constructor_elt
, va_gc
> *v_ctor
)
12486 unsigned len
= vec_safe_length (v_decls
);
12487 for (unsigned i
= 0; i
< len
; i
++)
12489 tree it
= (*v_decls
)[i
];
12490 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
12492 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
12494 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
12495 fold_convert (const_ptr_type_node
,
12496 DECL_SIZE_UNIT (it
)));
12500 /* Create new symbols containing (address, size) pairs for global variables,
12501 marked with "omp declare target" attribute, as well as addresses for the
12502 functions, which are outlined target regions. */
12504 omp_finish_file (void)
12506 unsigned num_funcs
= vec_safe_length (offload_funcs
);
12507 unsigned num_vars
= vec_safe_length (offload_vars
);
12509 if (num_funcs
== 0 && num_vars
== 0)
12512 if (targetm_common
.have_named_sections
)
12514 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
12515 vec_alloc (v_f
, num_funcs
);
12516 vec_alloc (v_v
, num_vars
* 2);
12518 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
12519 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
12521 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
12523 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
12525 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
12526 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
12527 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
12528 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
12529 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
12530 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
12531 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
12532 get_identifier (".offload_func_table"),
12534 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
12535 get_identifier (".offload_var_table"),
12537 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
12538 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
12539 otherwise a joint table in a binary will contain padding between
12540 tables from multiple object files. */
12541 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
12542 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
12543 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
12544 DECL_INITIAL (funcs_decl
) = ctor_f
;
12545 DECL_INITIAL (vars_decl
) = ctor_v
;
12546 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
12547 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
12549 varpool_node::finalize_decl (vars_decl
);
12550 varpool_node::finalize_decl (funcs_decl
);
12554 for (unsigned i
= 0; i
< num_funcs
; i
++)
12556 tree it
= (*offload_funcs
)[i
];
12557 targetm
.record_offload_symbol (it
);
12559 for (unsigned i
= 0; i
< num_vars
; i
++)
12561 tree it
= (*offload_vars
)[i
];
12562 targetm
.record_offload_symbol (it
);
12567 #include "gt-omp-low.h"