1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
208 static void scan_omp (gimple_seq
*, omp_context
*);
209 static tree
scan_omp_1_op (tree
*, int *, void *);
211 #define WALK_SUBSTMTS \
215 case GIMPLE_EH_FILTER: \
216 case GIMPLE_TRANSACTION: \
217 /* The sub-statements for these should be walked. */ \
218 *handled_ops_p = false; \
221 /* Convenience function for calling scan_omp_1_op on tree operands. */
224 scan_omp_op (tree
*tp
, omp_context
*ctx
)
226 struct walk_stmt_info wi
;
228 memset (&wi
, 0, sizeof (wi
));
230 wi
.want_locations
= true;
232 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
235 static void lower_omp (gimple_seq
*, omp_context
*);
236 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
237 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
239 /* Find an OpenMP clause of type KIND within CLAUSES. */
242 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
244 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
245 if (OMP_CLAUSE_CODE (clauses
) == kind
)
251 /* Return true if CTX is for an omp parallel. */
254 is_parallel_ctx (omp_context
*ctx
)
256 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
260 /* Return true if CTX is for an omp task. */
263 is_task_ctx (omp_context
*ctx
)
265 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
269 /* Return true if CTX is for an omp parallel or omp task. */
272 is_taskreg_ctx (omp_context
*ctx
)
274 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
275 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
279 /* Return true if REGION is a combined parallel+workshare region. */
282 is_combined_parallel (struct omp_region
*region
)
284 return region
->is_combined_parallel
;
288 /* Extract the header elements of parallel loop FOR_STMT and store
292 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
293 struct omp_for_data_loop
*loops
)
295 tree t
, var
, *collapse_iter
, *collapse_count
;
296 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
297 struct omp_for_data_loop
*loop
;
299 struct omp_for_data_loop dummy_loop
;
300 location_t loc
= gimple_location (for_stmt
);
301 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_KIND_SIMD
;
302 bool distribute
= gimple_omp_for_kind (for_stmt
)
303 == GF_OMP_FOR_KIND_DISTRIBUTE
;
305 fd
->for_stmt
= for_stmt
;
307 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
308 if (fd
->collapse
> 1)
311 fd
->loops
= &fd
->loop
;
313 fd
->have_nowait
= distribute
|| simd
;
314 fd
->have_ordered
= false;
315 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
316 fd
->chunk_size
= NULL_TREE
;
317 collapse_iter
= NULL
;
318 collapse_count
= NULL
;
320 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
321 switch (OMP_CLAUSE_CODE (t
))
323 case OMP_CLAUSE_NOWAIT
:
324 fd
->have_nowait
= true;
326 case OMP_CLAUSE_ORDERED
:
327 fd
->have_ordered
= true;
329 case OMP_CLAUSE_SCHEDULE
:
330 gcc_assert (!distribute
);
331 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
332 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
334 case OMP_CLAUSE_DIST_SCHEDULE
:
335 gcc_assert (distribute
);
336 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
338 case OMP_CLAUSE_COLLAPSE
:
339 if (fd
->collapse
> 1)
341 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
342 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
348 /* FIXME: for now map schedule(auto) to schedule(static).
349 There should be analysis to determine whether all iterations
350 are approximately the same amount of work (then schedule(static)
351 is best) or if it varies (then schedule(dynamic,N) is better). */
352 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
354 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
355 gcc_assert (fd
->chunk_size
== NULL
);
357 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
358 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
359 gcc_assert (fd
->chunk_size
== NULL
);
360 else if (fd
->chunk_size
== NULL
)
362 /* We only need to compute a default chunk size for ordered
363 static loops and dynamic loops. */
364 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
366 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
367 ? integer_zero_node
: integer_one_node
;
370 for (i
= 0; i
< fd
->collapse
; i
++)
372 if (fd
->collapse
== 1)
374 else if (loops
!= NULL
)
379 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
380 gcc_assert (SSA_VAR_P (loop
->v
));
381 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
382 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
383 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
384 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
386 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
387 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
388 switch (loop
->cond_code
)
394 gcc_assert (gimple_omp_for_kind (for_stmt
)
395 == GF_OMP_FOR_KIND_CILKSIMD
);
398 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
399 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
401 loop
->n2
= fold_build2_loc (loc
,
402 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
403 build_int_cst (TREE_TYPE (loop
->n2
), 1));
404 loop
->cond_code
= LT_EXPR
;
407 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
408 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
410 loop
->n2
= fold_build2_loc (loc
,
411 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
412 build_int_cst (TREE_TYPE (loop
->n2
), 1));
413 loop
->cond_code
= GT_EXPR
;
419 t
= gimple_omp_for_incr (for_stmt
, i
);
420 gcc_assert (TREE_OPERAND (t
, 0) == var
);
421 switch (TREE_CODE (t
))
424 loop
->step
= TREE_OPERAND (t
, 1);
426 case POINTER_PLUS_EXPR
:
427 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
430 loop
->step
= TREE_OPERAND (t
, 1);
431 loop
->step
= fold_build1_loc (loc
,
432 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
440 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
441 && !fd
->have_ordered
))
443 if (fd
->collapse
== 1)
444 iter_type
= TREE_TYPE (loop
->v
);
446 || TYPE_PRECISION (iter_type
)
447 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
449 = build_nonstandard_integer_type
450 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
452 else if (iter_type
!= long_long_unsigned_type_node
)
454 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
455 iter_type
= long_long_unsigned_type_node
;
456 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
457 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
458 >= TYPE_PRECISION (iter_type
))
462 if (loop
->cond_code
== LT_EXPR
)
463 n
= fold_build2_loc (loc
,
464 PLUS_EXPR
, TREE_TYPE (loop
->v
),
465 loop
->n2
, loop
->step
);
468 if (TREE_CODE (n
) != INTEGER_CST
469 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
470 iter_type
= long_long_unsigned_type_node
;
472 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
473 > TYPE_PRECISION (iter_type
))
477 if (loop
->cond_code
== LT_EXPR
)
480 n2
= fold_build2_loc (loc
,
481 PLUS_EXPR
, TREE_TYPE (loop
->v
),
482 loop
->n2
, loop
->step
);
486 n1
= fold_build2_loc (loc
,
487 MINUS_EXPR
, TREE_TYPE (loop
->v
),
488 loop
->n2
, loop
->step
);
491 if (TREE_CODE (n1
) != INTEGER_CST
492 || TREE_CODE (n2
) != INTEGER_CST
493 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
494 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
495 iter_type
= long_long_unsigned_type_node
;
499 if (collapse_count
&& *collapse_count
== NULL
)
501 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
502 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
503 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
504 if (t
&& integer_zerop (t
))
505 count
= build_zero_cst (long_long_unsigned_type_node
);
506 else if ((i
== 0 || count
!= NULL_TREE
)
507 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
508 && TREE_CONSTANT (loop
->n1
)
509 && TREE_CONSTANT (loop
->n2
)
510 && TREE_CODE (loop
->step
) == INTEGER_CST
)
512 tree itype
= TREE_TYPE (loop
->v
);
514 if (POINTER_TYPE_P (itype
))
515 itype
= signed_type_for (itype
);
516 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
517 t
= fold_build2_loc (loc
,
519 fold_convert_loc (loc
, itype
, loop
->step
), t
);
520 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
521 fold_convert_loc (loc
, itype
, loop
->n2
));
522 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
523 fold_convert_loc (loc
, itype
, loop
->n1
));
524 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
525 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
526 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
527 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
528 fold_convert_loc (loc
, itype
,
531 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
532 fold_convert_loc (loc
, itype
, loop
->step
));
533 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
534 if (count
!= NULL_TREE
)
535 count
= fold_build2_loc (loc
,
536 MULT_EXPR
, long_long_unsigned_type_node
,
540 if (TREE_CODE (count
) != INTEGER_CST
)
543 else if (count
&& !integer_zerop (count
))
550 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
551 || fd
->have_ordered
))
553 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
554 iter_type
= long_long_unsigned_type_node
;
556 iter_type
= long_integer_type_node
;
558 else if (collapse_iter
&& *collapse_iter
!= NULL
)
559 iter_type
= TREE_TYPE (*collapse_iter
);
560 fd
->iter_type
= iter_type
;
561 if (collapse_iter
&& *collapse_iter
== NULL
)
562 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
563 if (collapse_count
&& *collapse_count
== NULL
)
566 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
568 *collapse_count
= create_tmp_var (iter_type
, ".count");
571 if (fd
->collapse
> 1)
573 fd
->loop
.v
= *collapse_iter
;
574 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
575 fd
->loop
.n2
= *collapse_count
;
576 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
577 fd
->loop
.cond_code
= LT_EXPR
;
582 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
583 is the immediate dominator of PAR_ENTRY_BB, return true if there
584 are no data dependencies that would prevent expanding the parallel
585 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
587 When expanding a combined parallel+workshare region, the call to
588 the child function may need additional arguments in the case of
589 GIMPLE_OMP_FOR regions. In some cases, these arguments are
590 computed out of variables passed in from the parent to the child
591 via 'struct .omp_data_s'. For instance:
593 #pragma omp parallel for schedule (guided, i * 4)
598 # BLOCK 2 (PAR_ENTRY_BB)
600 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
602 # BLOCK 3 (WS_ENTRY_BB)
603 .omp_data_i = &.omp_data_o;
604 D.1667 = .omp_data_i->i;
606 #pragma omp for schedule (guided, D.1598)
608 When we outline the parallel region, the call to the child function
609 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
610 that value is computed *after* the call site. So, in principle we
611 cannot do the transformation.
613 To see whether the code in WS_ENTRY_BB blocks the combined
614 parallel+workshare call, we collect all the variables used in the
615 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
616 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
619 FIXME. If we had the SSA form built at this point, we could merely
620 hoist the code in block 3 into block 2 and be done with it. But at
621 this point we don't have dataflow information and though we could
622 hack something up here, it is really not worth the aggravation. */
625 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
627 struct omp_for_data fd
;
628 gimple ws_stmt
= last_stmt (ws_entry_bb
);
630 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
633 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
635 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
637 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
639 if (fd
.iter_type
!= long_integer_type_node
)
642 /* FIXME. We give up too easily here. If any of these arguments
643 are not constants, they will likely involve variables that have
644 been mapped into fields of .omp_data_s for sharing with the child
645 function. With appropriate data flow, it would be possible to
647 if (!is_gimple_min_invariant (fd
.loop
.n1
)
648 || !is_gimple_min_invariant (fd
.loop
.n2
)
649 || !is_gimple_min_invariant (fd
.loop
.step
)
650 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
657 /* Collect additional arguments needed to emit a combined
658 parallel+workshare call. WS_STMT is the workshare directive being
661 static vec
<tree
, va_gc
> *
662 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
665 location_t loc
= gimple_location (ws_stmt
);
666 vec
<tree
, va_gc
> *ws_args
;
668 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
670 struct omp_for_data fd
;
673 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
677 if (gimple_omp_for_combined_into_p (ws_stmt
))
680 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
681 OMP_CLAUSE__LOOPTEMP_
);
683 n1
= OMP_CLAUSE_DECL (innerc
);
684 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
685 OMP_CLAUSE__LOOPTEMP_
);
687 n2
= OMP_CLAUSE_DECL (innerc
);
690 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
692 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
693 ws_args
->quick_push (t
);
695 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
696 ws_args
->quick_push (t
);
698 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
699 ws_args
->quick_push (t
);
703 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
704 ws_args
->quick_push (t
);
709 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
711 /* Number of sections is equal to the number of edges from the
712 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
713 the exit of the sections region. */
714 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
715 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
716 vec_alloc (ws_args
, 1);
717 ws_args
->quick_push (t
);
725 /* Discover whether REGION is a combined parallel+workshare region. */
728 determine_parallel_type (struct omp_region
*region
)
730 basic_block par_entry_bb
, par_exit_bb
;
731 basic_block ws_entry_bb
, ws_exit_bb
;
733 if (region
== NULL
|| region
->inner
== NULL
734 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
735 || region
->inner
->cont
== NULL
)
738 /* We only support parallel+for and parallel+sections. */
739 if (region
->type
!= GIMPLE_OMP_PARALLEL
740 || (region
->inner
->type
!= GIMPLE_OMP_FOR
741 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
744 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
745 WS_EXIT_BB -> PAR_EXIT_BB. */
746 par_entry_bb
= region
->entry
;
747 par_exit_bb
= region
->exit
;
748 ws_entry_bb
= region
->inner
->entry
;
749 ws_exit_bb
= region
->inner
->exit
;
751 if (single_succ (par_entry_bb
) == ws_entry_bb
752 && single_succ (ws_exit_bb
) == par_exit_bb
753 && workshare_safe_to_combine_p (ws_entry_bb
)
754 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
755 || (last_and_only_stmt (ws_entry_bb
)
756 && last_and_only_stmt (par_exit_bb
))))
758 gimple par_stmt
= last_stmt (par_entry_bb
);
759 gimple ws_stmt
= last_stmt (ws_entry_bb
);
761 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
763 /* If this is a combined parallel loop, we need to determine
764 whether or not to use the combined library calls. There
765 are two cases where we do not apply the transformation:
766 static loops and any kind of ordered loop. In the first
767 case, we already open code the loop so there is no need
768 to do anything else. In the latter case, the combined
769 parallel loop call would still need extra synchronization
770 to implement ordered semantics, so there would not be any
771 gain in using the combined call. */
772 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
773 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
775 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
776 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
778 region
->is_combined_parallel
= false;
779 region
->inner
->is_combined_parallel
= false;
784 region
->is_combined_parallel
= true;
785 region
->inner
->is_combined_parallel
= true;
786 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
791 /* Return true if EXPR is variable sized. */
794 is_variable_sized (const_tree expr
)
796 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
799 /* Return true if DECL is a reference type. */
802 is_reference (tree decl
)
804 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
807 /* Lookup variables in the decl or field splay trees. The "maybe" form
808 allows for the variable form to not have been entered, otherwise we
809 assert that the variable must have been entered. */
812 lookup_decl (tree var
, omp_context
*ctx
)
815 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
820 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
823 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
824 return n
? *n
: NULL_TREE
;
828 lookup_field (tree var
, omp_context
*ctx
)
831 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
832 return (tree
) n
->value
;
836 lookup_sfield (tree var
, omp_context
*ctx
)
839 n
= splay_tree_lookup (ctx
->sfield_map
840 ? ctx
->sfield_map
: ctx
->field_map
,
841 (splay_tree_key
) var
);
842 return (tree
) n
->value
;
846 maybe_lookup_field (tree var
, omp_context
*ctx
)
849 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
850 return n
? (tree
) n
->value
: NULL_TREE
;
853 /* Return true if DECL should be copied by pointer. SHARED_CTX is
854 the parallel context if DECL is to be shared. */
857 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
859 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
862 /* We can only use copy-in/copy-out semantics for shared variables
863 when we know the value is not accessible from an outer scope. */
866 /* ??? Trivially accessible from anywhere. But why would we even
867 be passing an address in this case? Should we simply assert
868 this to be false, or should we have a cleanup pass that removes
869 these from the list of mappings? */
870 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
873 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
874 without analyzing the expression whether or not its location
875 is accessible to anyone else. In the case of nested parallel
876 regions it certainly may be. */
877 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
880 /* Do not use copy-in/copy-out for variables that have their
882 if (TREE_ADDRESSABLE (decl
))
885 /* lower_send_shared_vars only uses copy-in, but not copy-out
887 if (TREE_READONLY (decl
)
888 || ((TREE_CODE (decl
) == RESULT_DECL
889 || TREE_CODE (decl
) == PARM_DECL
)
890 && DECL_BY_REFERENCE (decl
)))
893 /* Disallow copy-in/out in nested parallel if
894 decl is shared in outer parallel, otherwise
895 each thread could store the shared variable
896 in its own copy-in location, making the
897 variable no longer really shared. */
898 if (shared_ctx
->is_nested
)
902 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
903 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
910 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
911 c
; c
= OMP_CLAUSE_CHAIN (c
))
912 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
913 && OMP_CLAUSE_DECL (c
) == decl
)
917 goto maybe_mark_addressable_and_ret
;
921 /* For tasks avoid using copy-in/out. As tasks can be
922 deferred or executed in different thread, when GOMP_task
923 returns, the task hasn't necessarily terminated. */
924 if (is_task_ctx (shared_ctx
))
927 maybe_mark_addressable_and_ret
:
928 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
929 if (is_gimple_reg (outer
))
931 /* Taking address of OUTER in lower_send_shared_vars
932 might need regimplification of everything that uses the
934 if (!task_shared_vars
)
935 task_shared_vars
= BITMAP_ALLOC (NULL
);
936 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
937 TREE_ADDRESSABLE (outer
) = 1;
946 /* Construct a new automatic decl similar to VAR. */
949 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
951 tree copy
= copy_var_decl (var
, name
, type
);
953 DECL_CONTEXT (copy
) = current_function_decl
;
954 DECL_CHAIN (copy
) = ctx
->block_vars
;
955 ctx
->block_vars
= copy
;
961 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
963 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
966 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
969 omp_build_component_ref (tree obj
, tree field
)
971 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
972 if (TREE_THIS_VOLATILE (field
))
973 TREE_THIS_VOLATILE (ret
) |= 1;
974 if (TREE_READONLY (field
))
975 TREE_READONLY (ret
) |= 1;
979 /* Build tree nodes to access the field for VAR on the receiver side. */
982 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
984 tree x
, field
= lookup_field (var
, ctx
);
986 /* If the receiver record type was remapped in the child function,
987 remap the field into the new record type. */
988 x
= maybe_lookup_field (field
, ctx
);
992 x
= build_simple_mem_ref (ctx
->receiver_decl
);
993 x
= omp_build_component_ref (x
, field
);
995 x
= build_simple_mem_ref (x
);
1000 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1001 of a parallel, this is a component reference; for workshare constructs
1002 this is some variable. */
1005 build_outer_var_ref (tree var
, omp_context
*ctx
)
1009 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1011 else if (is_variable_sized (var
))
1013 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1014 x
= build_outer_var_ref (x
, ctx
);
1015 x
= build_simple_mem_ref (x
);
1017 else if (is_taskreg_ctx (ctx
))
1019 bool by_ref
= use_pointer_for_field (var
, NULL
);
1020 x
= build_receiver_ref (var
, by_ref
, ctx
);
1022 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1023 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
1025 /* #pragma omp simd isn't a worksharing construct, and can reference even
1026 private vars in its linear etc. clauses. */
1028 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1029 x
= lookup_decl (var
, ctx
->outer
);
1030 else if (ctx
->outer
)
1031 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1035 else if (ctx
->outer
)
1036 x
= lookup_decl (var
, ctx
->outer
);
1037 else if (is_reference (var
))
1038 /* This can happen with orphaned constructs. If var is reference, it is
1039 possible it is shared and as such valid. */
1044 if (is_reference (var
))
1045 x
= build_simple_mem_ref (x
);
1050 /* Build tree nodes to access the field for VAR on the sender side. */
1053 build_sender_ref (tree var
, omp_context
*ctx
)
1055 tree field
= lookup_sfield (var
, ctx
);
1056 return omp_build_component_ref (ctx
->sender_decl
, field
);
1059 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1062 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1064 tree field
, type
, sfield
= NULL_TREE
;
1066 gcc_assert ((mask
& 1) == 0
1067 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1068 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1069 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1071 type
= TREE_TYPE (var
);
1074 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1075 type
= build_pointer_type (build_pointer_type (type
));
1078 type
= build_pointer_type (type
);
1079 else if ((mask
& 3) == 1 && is_reference (var
))
1080 type
= TREE_TYPE (type
);
1082 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1083 FIELD_DECL
, DECL_NAME (var
), type
);
1085 /* Remember what variable this field was created for. This does have a
1086 side effect of making dwarf2out ignore this member, so for helpful
1087 debugging we clear it later in delete_omp_context. */
1088 DECL_ABSTRACT_ORIGIN (field
) = var
;
1089 if (type
== TREE_TYPE (var
))
1091 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1092 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1093 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1096 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1098 if ((mask
& 3) == 3)
1100 insert_field_into_struct (ctx
->record_type
, field
);
1101 if (ctx
->srecord_type
)
1103 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1104 FIELD_DECL
, DECL_NAME (var
), type
);
1105 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1106 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1107 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1108 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1109 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1114 if (ctx
->srecord_type
== NULL_TREE
)
1118 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1119 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1120 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1122 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1123 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1124 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1125 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1126 splay_tree_insert (ctx
->sfield_map
,
1127 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1128 (splay_tree_value
) sfield
);
1132 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1133 : ctx
->srecord_type
, field
);
1137 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1138 (splay_tree_value
) field
);
1139 if ((mask
& 2) && ctx
->sfield_map
)
1140 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1141 (splay_tree_value
) sfield
);
1145 install_var_local (tree var
, omp_context
*ctx
)
1147 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1148 insert_decl_map (&ctx
->cb
, var
, new_var
);
1152 /* Adjust the replacement for DECL in CTX for the new context. This means
1153 copying the DECL_VALUE_EXPR, and fixing up the type. */
1156 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1158 tree new_decl
, size
;
1160 new_decl
= lookup_decl (decl
, ctx
);
1162 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1164 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1165 && DECL_HAS_VALUE_EXPR_P (decl
))
1167 tree ve
= DECL_VALUE_EXPR (decl
);
1168 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1169 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1170 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1173 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1175 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1176 if (size
== error_mark_node
)
1177 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1178 DECL_SIZE (new_decl
) = size
;
1180 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1181 if (size
== error_mark_node
)
1182 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1183 DECL_SIZE_UNIT (new_decl
) = size
;
1187 /* The callback for remap_decl. Search all containing contexts for a
1188 mapping of the variable; this avoids having to duplicate the splay
1189 tree ahead of time. We know a mapping doesn't already exist in the
1190 given context. Create new mappings to implement default semantics. */
1193 omp_copy_decl (tree var
, copy_body_data
*cb
)
1195 omp_context
*ctx
= (omp_context
*) cb
;
1198 if (TREE_CODE (var
) == LABEL_DECL
)
1200 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1201 DECL_CONTEXT (new_var
) = current_function_decl
;
1202 insert_decl_map (&ctx
->cb
, var
, new_var
);
1206 while (!is_taskreg_ctx (ctx
))
1211 new_var
= maybe_lookup_decl (var
, ctx
);
1216 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1219 return error_mark_node
;
1223 /* Return the parallel region associated with STMT. */
1225 /* Debugging dumps for parallel regions. */
1226 void dump_omp_region (FILE *, struct omp_region
*, int);
1227 void debug_omp_region (struct omp_region
*);
1228 void debug_all_omp_regions (void);
1230 /* Dump the parallel region tree rooted at REGION. */
1233 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1235 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1236 gimple_code_name
[region
->type
]);
1239 dump_omp_region (file
, region
->inner
, indent
+ 4);
1243 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1244 region
->cont
->index
);
1248 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1249 region
->exit
->index
);
1251 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1254 dump_omp_region (file
, region
->next
, indent
);
1258 debug_omp_region (struct omp_region
*region
)
1260 dump_omp_region (stderr
, region
, 0);
1264 debug_all_omp_regions (void)
1266 dump_omp_region (stderr
, root_omp_region
, 0);
1270 /* Create a new parallel region starting at STMT inside region PARENT. */
1272 static struct omp_region
*
1273 new_omp_region (basic_block bb
, enum gimple_code type
,
1274 struct omp_region
*parent
)
1276 struct omp_region
*region
= XCNEW (struct omp_region
);
1278 region
->outer
= parent
;
1280 region
->type
= type
;
1284 /* This is a nested region. Add it to the list of inner
1285 regions in PARENT. */
1286 region
->next
= parent
->inner
;
1287 parent
->inner
= region
;
1291 /* This is a toplevel region. Add it to the list of toplevel
1292 regions in ROOT_OMP_REGION. */
1293 region
->next
= root_omp_region
;
1294 root_omp_region
= region
;
1300 /* Release the memory associated with the region tree rooted at REGION. */
1303 free_omp_region_1 (struct omp_region
*region
)
1305 struct omp_region
*i
, *n
;
1307 for (i
= region
->inner
; i
; i
= n
)
1310 free_omp_region_1 (i
);
1316 /* Release the memory for the entire omp region tree. */
1319 free_omp_regions (void)
1321 struct omp_region
*r
, *n
;
1322 for (r
= root_omp_region
; r
; r
= n
)
1325 free_omp_region_1 (r
);
1327 root_omp_region
= NULL
;
1331 /* Create a new context, with OUTER_CTX being the surrounding context. */
1333 static omp_context
*
1334 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1336 omp_context
*ctx
= XCNEW (omp_context
);
1338 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1339 (splay_tree_value
) ctx
);
1344 ctx
->outer
= outer_ctx
;
1345 ctx
->cb
= outer_ctx
->cb
;
1346 ctx
->cb
.block
= NULL
;
1347 ctx
->depth
= outer_ctx
->depth
+ 1;
1351 ctx
->cb
.src_fn
= current_function_decl
;
1352 ctx
->cb
.dst_fn
= current_function_decl
;
1353 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1354 gcc_checking_assert (ctx
->cb
.src_node
);
1355 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1356 ctx
->cb
.src_cfun
= cfun
;
1357 ctx
->cb
.copy_decl
= omp_copy_decl
;
1358 ctx
->cb
.eh_lp_nr
= 0;
1359 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1363 ctx
->cb
.decl_map
= pointer_map_create ();
1368 static gimple_seq
maybe_catch_exception (gimple_seq
);
1370 /* Finalize task copyfn. */
1373 finalize_task_copyfn (gimple task_stmt
)
1375 struct function
*child_cfun
;
1377 gimple_seq seq
= NULL
, new_seq
;
1380 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1381 if (child_fn
== NULL_TREE
)
1384 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1385 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1387 push_cfun (child_cfun
);
1388 bind
= gimplify_body (child_fn
, false);
1389 gimple_seq_add_stmt (&seq
, bind
);
1390 new_seq
= maybe_catch_exception (seq
);
1393 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1395 gimple_seq_add_stmt (&seq
, bind
);
1397 gimple_set_body (child_fn
, seq
);
1400 /* Inform the callgraph about the new function. */
1401 cgraph_add_new_function (child_fn
, false);
1404 /* Destroy a omp_context data structures. Called through the splay tree
1405 value delete callback. */
1408 delete_omp_context (splay_tree_value value
)
1410 omp_context
*ctx
= (omp_context
*) value
;
1412 pointer_map_destroy (ctx
->cb
.decl_map
);
1415 splay_tree_delete (ctx
->field_map
);
1416 if (ctx
->sfield_map
)
1417 splay_tree_delete (ctx
->sfield_map
);
1419 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1420 it produces corrupt debug information. */
1421 if (ctx
->record_type
)
1424 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1425 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1427 if (ctx
->srecord_type
)
1430 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1431 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1434 if (is_task_ctx (ctx
))
1435 finalize_task_copyfn (ctx
->stmt
);
1440 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1444 fixup_child_record_type (omp_context
*ctx
)
1446 tree f
, type
= ctx
->record_type
;
1448 /* ??? It isn't sufficient to just call remap_type here, because
1449 variably_modified_type_p doesn't work the way we expect for
1450 record types. Testing each field for whether it needs remapping
1451 and creating a new record by hand works, however. */
1452 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1453 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1457 tree name
, new_fields
= NULL
;
1459 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1460 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1461 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1462 TYPE_DECL
, name
, type
);
1463 TYPE_NAME (type
) = name
;
1465 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1467 tree new_f
= copy_node (f
);
1468 DECL_CONTEXT (new_f
) = type
;
1469 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1470 DECL_CHAIN (new_f
) = new_fields
;
1471 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1472 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1474 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1478 /* Arrange to be able to look up the receiver field
1479 given the sender field. */
1480 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1481 (splay_tree_value
) new_f
);
1483 TYPE_FIELDS (type
) = nreverse (new_fields
);
1487 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1490 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1491 specified by CLAUSES. */
1494 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1497 bool scan_array_reductions
= false;
1499 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1503 switch (OMP_CLAUSE_CODE (c
))
1505 case OMP_CLAUSE_PRIVATE
:
1506 decl
= OMP_CLAUSE_DECL (c
);
1507 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1509 else if (!is_variable_sized (decl
))
1510 install_var_local (decl
, ctx
);
1513 case OMP_CLAUSE_SHARED
:
1514 /* Ignore shared directives in teams construct. */
1515 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1517 gcc_assert (is_taskreg_ctx (ctx
));
1518 decl
= OMP_CLAUSE_DECL (c
);
1519 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1520 || !is_variable_sized (decl
));
1521 /* Global variables don't need to be copied,
1522 the receiver side will use them directly. */
1523 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1525 by_ref
= use_pointer_for_field (decl
, ctx
);
1526 if (! TREE_READONLY (decl
)
1527 || TREE_ADDRESSABLE (decl
)
1529 || is_reference (decl
))
1531 install_var_field (decl
, by_ref
, 3, ctx
);
1532 install_var_local (decl
, ctx
);
1535 /* We don't need to copy const scalar vars back. */
1536 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1539 case OMP_CLAUSE_LASTPRIVATE
:
1540 /* Let the corresponding firstprivate clause create
1542 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1546 case OMP_CLAUSE_FIRSTPRIVATE
:
1547 case OMP_CLAUSE_REDUCTION
:
1548 case OMP_CLAUSE_LINEAR
:
1549 decl
= OMP_CLAUSE_DECL (c
);
1551 if (is_variable_sized (decl
))
1553 if (is_task_ctx (ctx
))
1554 install_var_field (decl
, false, 1, ctx
);
1557 else if (is_taskreg_ctx (ctx
))
1560 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1561 by_ref
= use_pointer_for_field (decl
, NULL
);
1563 if (is_task_ctx (ctx
)
1564 && (global
|| by_ref
|| is_reference (decl
)))
1566 install_var_field (decl
, false, 1, ctx
);
1568 install_var_field (decl
, by_ref
, 2, ctx
);
1571 install_var_field (decl
, by_ref
, 3, ctx
);
1573 install_var_local (decl
, ctx
);
1576 case OMP_CLAUSE__LOOPTEMP_
:
1577 gcc_assert (is_parallel_ctx (ctx
));
1578 decl
= OMP_CLAUSE_DECL (c
);
1579 install_var_field (decl
, false, 3, ctx
);
1580 install_var_local (decl
, ctx
);
1583 case OMP_CLAUSE_COPYPRIVATE
:
1584 case OMP_CLAUSE_COPYIN
:
1585 decl
= OMP_CLAUSE_DECL (c
);
1586 by_ref
= use_pointer_for_field (decl
, NULL
);
1587 install_var_field (decl
, by_ref
, 3, ctx
);
1590 case OMP_CLAUSE_DEFAULT
:
1591 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1594 case OMP_CLAUSE_FINAL
:
1596 case OMP_CLAUSE_NUM_THREADS
:
1597 case OMP_CLAUSE_NUM_TEAMS
:
1598 case OMP_CLAUSE_THREAD_LIMIT
:
1599 case OMP_CLAUSE_DEVICE
:
1600 case OMP_CLAUSE_SCHEDULE
:
1601 case OMP_CLAUSE_DIST_SCHEDULE
:
1602 case OMP_CLAUSE_DEPEND
:
1604 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1608 case OMP_CLAUSE_FROM
:
1609 case OMP_CLAUSE_MAP
:
1611 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1612 decl
= OMP_CLAUSE_DECL (c
);
1613 /* Global variables with "omp declare target" attribute
1614 don't need to be copied, the receiver side will use them
1616 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1618 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1619 && lookup_attribute ("omp declare target",
1620 DECL_ATTRIBUTES (decl
)))
1622 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1623 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1625 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1626 #pragma omp target data, there is nothing to map for
1628 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1629 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1634 if (DECL_SIZE (decl
)
1635 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1637 tree decl2
= DECL_VALUE_EXPR (decl
);
1638 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1639 decl2
= TREE_OPERAND (decl2
, 0);
1640 gcc_assert (DECL_P (decl2
));
1641 install_var_field (decl2
, true, 3, ctx
);
1642 install_var_local (decl2
, ctx
);
1643 install_var_local (decl
, ctx
);
1647 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1648 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1649 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1650 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1651 install_var_field (decl
, true, 7, ctx
);
1653 install_var_field (decl
, true, 3, ctx
);
1654 if (gimple_omp_target_kind (ctx
->stmt
)
1655 == GF_OMP_TARGET_KIND_REGION
)
1656 install_var_local (decl
, ctx
);
1661 tree base
= get_base_address (decl
);
1662 tree nc
= OMP_CLAUSE_CHAIN (c
);
1665 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1666 && OMP_CLAUSE_DECL (nc
) == base
1667 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1668 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1670 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1671 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1675 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1676 (splay_tree_key
) decl
));
1678 = build_decl (OMP_CLAUSE_LOCATION (c
),
1679 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1680 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1681 insert_field_into_struct (ctx
->record_type
, field
);
1682 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1683 (splay_tree_value
) field
);
1688 case OMP_CLAUSE_NOWAIT
:
1689 case OMP_CLAUSE_ORDERED
:
1690 case OMP_CLAUSE_COLLAPSE
:
1691 case OMP_CLAUSE_UNTIED
:
1692 case OMP_CLAUSE_MERGEABLE
:
1693 case OMP_CLAUSE_PROC_BIND
:
1694 case OMP_CLAUSE_SAFELEN
:
1697 case OMP_CLAUSE_ALIGNED
:
1698 decl
= OMP_CLAUSE_DECL (c
);
1699 if (is_global_var (decl
)
1700 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1701 install_var_local (decl
, ctx
);
1709 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1711 switch (OMP_CLAUSE_CODE (c
))
1713 case OMP_CLAUSE_LASTPRIVATE
:
1714 /* Let the corresponding firstprivate clause create
1716 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1717 scan_array_reductions
= true;
1718 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1722 case OMP_CLAUSE_PRIVATE
:
1723 case OMP_CLAUSE_FIRSTPRIVATE
:
1724 case OMP_CLAUSE_REDUCTION
:
1725 case OMP_CLAUSE_LINEAR
:
1726 decl
= OMP_CLAUSE_DECL (c
);
1727 if (is_variable_sized (decl
))
1728 install_var_local (decl
, ctx
);
1729 fixup_remapped_decl (decl
, ctx
,
1730 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1731 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1732 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1733 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1734 scan_array_reductions
= true;
1737 case OMP_CLAUSE_SHARED
:
1738 /* Ignore shared directives in teams construct. */
1739 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1741 decl
= OMP_CLAUSE_DECL (c
);
1742 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1743 fixup_remapped_decl (decl
, ctx
, false);
1746 case OMP_CLAUSE_MAP
:
1747 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1749 decl
= OMP_CLAUSE_DECL (c
);
1751 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1752 && lookup_attribute ("omp declare target",
1753 DECL_ATTRIBUTES (decl
)))
1757 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1758 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1759 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1761 tree new_decl
= lookup_decl (decl
, ctx
);
1762 TREE_TYPE (new_decl
)
1763 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1765 else if (DECL_SIZE (decl
)
1766 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1768 tree decl2
= DECL_VALUE_EXPR (decl
);
1769 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1770 decl2
= TREE_OPERAND (decl2
, 0);
1771 gcc_assert (DECL_P (decl2
));
1772 fixup_remapped_decl (decl2
, ctx
, false);
1773 fixup_remapped_decl (decl
, ctx
, true);
1776 fixup_remapped_decl (decl
, ctx
, false);
1780 case OMP_CLAUSE_COPYPRIVATE
:
1781 case OMP_CLAUSE_COPYIN
:
1782 case OMP_CLAUSE_DEFAULT
:
1784 case OMP_CLAUSE_NUM_THREADS
:
1785 case OMP_CLAUSE_NUM_TEAMS
:
1786 case OMP_CLAUSE_THREAD_LIMIT
:
1787 case OMP_CLAUSE_DEVICE
:
1788 case OMP_CLAUSE_SCHEDULE
:
1789 case OMP_CLAUSE_DIST_SCHEDULE
:
1790 case OMP_CLAUSE_NOWAIT
:
1791 case OMP_CLAUSE_ORDERED
:
1792 case OMP_CLAUSE_COLLAPSE
:
1793 case OMP_CLAUSE_UNTIED
:
1794 case OMP_CLAUSE_FINAL
:
1795 case OMP_CLAUSE_MERGEABLE
:
1796 case OMP_CLAUSE_PROC_BIND
:
1797 case OMP_CLAUSE_SAFELEN
:
1798 case OMP_CLAUSE_ALIGNED
:
1799 case OMP_CLAUSE_DEPEND
:
1800 case OMP_CLAUSE__LOOPTEMP_
:
1802 case OMP_CLAUSE_FROM
:
1810 if (scan_array_reductions
)
1811 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1812 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1813 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1815 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1816 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1818 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1819 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1820 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1823 /* Create a new name for omp child function. Returns an identifier. */
1825 static GTY(()) unsigned int tmp_ompfn_id_num
;
1828 create_omp_child_function_name (bool task_copy
)
1830 return (clone_function_name (current_function_decl
,
1831 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1834 /* Build a decl for the omp child function. It'll not contain a body
1835 yet, just the bare decl. */
1838 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1840 tree decl
, type
, name
, t
;
1842 name
= create_omp_child_function_name (task_copy
);
1844 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1845 ptr_type_node
, NULL_TREE
);
1847 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1849 decl
= build_decl (gimple_location (ctx
->stmt
),
1850 FUNCTION_DECL
, name
, type
);
1853 ctx
->cb
.dst_fn
= decl
;
1855 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1857 TREE_STATIC (decl
) = 1;
1858 TREE_USED (decl
) = 1;
1859 DECL_ARTIFICIAL (decl
) = 1;
1860 DECL_NAMELESS (decl
) = 1;
1861 DECL_IGNORED_P (decl
) = 0;
1862 TREE_PUBLIC (decl
) = 0;
1863 DECL_UNINLINABLE (decl
) = 1;
1864 DECL_EXTERNAL (decl
) = 0;
1865 DECL_CONTEXT (decl
) = NULL_TREE
;
1866 DECL_INITIAL (decl
) = make_node (BLOCK
);
1867 bool target_p
= false;
1868 if (lookup_attribute ("omp declare target",
1869 DECL_ATTRIBUTES (current_function_decl
)))
1874 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1875 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1876 && gimple_omp_target_kind (octx
->stmt
)
1877 == GF_OMP_TARGET_KIND_REGION
)
1884 DECL_ATTRIBUTES (decl
)
1885 = tree_cons (get_identifier ("omp declare target"),
1886 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1888 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1889 RESULT_DECL
, NULL_TREE
, void_type_node
);
1890 DECL_ARTIFICIAL (t
) = 1;
1891 DECL_IGNORED_P (t
) = 1;
1892 DECL_CONTEXT (t
) = decl
;
1893 DECL_RESULT (decl
) = t
;
1895 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1896 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1897 DECL_ARTIFICIAL (t
) = 1;
1898 DECL_NAMELESS (t
) = 1;
1899 DECL_ARG_TYPE (t
) = ptr_type_node
;
1900 DECL_CONTEXT (t
) = current_function_decl
;
1902 DECL_ARGUMENTS (decl
) = t
;
1904 ctx
->receiver_decl
= t
;
1907 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1908 PARM_DECL
, get_identifier (".omp_data_o"),
1910 DECL_ARTIFICIAL (t
) = 1;
1911 DECL_NAMELESS (t
) = 1;
1912 DECL_ARG_TYPE (t
) = ptr_type_node
;
1913 DECL_CONTEXT (t
) = current_function_decl
;
1915 TREE_ADDRESSABLE (t
) = 1;
1916 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1917 DECL_ARGUMENTS (decl
) = t
;
1920 /* Allocate memory for the function structure. The call to
1921 allocate_struct_function clobbers CFUN, so we need to restore
1923 push_struct_function (decl
);
1924 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1928 /* Callback for walk_gimple_seq. Check if combined parallel
1929 contains gimple_omp_for_combined_into_p OMP_FOR. */
1932 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1933 bool *handled_ops_p
,
1934 struct walk_stmt_info
*wi
)
1936 gimple stmt
= gsi_stmt (*gsi_p
);
1938 *handled_ops_p
= true;
1939 switch (gimple_code (stmt
))
1943 case GIMPLE_OMP_FOR
:
1944 if (gimple_omp_for_combined_into_p (stmt
)
1945 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1948 return integer_zero_node
;
1957 /* Scan an OpenMP parallel directive. */
1960 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1964 gimple stmt
= gsi_stmt (*gsi
);
1966 /* Ignore parallel directives with empty bodies, unless there
1967 are copyin clauses. */
1969 && empty_body_p (gimple_omp_body (stmt
))
1970 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1971 OMP_CLAUSE_COPYIN
) == NULL
)
1973 gsi_replace (gsi
, gimple_build_nop (), false);
1977 if (gimple_omp_parallel_combined_p (stmt
))
1980 struct walk_stmt_info wi
;
1982 memset (&wi
, 0, sizeof (wi
));
1984 walk_gimple_seq (gimple_omp_body (stmt
),
1985 find_combined_for
, NULL
, &wi
);
1986 for_stmt
= (gimple
) wi
.info
;
1989 struct omp_for_data fd
;
1990 extract_omp_for_data (for_stmt
, &fd
, NULL
);
1991 /* We need two temporaries with fd.loop.v type (istart/iend)
1992 and then (fd.collapse - 1) temporaries with the same
1993 type for count2 ... countN-1 vars if not constant. */
1994 size_t count
= 2, i
;
1995 tree type
= fd
.iter_type
;
1997 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
1998 count
+= fd
.collapse
- 1;
1999 for (i
= 0; i
< count
; i
++)
2001 tree temp
= create_tmp_var (type
, NULL
);
2002 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2003 OMP_CLAUSE__LOOPTEMP_
);
2004 OMP_CLAUSE_DECL (c
) = temp
;
2005 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2006 gimple_omp_parallel_set_clauses (stmt
, c
);
2011 ctx
= new_omp_context (stmt
, outer_ctx
);
2012 if (taskreg_nesting_level
> 1)
2013 ctx
->is_nested
= true;
2014 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2015 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2016 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2017 name
= create_tmp_var_name (".omp_data_s");
2018 name
= build_decl (gimple_location (stmt
),
2019 TYPE_DECL
, name
, ctx
->record_type
);
2020 DECL_ARTIFICIAL (name
) = 1;
2021 DECL_NAMELESS (name
) = 1;
2022 TYPE_NAME (ctx
->record_type
) = name
;
2023 create_omp_child_function (ctx
, false);
2024 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2026 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2027 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2029 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2030 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2033 layout_type (ctx
->record_type
);
2034 fixup_child_record_type (ctx
);
2038 /* Scan an OpenMP task directive. */
2041 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2045 gimple stmt
= gsi_stmt (*gsi
);
2046 location_t loc
= gimple_location (stmt
);
2048 /* Ignore task directives with empty bodies. */
2050 && empty_body_p (gimple_omp_body (stmt
)))
2052 gsi_replace (gsi
, gimple_build_nop (), false);
2056 ctx
= new_omp_context (stmt
, outer_ctx
);
2057 if (taskreg_nesting_level
> 1)
2058 ctx
->is_nested
= true;
2059 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2060 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2061 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2062 name
= create_tmp_var_name (".omp_data_s");
2063 name
= build_decl (gimple_location (stmt
),
2064 TYPE_DECL
, name
, ctx
->record_type
);
2065 DECL_ARTIFICIAL (name
) = 1;
2066 DECL_NAMELESS (name
) = 1;
2067 TYPE_NAME (ctx
->record_type
) = name
;
2068 create_omp_child_function (ctx
, false);
2069 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2071 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2073 if (ctx
->srecord_type
)
2075 name
= create_tmp_var_name (".omp_data_a");
2076 name
= build_decl (gimple_location (stmt
),
2077 TYPE_DECL
, name
, ctx
->srecord_type
);
2078 DECL_ARTIFICIAL (name
) = 1;
2079 DECL_NAMELESS (name
) = 1;
2080 TYPE_NAME (ctx
->srecord_type
) = name
;
2081 create_omp_child_function (ctx
, true);
2084 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2086 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2088 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2089 t
= build_int_cst (long_integer_type_node
, 0);
2090 gimple_omp_task_set_arg_size (stmt
, t
);
2091 t
= build_int_cst (long_integer_type_node
, 1);
2092 gimple_omp_task_set_arg_align (stmt
, t
);
2096 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2097 /* Move VLA fields to the end. */
2098 p
= &TYPE_FIELDS (ctx
->record_type
);
2100 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2101 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2104 *p
= TREE_CHAIN (*p
);
2105 TREE_CHAIN (*q
) = NULL_TREE
;
2106 q
= &TREE_CHAIN (*q
);
2109 p
= &DECL_CHAIN (*p
);
2111 layout_type (ctx
->record_type
);
2112 fixup_child_record_type (ctx
);
2113 if (ctx
->srecord_type
)
2114 layout_type (ctx
->srecord_type
);
2115 t
= fold_convert_loc (loc
, long_integer_type_node
,
2116 TYPE_SIZE_UNIT (ctx
->record_type
));
2117 gimple_omp_task_set_arg_size (stmt
, t
);
2118 t
= build_int_cst (long_integer_type_node
,
2119 TYPE_ALIGN_UNIT (ctx
->record_type
));
2120 gimple_omp_task_set_arg_align (stmt
, t
);
2125 /* Scan an OpenMP loop directive. */
2128 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2133 ctx
= new_omp_context (stmt
, outer_ctx
);
2135 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2137 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2138 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2140 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2141 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2142 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2143 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2145 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2148 /* Scan an OpenMP sections directive. */
2151 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2155 ctx
= new_omp_context (stmt
, outer_ctx
);
2156 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2157 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2160 /* Scan an OpenMP single directive. */
2163 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2168 ctx
= new_omp_context (stmt
, outer_ctx
);
2169 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2170 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2171 name
= create_tmp_var_name (".omp_copy_s");
2172 name
= build_decl (gimple_location (stmt
),
2173 TYPE_DECL
, name
, ctx
->record_type
);
2174 TYPE_NAME (ctx
->record_type
) = name
;
2176 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2177 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2179 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2180 ctx
->record_type
= NULL
;
2182 layout_type (ctx
->record_type
);
2185 /* Scan an OpenMP target{, data, update} directive. */
2188 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2192 int kind
= gimple_omp_target_kind (stmt
);
2194 ctx
= new_omp_context (stmt
, outer_ctx
);
2195 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2196 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2197 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2198 name
= create_tmp_var_name (".omp_data_t");
2199 name
= build_decl (gimple_location (stmt
),
2200 TYPE_DECL
, name
, ctx
->record_type
);
2201 DECL_ARTIFICIAL (name
) = 1;
2202 DECL_NAMELESS (name
) = 1;
2203 TYPE_NAME (ctx
->record_type
) = name
;
2204 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2206 create_omp_child_function (ctx
, false);
2207 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2210 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2211 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2213 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2214 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2217 TYPE_FIELDS (ctx
->record_type
)
2218 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2219 #ifdef ENABLE_CHECKING
2221 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2222 for (field
= TYPE_FIELDS (ctx
->record_type
);
2224 field
= DECL_CHAIN (field
))
2225 gcc_assert (DECL_ALIGN (field
) == align
);
2227 layout_type (ctx
->record_type
);
2228 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2229 fixup_child_record_type (ctx
);
2233 /* Scan an OpenMP teams directive. */
2236 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2238 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2239 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2240 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2243 /* Check OpenMP nesting restrictions. */
2245 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2249 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2250 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2252 error_at (gimple_location (stmt
),
2253 "OpenMP constructs may not be nested inside simd region");
2256 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2258 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2259 || (gimple_omp_for_kind (stmt
)
2260 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2261 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2263 error_at (gimple_location (stmt
),
2264 "only distribute or parallel constructs are allowed to "
2265 "be closely nested inside teams construct");
2270 switch (gimple_code (stmt
))
2272 case GIMPLE_OMP_FOR
:
2273 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_KIND_SIMD
)
2275 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2277 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2279 error_at (gimple_location (stmt
),
2280 "distribute construct must be closely nested inside "
2288 if (is_gimple_call (stmt
)
2289 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2290 == BUILT_IN_GOMP_CANCEL
2291 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2292 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2294 const char *bad
= NULL
;
2295 const char *kind
= NULL
;
2298 error_at (gimple_location (stmt
), "orphaned %qs construct",
2299 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2300 == BUILT_IN_GOMP_CANCEL
2301 ? "#pragma omp cancel"
2302 : "#pragma omp cancellation point");
2305 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2306 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2310 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2311 bad
= "#pragma omp parallel";
2312 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2313 == BUILT_IN_GOMP_CANCEL
2314 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2315 ctx
->cancellable
= true;
2319 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2320 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2321 bad
= "#pragma omp for";
2322 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2323 == BUILT_IN_GOMP_CANCEL
2324 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2326 ctx
->cancellable
= true;
2327 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2329 warning_at (gimple_location (stmt
), 0,
2330 "%<#pragma omp cancel for%> inside "
2331 "%<nowait%> for construct");
2332 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2333 OMP_CLAUSE_ORDERED
))
2334 warning_at (gimple_location (stmt
), 0,
2335 "%<#pragma omp cancel for%> inside "
2336 "%<ordered%> for construct");
2341 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2342 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2343 bad
= "#pragma omp sections";
2344 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2345 == BUILT_IN_GOMP_CANCEL
2346 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2348 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2350 ctx
->cancellable
= true;
2351 if (find_omp_clause (gimple_omp_sections_clauses
2354 warning_at (gimple_location (stmt
), 0,
2355 "%<#pragma omp cancel sections%> inside "
2356 "%<nowait%> sections construct");
2360 gcc_assert (ctx
->outer
2361 && gimple_code (ctx
->outer
->stmt
)
2362 == GIMPLE_OMP_SECTIONS
);
2363 ctx
->outer
->cancellable
= true;
2364 if (find_omp_clause (gimple_omp_sections_clauses
2367 warning_at (gimple_location (stmt
), 0,
2368 "%<#pragma omp cancel sections%> inside "
2369 "%<nowait%> sections construct");
2375 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2376 bad
= "#pragma omp task";
2378 ctx
->cancellable
= true;
2382 error_at (gimple_location (stmt
), "invalid arguments");
2387 error_at (gimple_location (stmt
),
2388 "%<%s %s%> construct not closely nested inside of %qs",
2389 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2390 == BUILT_IN_GOMP_CANCEL
2391 ? "#pragma omp cancel"
2392 : "#pragma omp cancellation point", kind
, bad
);
2397 case GIMPLE_OMP_SECTIONS
:
2398 case GIMPLE_OMP_SINGLE
:
2399 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2400 switch (gimple_code (ctx
->stmt
))
2402 case GIMPLE_OMP_FOR
:
2403 case GIMPLE_OMP_SECTIONS
:
2404 case GIMPLE_OMP_SINGLE
:
2405 case GIMPLE_OMP_ORDERED
:
2406 case GIMPLE_OMP_MASTER
:
2407 case GIMPLE_OMP_TASK
:
2408 case GIMPLE_OMP_CRITICAL
:
2409 if (is_gimple_call (stmt
))
2411 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2412 != BUILT_IN_GOMP_BARRIER
)
2414 error_at (gimple_location (stmt
),
2415 "barrier region may not be closely nested inside "
2416 "of work-sharing, critical, ordered, master or "
2417 "explicit task region");
2420 error_at (gimple_location (stmt
),
2421 "work-sharing region may not be closely nested inside "
2422 "of work-sharing, critical, ordered, master or explicit "
2425 case GIMPLE_OMP_PARALLEL
:
2431 case GIMPLE_OMP_MASTER
:
2432 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2433 switch (gimple_code (ctx
->stmt
))
2435 case GIMPLE_OMP_FOR
:
2436 case GIMPLE_OMP_SECTIONS
:
2437 case GIMPLE_OMP_SINGLE
:
2438 case GIMPLE_OMP_TASK
:
2439 error_at (gimple_location (stmt
),
2440 "master region may not be closely nested inside "
2441 "of work-sharing or explicit task region");
2443 case GIMPLE_OMP_PARALLEL
:
2449 case GIMPLE_OMP_ORDERED
:
2450 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2451 switch (gimple_code (ctx
->stmt
))
2453 case GIMPLE_OMP_CRITICAL
:
2454 case GIMPLE_OMP_TASK
:
2455 error_at (gimple_location (stmt
),
2456 "ordered region may not be closely nested inside "
2457 "of critical or explicit task region");
2459 case GIMPLE_OMP_FOR
:
2460 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2461 OMP_CLAUSE_ORDERED
) == NULL
)
2463 error_at (gimple_location (stmt
),
2464 "ordered region must be closely nested inside "
2465 "a loop region with an ordered clause");
2469 case GIMPLE_OMP_PARALLEL
:
2470 error_at (gimple_location (stmt
),
2471 "ordered region must be closely nested inside "
2472 "a loop region with an ordered clause");
2478 case GIMPLE_OMP_CRITICAL
:
2479 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2480 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2481 && (gimple_omp_critical_name (stmt
)
2482 == gimple_omp_critical_name (ctx
->stmt
)))
2484 error_at (gimple_location (stmt
),
2485 "critical region may not be nested inside a critical "
2486 "region with the same name");
2490 case GIMPLE_OMP_TEAMS
:
2492 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2493 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2495 error_at (gimple_location (stmt
),
2496 "teams construct not closely nested inside of target "
2508 /* Helper function scan_omp.
2510 Callback for walk_tree or operators in walk_gimple_stmt used to
2511 scan for OpenMP directives in TP. */
2514 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2516 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2517 omp_context
*ctx
= (omp_context
*) wi
->info
;
2520 switch (TREE_CODE (t
))
2527 *tp
= remap_decl (t
, &ctx
->cb
);
2531 if (ctx
&& TYPE_P (t
))
2532 *tp
= remap_type (t
, &ctx
->cb
);
2533 else if (!DECL_P (t
))
2538 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2539 if (tem
!= TREE_TYPE (t
))
2541 if (TREE_CODE (t
) == INTEGER_CST
)
2542 *tp
= build_int_cst_wide (tem
,
2543 TREE_INT_CST_LOW (t
),
2544 TREE_INT_CST_HIGH (t
));
2546 TREE_TYPE (t
) = tem
;
2556 /* Return true if FNDECL is a setjmp or a longjmp. */
2559 setjmp_or_longjmp_p (const_tree fndecl
)
2561 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2562 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2563 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2566 tree declname
= DECL_NAME (fndecl
);
2569 const char *name
= IDENTIFIER_POINTER (declname
);
2570 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2574 /* Helper function for scan_omp.
2576 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2577 the current statement in GSI. */
2580 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2581 struct walk_stmt_info
*wi
)
2583 gimple stmt
= gsi_stmt (*gsi
);
2584 omp_context
*ctx
= (omp_context
*) wi
->info
;
2586 if (gimple_has_location (stmt
))
2587 input_location
= gimple_location (stmt
);
2589 /* Check the OpenMP nesting restrictions. */
2590 bool remove
= false;
2591 if (is_gimple_omp (stmt
))
2592 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2593 else if (is_gimple_call (stmt
))
2595 tree fndecl
= gimple_call_fndecl (stmt
);
2598 if (setjmp_or_longjmp_p (fndecl
)
2600 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2601 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2604 error_at (gimple_location (stmt
),
2605 "setjmp/longjmp inside simd construct");
2607 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2608 switch (DECL_FUNCTION_CODE (fndecl
))
2610 case BUILT_IN_GOMP_BARRIER
:
2611 case BUILT_IN_GOMP_CANCEL
:
2612 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2613 case BUILT_IN_GOMP_TASKYIELD
:
2614 case BUILT_IN_GOMP_TASKWAIT
:
2615 case BUILT_IN_GOMP_TASKGROUP_START
:
2616 case BUILT_IN_GOMP_TASKGROUP_END
:
2617 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2626 stmt
= gimple_build_nop ();
2627 gsi_replace (gsi
, stmt
, false);
2630 *handled_ops_p
= true;
2632 switch (gimple_code (stmt
))
2634 case GIMPLE_OMP_PARALLEL
:
2635 taskreg_nesting_level
++;
2636 scan_omp_parallel (gsi
, ctx
);
2637 taskreg_nesting_level
--;
2640 case GIMPLE_OMP_TASK
:
2641 taskreg_nesting_level
++;
2642 scan_omp_task (gsi
, ctx
);
2643 taskreg_nesting_level
--;
2646 case GIMPLE_OMP_FOR
:
2647 scan_omp_for (stmt
, ctx
);
2650 case GIMPLE_OMP_SECTIONS
:
2651 scan_omp_sections (stmt
, ctx
);
2654 case GIMPLE_OMP_SINGLE
:
2655 scan_omp_single (stmt
, ctx
);
2658 case GIMPLE_OMP_SECTION
:
2659 case GIMPLE_OMP_MASTER
:
2660 case GIMPLE_OMP_TASKGROUP
:
2661 case GIMPLE_OMP_ORDERED
:
2662 case GIMPLE_OMP_CRITICAL
:
2663 ctx
= new_omp_context (stmt
, ctx
);
2664 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2667 case GIMPLE_OMP_TARGET
:
2668 scan_omp_target (stmt
, ctx
);
2671 case GIMPLE_OMP_TEAMS
:
2672 scan_omp_teams (stmt
, ctx
);
2679 *handled_ops_p
= false;
2681 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2682 insert_decl_map (&ctx
->cb
, var
, var
);
2686 *handled_ops_p
= false;
2694 /* Scan all the statements starting at the current statement. CTX
2695 contains context information about the OpenMP directives and
2696 clauses found during the scan. */
2699 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2701 location_t saved_location
;
2702 struct walk_stmt_info wi
;
2704 memset (&wi
, 0, sizeof (wi
));
2706 wi
.want_locations
= true;
2708 saved_location
= input_location
;
2709 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2710 input_location
= saved_location
;
2713 /* Re-gimplification and code generation routines. */
2715 /* Build a call to GOMP_barrier. */
2718 build_omp_barrier (tree lhs
)
2720 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2721 : BUILT_IN_GOMP_BARRIER
);
2722 gimple g
= gimple_build_call (fndecl
, 0);
2724 gimple_call_set_lhs (g
, lhs
);
2728 /* If a context was created for STMT when it was scanned, return it. */
2730 static omp_context
*
2731 maybe_lookup_ctx (gimple stmt
)
2734 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2735 return n
? (omp_context
*) n
->value
: NULL
;
2739 /* Find the mapping for DECL in CTX or the immediately enclosing
2740 context that has a mapping for DECL.
2742 If CTX is a nested parallel directive, we may have to use the decl
2743 mappings created in CTX's parent context. Suppose that we have the
2744 following parallel nesting (variable UIDs showed for clarity):
2747 #omp parallel shared(iD.1562) -> outer parallel
2748 iD.1562 = iD.1562 + 1;
2750 #omp parallel shared (iD.1562) -> inner parallel
2751 iD.1562 = iD.1562 - 1;
2753 Each parallel structure will create a distinct .omp_data_s structure
2754 for copying iD.1562 in/out of the directive:
2756 outer parallel .omp_data_s.1.i -> iD.1562
2757 inner parallel .omp_data_s.2.i -> iD.1562
2759 A shared variable mapping will produce a copy-out operation before
2760 the parallel directive and a copy-in operation after it. So, in
2761 this case we would have:
2764 .omp_data_o.1.i = iD.1562;
2765 #omp parallel shared(iD.1562) -> outer parallel
2766 .omp_data_i.1 = &.omp_data_o.1
2767 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2769 .omp_data_o.2.i = iD.1562; -> **
2770 #omp parallel shared(iD.1562) -> inner parallel
2771 .omp_data_i.2 = &.omp_data_o.2
2772 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2775 ** This is a problem. The symbol iD.1562 cannot be referenced
2776 inside the body of the outer parallel region. But since we are
2777 emitting this copy operation while expanding the inner parallel
2778 directive, we need to access the CTX structure of the outer
2779 parallel directive to get the correct mapping:
2781 .omp_data_o.2.i = .omp_data_i.1->i
2783 Since there may be other workshare or parallel directives enclosing
2784 the parallel directive, it may be necessary to walk up the context
2785 parent chain. This is not a problem in general because nested
2786 parallelism happens only rarely. */
2789 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2794 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2795 t
= maybe_lookup_decl (decl
, up
);
2797 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2799 return t
? t
: decl
;
2803 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2804 in outer contexts. */
2807 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2812 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2813 t
= maybe_lookup_decl (decl
, up
);
2815 return t
? t
: decl
;
2819 /* Construct the initialization value for reduction CLAUSE. */
2822 omp_reduction_init (tree clause
, tree type
)
2824 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2825 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2832 case TRUTH_ORIF_EXPR
:
2833 case TRUTH_XOR_EXPR
:
2835 return build_zero_cst (type
);
2838 case TRUTH_AND_EXPR
:
2839 case TRUTH_ANDIF_EXPR
:
2841 return fold_convert_loc (loc
, type
, integer_one_node
);
2844 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2847 if (SCALAR_FLOAT_TYPE_P (type
))
2849 REAL_VALUE_TYPE max
, min
;
2850 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2853 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2856 real_maxval (&min
, 1, TYPE_MODE (type
));
2857 return build_real (type
, min
);
2861 gcc_assert (INTEGRAL_TYPE_P (type
));
2862 return TYPE_MIN_VALUE (type
);
2866 if (SCALAR_FLOAT_TYPE_P (type
))
2868 REAL_VALUE_TYPE max
;
2869 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2872 real_maxval (&max
, 0, TYPE_MODE (type
));
2873 return build_real (type
, max
);
2877 gcc_assert (INTEGRAL_TYPE_P (type
));
2878 return TYPE_MAX_VALUE (type
);
2886 /* Return alignment to be assumed for var in CLAUSE, which should be
2887 OMP_CLAUSE_ALIGNED. */
2890 omp_clause_aligned_alignment (tree clause
)
2892 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2893 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2895 /* Otherwise return implementation defined alignment. */
2896 unsigned int al
= 1;
2897 enum machine_mode mode
, vmode
;
2898 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2900 vs
= 1 << floor_log2 (vs
);
2901 static enum mode_class classes
[]
2902 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2903 for (int i
= 0; i
< 4; i
+= 2)
2904 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2906 mode
= GET_MODE_WIDER_MODE (mode
))
2908 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2909 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2912 && GET_MODE_SIZE (vmode
) < vs
2913 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2914 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2916 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2917 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2919 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2920 / GET_MODE_SIZE (mode
));
2921 if (TYPE_MODE (type
) != vmode
)
2923 if (TYPE_ALIGN_UNIT (type
) > al
)
2924 al
= TYPE_ALIGN_UNIT (type
);
2926 return build_int_cst (integer_type_node
, al
);
2929 /* Return maximum possible vectorization factor for the target. */
2936 || (!flag_tree_loop_vectorize
2937 && (global_options_set
.x_flag_tree_loop_vectorize
2938 || global_options_set
.x_flag_tree_vectorize
)))
2941 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2944 vs
= 1 << floor_log2 (vs
);
2947 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2948 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2949 return GET_MODE_NUNITS (vqimode
);
2953 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2957 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2958 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2962 max_vf
= omp_max_vf ();
2965 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2966 OMP_CLAUSE_SAFELEN
);
2968 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
), max_vf
) == -1)
2969 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
2973 idx
= create_tmp_var (unsigned_type_node
, NULL
);
2974 lane
= create_tmp_var (unsigned_type_node
, NULL
);
2980 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
2981 tree avar
= create_tmp_var_raw (atype
, NULL
);
2982 if (TREE_ADDRESSABLE (new_var
))
2983 TREE_ADDRESSABLE (avar
) = 1;
2984 DECL_ATTRIBUTES (avar
)
2985 = tree_cons (get_identifier ("omp simd array"), NULL
,
2986 DECL_ATTRIBUTES (avar
));
2987 gimple_add_tmp_var (avar
);
2988 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
2989 NULL_TREE
, NULL_TREE
);
2990 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
2991 NULL_TREE
, NULL_TREE
);
2992 if (DECL_P (new_var
))
2994 SET_DECL_VALUE_EXPR (new_var
, lvar
);
2995 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3000 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3001 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3002 private variables. Initialization statements go in ILIST, while calls
3003 to destructors go in DLIST. */
3006 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3007 omp_context
*ctx
, struct omp_for_data
*fd
)
3009 tree c
, dtor
, copyin_seq
, x
, ptr
;
3010 bool copyin_by_ref
= false;
3011 bool lastprivate_firstprivate
= false;
3012 bool reduction_omp_orig_ref
= false;
3014 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3015 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
);
3017 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3018 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3019 gimple_seq llist
[2] = { NULL
, NULL
};
3023 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3024 with data sharing clauses referencing variable sized vars. That
3025 is unnecessarily hard to support and very unlikely to result in
3026 vectorized code anyway. */
3028 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3029 switch (OMP_CLAUSE_CODE (c
))
3031 case OMP_CLAUSE_REDUCTION
:
3032 case OMP_CLAUSE_PRIVATE
:
3033 case OMP_CLAUSE_FIRSTPRIVATE
:
3034 case OMP_CLAUSE_LASTPRIVATE
:
3035 case OMP_CLAUSE_LINEAR
:
3036 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3043 /* Do all the fixed sized types in the first pass, and the variable sized
3044 types in the second pass. This makes sure that the scalar arguments to
3045 the variable sized types are processed before we use them in the
3046 variable sized operations. */
3047 for (pass
= 0; pass
< 2; ++pass
)
3049 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3051 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3054 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3058 case OMP_CLAUSE_PRIVATE
:
3059 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3062 case OMP_CLAUSE_SHARED
:
3063 /* Ignore shared directives in teams construct. */
3064 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3066 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3068 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3071 case OMP_CLAUSE_FIRSTPRIVATE
:
3072 case OMP_CLAUSE_COPYIN
:
3073 case OMP_CLAUSE_LINEAR
:
3075 case OMP_CLAUSE_REDUCTION
:
3076 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3077 reduction_omp_orig_ref
= true;
3079 case OMP_CLAUSE__LOOPTEMP_
:
3080 /* Handle _looptemp_ clauses only on parallel. */
3084 case OMP_CLAUSE_LASTPRIVATE
:
3085 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3087 lastprivate_firstprivate
= true;
3092 case OMP_CLAUSE_ALIGNED
:
3095 var
= OMP_CLAUSE_DECL (c
);
3096 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3097 && !is_global_var (var
))
3099 new_var
= maybe_lookup_decl (var
, ctx
);
3100 if (new_var
== NULL_TREE
)
3101 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3102 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3103 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3104 omp_clause_aligned_alignment (c
));
3105 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3106 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3107 gimplify_and_add (x
, ilist
);
3109 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3110 && is_global_var (var
))
3112 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3113 new_var
= lookup_decl (var
, ctx
);
3114 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3115 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3116 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3117 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3118 omp_clause_aligned_alignment (c
));
3119 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3120 x
= create_tmp_var (ptype
, NULL
);
3121 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3122 gimplify_and_add (t
, ilist
);
3123 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3124 SET_DECL_VALUE_EXPR (new_var
, t
);
3125 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3132 new_var
= var
= OMP_CLAUSE_DECL (c
);
3133 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3134 new_var
= lookup_decl (var
, ctx
);
3136 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3141 else if (is_variable_sized (var
))
3143 /* For variable sized types, we need to allocate the
3144 actual storage here. Call alloca and store the
3145 result in the pointer decl that we created elsewhere. */
3149 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3154 ptr
= DECL_VALUE_EXPR (new_var
);
3155 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3156 ptr
= TREE_OPERAND (ptr
, 0);
3157 gcc_assert (DECL_P (ptr
));
3158 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3160 /* void *tmp = __builtin_alloca */
3161 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3162 stmt
= gimple_build_call (atmp
, 1, x
);
3163 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3164 gimple_add_tmp_var (tmp
);
3165 gimple_call_set_lhs (stmt
, tmp
);
3167 gimple_seq_add_stmt (ilist
, stmt
);
3169 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3170 gimplify_assign (ptr
, x
, ilist
);
3173 else if (is_reference (var
))
3175 /* For references that are being privatized for Fortran,
3176 allocate new backing storage for the new pointer
3177 variable. This allows us to avoid changing all the
3178 code that expects a pointer to something that expects
3179 a direct variable. */
3183 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3184 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3186 x
= build_receiver_ref (var
, false, ctx
);
3187 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3189 else if (TREE_CONSTANT (x
))
3191 /* For reduction with placeholder in SIMD loop,
3192 defer adding the initialization of the reference,
3193 because if we decide to use SIMD array for it,
3194 the initilization could cause expansion ICE. */
3195 if (c_kind
== OMP_CLAUSE_REDUCTION
3196 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
3201 const char *name
= NULL
;
3202 if (DECL_NAME (var
))
3203 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3205 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3207 gimple_add_tmp_var (x
);
3208 TREE_ADDRESSABLE (x
) = 1;
3209 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3214 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3215 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3220 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3221 gimplify_assign (new_var
, x
, ilist
);
3224 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3226 else if (c_kind
== OMP_CLAUSE_REDUCTION
3227 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3235 switch (OMP_CLAUSE_CODE (c
))
3237 case OMP_CLAUSE_SHARED
:
3238 /* Ignore shared directives in teams construct. */
3239 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3241 /* Shared global vars are just accessed directly. */
3242 if (is_global_var (new_var
))
3244 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3245 needs to be delayed until after fixup_child_record_type so
3246 that we get the correct type during the dereference. */
3247 by_ref
= use_pointer_for_field (var
, ctx
);
3248 x
= build_receiver_ref (var
, by_ref
, ctx
);
3249 SET_DECL_VALUE_EXPR (new_var
, x
);
3250 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3252 /* ??? If VAR is not passed by reference, and the variable
3253 hasn't been initialized yet, then we'll get a warning for
3254 the store into the omp_data_s structure. Ideally, we'd be
3255 able to notice this and not store anything at all, but
3256 we're generating code too early. Suppress the warning. */
3258 TREE_NO_WARNING (var
) = 1;
3261 case OMP_CLAUSE_LASTPRIVATE
:
3262 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3266 case OMP_CLAUSE_PRIVATE
:
3267 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3268 x
= build_outer_var_ref (var
, ctx
);
3269 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3271 if (is_task_ctx (ctx
))
3272 x
= build_receiver_ref (var
, false, ctx
);
3274 x
= build_outer_var_ref (var
, ctx
);
3280 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3283 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3284 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3285 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3286 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3287 idx
, lane
, ivar
, lvar
))
3290 x
= lang_hooks
.decls
.omp_clause_default_ctor
3291 (c
, unshare_expr (ivar
), x
);
3293 gimplify_and_add (x
, &llist
[0]);
3296 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3299 gimple_seq tseq
= NULL
;
3302 gimplify_stmt (&dtor
, &tseq
);
3303 gimple_seq_add_seq (&llist
[1], tseq
);
3310 gimplify_and_add (nx
, ilist
);
3314 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3317 gimple_seq tseq
= NULL
;
3320 gimplify_stmt (&dtor
, &tseq
);
3321 gimple_seq_add_seq (dlist
, tseq
);
3325 case OMP_CLAUSE_LINEAR
:
3326 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3327 goto do_firstprivate
;
3328 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3331 x
= build_outer_var_ref (var
, ctx
);
3334 case OMP_CLAUSE_FIRSTPRIVATE
:
3335 if (is_task_ctx (ctx
))
3337 if (is_reference (var
) || is_variable_sized (var
))
3339 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3341 || use_pointer_for_field (var
, NULL
))
3343 x
= build_receiver_ref (var
, false, ctx
);
3344 SET_DECL_VALUE_EXPR (new_var
, x
);
3345 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3350 x
= build_outer_var_ref (var
, ctx
);
3353 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3354 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3356 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3357 ? sizetype
: TREE_TYPE (x
);
3358 tree t
= fold_convert (stept
,
3359 OMP_CLAUSE_LINEAR_STEP (c
));
3360 tree c
= find_omp_clause (clauses
,
3361 OMP_CLAUSE__LOOPTEMP_
);
3363 tree l
= OMP_CLAUSE_DECL (c
);
3364 if (fd
->collapse
== 1)
3366 tree n1
= fd
->loop
.n1
;
3367 tree step
= fd
->loop
.step
;
3368 tree itype
= TREE_TYPE (l
);
3369 if (POINTER_TYPE_P (itype
))
3370 itype
= signed_type_for (itype
);
3371 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3372 if (TYPE_UNSIGNED (itype
)
3373 && fd
->loop
.cond_code
== GT_EXPR
)
3374 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3375 fold_build1 (NEGATE_EXPR
,
3377 fold_build1 (NEGATE_EXPR
,
3380 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3382 t
= fold_build2 (MULT_EXPR
, stept
,
3383 fold_convert (stept
, l
), t
);
3384 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3385 x
= fold_build2 (POINTER_PLUS_EXPR
,
3386 TREE_TYPE (x
), x
, t
);
3388 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3391 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3392 || TREE_ADDRESSABLE (new_var
))
3393 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3394 idx
, lane
, ivar
, lvar
))
3396 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3398 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3399 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3400 gimplify_and_add (x
, ilist
);
3401 gimple_stmt_iterator gsi
3402 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3404 = gimple_build_assign (unshare_expr (lvar
), iv
);
3405 gsi_insert_before_without_update (&gsi
, g
,
3407 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3408 ? sizetype
: TREE_TYPE (x
);
3409 tree t
= fold_convert (stept
,
3410 OMP_CLAUSE_LINEAR_STEP (c
));
3411 enum tree_code code
= PLUS_EXPR
;
3412 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3413 code
= POINTER_PLUS_EXPR
;
3414 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3415 gsi_insert_before_without_update (&gsi
, g
,
3419 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3420 (c
, unshare_expr (ivar
), x
);
3421 gimplify_and_add (x
, &llist
[0]);
3422 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3425 gimple_seq tseq
= NULL
;
3428 gimplify_stmt (&dtor
, &tseq
);
3429 gimple_seq_add_seq (&llist
[1], tseq
);
3434 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3435 gimplify_and_add (x
, ilist
);
3438 case OMP_CLAUSE__LOOPTEMP_
:
3439 gcc_assert (is_parallel_ctx (ctx
));
3440 x
= build_outer_var_ref (var
, ctx
);
3441 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3442 gimplify_and_add (x
, ilist
);
3445 case OMP_CLAUSE_COPYIN
:
3446 by_ref
= use_pointer_for_field (var
, NULL
);
3447 x
= build_receiver_ref (var
, by_ref
, ctx
);
3448 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3449 append_to_statement_list (x
, ©in_seq
);
3450 copyin_by_ref
|= by_ref
;
3453 case OMP_CLAUSE_REDUCTION
:
3454 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3456 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3458 x
= build_outer_var_ref (var
, ctx
);
3460 if (is_reference (var
)
3461 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3463 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3464 SET_DECL_VALUE_EXPR (placeholder
, x
);
3465 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3466 tree new_vard
= new_var
;
3467 if (is_reference (var
))
3469 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3470 new_vard
= TREE_OPERAND (new_var
, 0);
3471 gcc_assert (DECL_P (new_vard
));
3474 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3475 idx
, lane
, ivar
, lvar
))
3477 if (new_vard
== new_var
)
3479 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3480 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3484 SET_DECL_VALUE_EXPR (new_vard
,
3485 build_fold_addr_expr (ivar
));
3486 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3488 x
= lang_hooks
.decls
.omp_clause_default_ctor
3489 (c
, unshare_expr (ivar
),
3490 build_outer_var_ref (var
, ctx
));
3492 gimplify_and_add (x
, &llist
[0]);
3493 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3495 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3496 lower_omp (&tseq
, ctx
);
3497 gimple_seq_add_seq (&llist
[0], tseq
);
3499 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3500 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3501 lower_omp (&tseq
, ctx
);
3502 gimple_seq_add_seq (&llist
[1], tseq
);
3503 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3504 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3505 if (new_vard
== new_var
)
3506 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3508 SET_DECL_VALUE_EXPR (new_vard
,
3509 build_fold_addr_expr (lvar
));
3510 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3515 gimplify_stmt (&dtor
, &tseq
);
3516 gimple_seq_add_seq (&llist
[1], tseq
);
3520 /* If this is a reference to constant size reduction var
3521 with placeholder, we haven't emitted the initializer
3522 for it because it is undesirable if SIMD arrays are used.
3523 But if they aren't used, we need to emit the deferred
3524 initialization now. */
3525 else if (is_reference (var
) && is_simd
)
3528 = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3529 if (TREE_CONSTANT (z
))
3531 const char *name
= NULL
;
3532 if (DECL_NAME (var
))
3533 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3535 z
= create_tmp_var_raw
3536 (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3537 gimple_add_tmp_var (z
);
3538 TREE_ADDRESSABLE (z
) = 1;
3539 z
= build_fold_addr_expr_loc (clause_loc
, z
);
3540 gimplify_assign (new_vard
, z
, ilist
);
3543 x
= lang_hooks
.decls
.omp_clause_default_ctor
3544 (c
, new_var
, unshare_expr (x
));
3546 gimplify_and_add (x
, ilist
);
3547 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3549 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3550 lower_omp (&tseq
, ctx
);
3551 gimple_seq_add_seq (ilist
, tseq
);
3553 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3556 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3557 lower_omp (&tseq
, ctx
);
3558 gimple_seq_add_seq (dlist
, tseq
);
3559 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3561 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3566 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3567 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3568 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3570 /* reduction(-:var) sums up the partial results, so it
3571 acts identically to reduction(+:var). */
3572 if (code
== MINUS_EXPR
)
3576 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3577 idx
, lane
, ivar
, lvar
))
3579 tree ref
= build_outer_var_ref (var
, ctx
);
3581 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3583 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3584 ref
= build_outer_var_ref (var
, ctx
);
3585 gimplify_assign (ref
, x
, &llist
[1]);
3589 gimplify_assign (new_var
, x
, ilist
);
3592 tree ref
= build_outer_var_ref (var
, ctx
);
3594 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3595 ref
= build_outer_var_ref (var
, ctx
);
3596 gimplify_assign (ref
, x
, dlist
);
3610 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3611 /* Don't want uninit warnings on simduid, it is always uninitialized,
3612 but we use it not for the value, but for the DECL_UID only. */
3613 TREE_NO_WARNING (uid
) = 1;
3615 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3616 gimple_call_set_lhs (g
, lane
);
3617 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3618 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3619 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3620 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3621 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3622 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3623 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3624 build_int_cst (unsigned_type_node
, 0),
3626 gimple_seq_add_stmt (ilist
, g
);
3627 for (int i
= 0; i
< 2; i
++)
3630 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3631 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3632 gimple_call_set_lhs (g
, vf
);
3633 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3634 gimple_seq_add_stmt (seq
, g
);
3635 tree t
= build_int_cst (unsigned_type_node
, 0);
3636 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3637 gimple_seq_add_stmt (seq
, g
);
3638 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3639 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3640 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3641 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3642 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3643 gimple_seq_add_seq (seq
, llist
[i
]);
3644 t
= build_int_cst (unsigned_type_node
, 1);
3645 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3646 gimple_seq_add_stmt (seq
, g
);
3647 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3648 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3649 gimple_seq_add_stmt (seq
, g
);
3650 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3654 /* The copyin sequence is not to be executed by the main thread, since
3655 that would result in self-copies. Perhaps not visible to scalars,
3656 but it certainly is to C++ operator=. */
3659 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3661 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3662 build_int_cst (TREE_TYPE (x
), 0));
3663 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3664 gimplify_and_add (x
, ilist
);
3667 /* If any copyin variable is passed by reference, we must ensure the
3668 master thread doesn't modify it before it is copied over in all
3669 threads. Similarly for variables in both firstprivate and
3670 lastprivate clauses we need to ensure the lastprivate copying
3671 happens after firstprivate copying in all threads. And similarly
3672 for UDRs if initializer expression refers to omp_orig. */
3673 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3675 /* Don't add any barrier for #pragma omp simd or
3676 #pragma omp distribute. */
3677 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3678 || gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_FOR
)
3679 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3682 /* If max_vf is non-zero, then we can use only a vectorization factor
3683 up to the max_vf we chose. So stick it into the safelen clause. */
3686 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3687 OMP_CLAUSE_SAFELEN
);
3689 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3692 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3693 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3695 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3696 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3702 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3703 both parallel and workshare constructs. PREDICATE may be NULL if it's
3707 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3710 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3711 bool par_clauses
= false;
3712 tree simduid
= NULL
, lastlane
= NULL
;
3714 /* Early exit if there are no lastprivate or linear clauses. */
3715 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3716 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3717 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3718 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3720 if (clauses
== NULL
)
3722 /* If this was a workshare clause, see if it had been combined
3723 with its parallel. In that case, look for the clauses on the
3724 parallel statement itself. */
3725 if (is_parallel_ctx (ctx
))
3729 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3732 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3733 OMP_CLAUSE_LASTPRIVATE
);
3734 if (clauses
== NULL
)
3742 tree label_true
, arm1
, arm2
;
3744 label
= create_artificial_label (UNKNOWN_LOCATION
);
3745 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3746 arm1
= TREE_OPERAND (predicate
, 0);
3747 arm2
= TREE_OPERAND (predicate
, 1);
3748 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3749 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3750 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3752 gimple_seq_add_stmt (stmt_list
, stmt
);
3753 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3756 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3757 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3759 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3761 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3764 for (c
= clauses
; c
;)
3767 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3769 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3770 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3771 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3773 var
= OMP_CLAUSE_DECL (c
);
3774 new_var
= lookup_decl (var
, ctx
);
3776 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3778 tree val
= DECL_VALUE_EXPR (new_var
);
3779 if (TREE_CODE (val
) == ARRAY_REF
3780 && VAR_P (TREE_OPERAND (val
, 0))
3781 && lookup_attribute ("omp simd array",
3782 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3785 if (lastlane
== NULL
)
3787 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3789 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3791 TREE_OPERAND (val
, 1));
3792 gimple_call_set_lhs (g
, lastlane
);
3793 gimple_seq_add_stmt (stmt_list
, g
);
3795 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3796 TREE_OPERAND (val
, 0), lastlane
,
3797 NULL_TREE
, NULL_TREE
);
3801 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3802 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3804 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3805 gimple_seq_add_seq (stmt_list
,
3806 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3807 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3810 x
= build_outer_var_ref (var
, ctx
);
3811 if (is_reference (var
))
3812 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3813 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3814 gimplify_and_add (x
, stmt_list
);
3816 c
= OMP_CLAUSE_CHAIN (c
);
3817 if (c
== NULL
&& !par_clauses
)
3819 /* If this was a workshare clause, see if it had been combined
3820 with its parallel. In that case, continue looking for the
3821 clauses also on the parallel statement itself. */
3822 if (is_parallel_ctx (ctx
))
3826 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3829 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3830 OMP_CLAUSE_LASTPRIVATE
);
3836 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3840 /* Generate code to implement the REDUCTION clauses. */
3843 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3845 gimple_seq sub_seq
= NULL
;
3850 /* SIMD reductions are handled in lower_rec_input_clauses. */
3851 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3852 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3855 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3856 update in that case, otherwise use a lock. */
3857 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3858 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3860 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3862 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3872 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3874 tree var
, ref
, new_var
;
3875 enum tree_code code
;
3876 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3878 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3881 var
= OMP_CLAUSE_DECL (c
);
3882 new_var
= lookup_decl (var
, ctx
);
3883 if (is_reference (var
))
3884 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3885 ref
= build_outer_var_ref (var
, ctx
);
3886 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3888 /* reduction(-:var) sums up the partial results, so it acts
3889 identically to reduction(+:var). */
3890 if (code
== MINUS_EXPR
)
3895 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3897 addr
= save_expr (addr
);
3898 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3899 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3900 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3901 gimplify_and_add (x
, stmt_seqp
);
3905 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3907 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3909 if (is_reference (var
)
3910 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3912 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3913 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3914 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3915 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3916 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3917 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3918 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3922 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3923 ref
= build_outer_var_ref (var
, ctx
);
3924 gimplify_assign (ref
, x
, &sub_seq
);
3928 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
3930 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3932 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
3934 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
3936 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3940 /* Generate code to implement the COPYPRIVATE clauses. */
3943 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
3948 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3950 tree var
, new_var
, ref
, x
;
3952 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3954 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
3957 var
= OMP_CLAUSE_DECL (c
);
3958 by_ref
= use_pointer_for_field (var
, NULL
);
3960 ref
= build_sender_ref (var
, ctx
);
3961 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
3964 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
3965 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
3967 gimplify_assign (ref
, x
, slist
);
3969 ref
= build_receiver_ref (var
, false, ctx
);
3972 ref
= fold_convert_loc (clause_loc
,
3973 build_pointer_type (TREE_TYPE (new_var
)),
3975 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
3977 if (is_reference (var
))
3979 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
3980 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
3981 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3983 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
3984 gimplify_and_add (x
, rlist
);
3989 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3990 and REDUCTION from the sender (aka parent) side. */
3993 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
3998 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4000 tree val
, ref
, x
, var
;
4001 bool by_ref
, do_in
= false, do_out
= false;
4002 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4004 switch (OMP_CLAUSE_CODE (c
))
4006 case OMP_CLAUSE_PRIVATE
:
4007 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4010 case OMP_CLAUSE_FIRSTPRIVATE
:
4011 case OMP_CLAUSE_COPYIN
:
4012 case OMP_CLAUSE_LASTPRIVATE
:
4013 case OMP_CLAUSE_REDUCTION
:
4014 case OMP_CLAUSE__LOOPTEMP_
:
4020 val
= OMP_CLAUSE_DECL (c
);
4021 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4023 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4024 && is_global_var (var
))
4026 if (is_variable_sized (val
))
4028 by_ref
= use_pointer_for_field (val
, NULL
);
4030 switch (OMP_CLAUSE_CODE (c
))
4032 case OMP_CLAUSE_PRIVATE
:
4033 case OMP_CLAUSE_FIRSTPRIVATE
:
4034 case OMP_CLAUSE_COPYIN
:
4035 case OMP_CLAUSE__LOOPTEMP_
:
4039 case OMP_CLAUSE_LASTPRIVATE
:
4040 if (by_ref
|| is_reference (val
))
4042 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4049 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4054 case OMP_CLAUSE_REDUCTION
:
4056 do_out
= !(by_ref
|| is_reference (val
));
4065 ref
= build_sender_ref (val
, ctx
);
4066 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4067 gimplify_assign (ref
, x
, ilist
);
4068 if (is_task_ctx (ctx
))
4069 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4074 ref
= build_sender_ref (val
, ctx
);
4075 gimplify_assign (var
, ref
, olist
);
4080 /* Generate code to implement SHARED from the sender (aka parent)
4081 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4082 list things that got automatically shared. */
4085 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4087 tree var
, ovar
, nvar
, f
, x
, record_type
;
4089 if (ctx
->record_type
== NULL
)
4092 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4093 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4095 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4096 nvar
= maybe_lookup_decl (ovar
, ctx
);
4097 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4100 /* If CTX is a nested parallel directive. Find the immediately
4101 enclosing parallel or workshare construct that contains a
4102 mapping for OVAR. */
4103 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4105 if (use_pointer_for_field (ovar
, ctx
))
4107 x
= build_sender_ref (ovar
, ctx
);
4108 var
= build_fold_addr_expr (var
);
4109 gimplify_assign (x
, var
, ilist
);
4113 x
= build_sender_ref (ovar
, ctx
);
4114 gimplify_assign (x
, var
, ilist
);
4116 if (!TREE_READONLY (var
)
4117 /* We don't need to receive a new reference to a result
4118 or parm decl. In fact we may not store to it as we will
4119 invalidate any pending RSO and generate wrong gimple
4121 && !((TREE_CODE (var
) == RESULT_DECL
4122 || TREE_CODE (var
) == PARM_DECL
)
4123 && DECL_BY_REFERENCE (var
)))
4125 x
= build_sender_ref (ovar
, ctx
);
4126 gimplify_assign (var
, x
, olist
);
4133 /* A convenience function to build an empty GIMPLE_COND with just the
4137 gimple_build_cond_empty (tree cond
)
4139 enum tree_code pred_code
;
4142 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4143 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4147 /* Build the function calls to GOMP_parallel_start etc to actually
4148 generate the parallel operation. REGION is the parallel region
4149 being expanded. BB is the block where to insert the code. WS_ARGS
4150 will be set if this is a call to a combined parallel+workshare
4151 construct, it contains the list of additional arguments needed by
4152 the workshare construct. */
4155 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4156 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4158 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4159 gimple_stmt_iterator gsi
;
4161 enum built_in_function start_ix
;
4163 location_t clause_loc
;
4164 vec
<tree
, va_gc
> *args
;
4166 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4168 /* Determine what flavor of GOMP_parallel we will be
4170 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4171 if (is_combined_parallel (region
))
4173 switch (region
->inner
->type
)
4175 case GIMPLE_OMP_FOR
:
4176 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4177 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4178 + (region
->inner
->sched_kind
4179 == OMP_CLAUSE_SCHEDULE_RUNTIME
4180 ? 3 : region
->inner
->sched_kind
));
4181 start_ix
= (enum built_in_function
)start_ix2
;
4183 case GIMPLE_OMP_SECTIONS
:
4184 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4191 /* By default, the value of NUM_THREADS is zero (selected at run time)
4192 and there is no conditional. */
4194 val
= build_int_cst (unsigned_type_node
, 0);
4195 flags
= build_int_cst (unsigned_type_node
, 0);
4197 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4199 cond
= OMP_CLAUSE_IF_EXPR (c
);
4201 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4204 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4205 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4208 clause_loc
= gimple_location (entry_stmt
);
4210 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4212 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4214 /* Ensure 'val' is of the correct type. */
4215 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4217 /* If we found the clause 'if (cond)', build either
4218 (cond != 0) or (cond ? val : 1u). */
4221 gimple_stmt_iterator gsi
;
4223 cond
= gimple_boolify (cond
);
4225 if (integer_zerop (val
))
4226 val
= fold_build2_loc (clause_loc
,
4227 EQ_EXPR
, unsigned_type_node
, cond
,
4228 build_int_cst (TREE_TYPE (cond
), 0));
4231 basic_block cond_bb
, then_bb
, else_bb
;
4232 edge e
, e_then
, e_else
;
4233 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4235 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4236 if (gimple_in_ssa_p (cfun
))
4238 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4239 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4240 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4249 e
= split_block (bb
, NULL
);
4254 then_bb
= create_empty_bb (cond_bb
);
4255 else_bb
= create_empty_bb (then_bb
);
4256 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4257 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4259 stmt
= gimple_build_cond_empty (cond
);
4260 gsi
= gsi_start_bb (cond_bb
);
4261 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4263 gsi
= gsi_start_bb (then_bb
);
4264 stmt
= gimple_build_assign (tmp_then
, val
);
4265 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4267 gsi
= gsi_start_bb (else_bb
);
4268 stmt
= gimple_build_assign
4269 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4270 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4272 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4273 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4276 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4277 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4279 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4280 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4282 if (gimple_in_ssa_p (cfun
))
4284 gimple phi
= create_phi_node (tmp_join
, bb
);
4285 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4286 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4292 gsi
= gsi_start_bb (bb
);
4293 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4294 false, GSI_CONTINUE_LINKING
);
4297 gsi
= gsi_last_bb (bb
);
4298 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4300 t1
= null_pointer_node
;
4302 t1
= build_fold_addr_expr (t
);
4303 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4305 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4306 args
->quick_push (t2
);
4307 args
->quick_push (t1
);
4308 args
->quick_push (val
);
4310 args
->splice (*ws_args
);
4311 args
->quick_push (flags
);
4313 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4314 builtin_decl_explicit (start_ix
), args
);
4316 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4317 false, GSI_CONTINUE_LINKING
);
4321 /* Build the function call to GOMP_task to actually
4322 generate the task operation. BB is the block where to insert the code. */
4325 expand_task_call (basic_block bb
, gimple entry_stmt
)
4327 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4328 gimple_stmt_iterator gsi
;
4329 location_t loc
= gimple_location (entry_stmt
);
4331 clauses
= gimple_omp_task_clauses (entry_stmt
);
4333 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4335 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4337 cond
= boolean_true_node
;
4339 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4340 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4341 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4342 flags
= build_int_cst (unsigned_type_node
,
4343 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4345 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4348 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4349 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4350 build_int_cst (unsigned_type_node
, 2),
4351 build_int_cst (unsigned_type_node
, 0));
4352 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4355 depend
= OMP_CLAUSE_DECL (depend
);
4357 depend
= build_int_cst (ptr_type_node
, 0);
4359 gsi
= gsi_last_bb (bb
);
4360 t
= gimple_omp_task_data_arg (entry_stmt
);
4362 t2
= null_pointer_node
;
4364 t2
= build_fold_addr_expr_loc (loc
, t
);
4365 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4366 t
= gimple_omp_task_copy_fn (entry_stmt
);
4368 t3
= null_pointer_node
;
4370 t3
= build_fold_addr_expr_loc (loc
, t
);
4372 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4374 gimple_omp_task_arg_size (entry_stmt
),
4375 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4378 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4379 false, GSI_CONTINUE_LINKING
);
4383 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4384 catch handler and return it. This prevents programs from violating the
4385 structured block semantics with throws. */
4388 maybe_catch_exception (gimple_seq body
)
4393 if (!flag_exceptions
)
4396 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4397 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4399 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4401 g
= gimple_build_eh_must_not_throw (decl
);
4402 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4405 return gimple_seq_alloc_with_stmt (g
);
4408 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4411 vec2chain (vec
<tree
, va_gc
> *v
)
4413 tree chain
= NULL_TREE
, t
;
4416 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4418 DECL_CHAIN (t
) = chain
;
4426 /* Remove barriers in REGION->EXIT's block. Note that this is only
4427 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4428 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4429 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4433 remove_exit_barrier (struct omp_region
*region
)
4435 gimple_stmt_iterator gsi
;
4436 basic_block exit_bb
;
4440 int any_addressable_vars
= -1;
4442 exit_bb
= region
->exit
;
4444 /* If the parallel region doesn't return, we don't have REGION->EXIT
4449 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4450 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4451 statements that can appear in between are extremely limited -- no
4452 memory operations at all. Here, we allow nothing at all, so the
4453 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4454 gsi
= gsi_last_bb (exit_bb
);
4455 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4457 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4460 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4462 gsi
= gsi_last_bb (e
->src
);
4463 if (gsi_end_p (gsi
))
4465 stmt
= gsi_stmt (gsi
);
4466 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4467 && !gimple_omp_return_nowait_p (stmt
))
4469 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4470 in many cases. If there could be tasks queued, the barrier
4471 might be needed to let the tasks run before some local
4472 variable of the parallel that the task uses as shared
4473 runs out of scope. The task can be spawned either
4474 from within current function (this would be easy to check)
4475 or from some function it calls and gets passed an address
4476 of such a variable. */
4477 if (any_addressable_vars
< 0)
4479 gimple parallel_stmt
= last_stmt (region
->entry
);
4480 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4481 tree local_decls
, block
, decl
;
4484 any_addressable_vars
= 0;
4485 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4486 if (TREE_ADDRESSABLE (decl
))
4488 any_addressable_vars
= 1;
4491 for (block
= gimple_block (stmt
);
4492 !any_addressable_vars
4494 && TREE_CODE (block
) == BLOCK
;
4495 block
= BLOCK_SUPERCONTEXT (block
))
4497 for (local_decls
= BLOCK_VARS (block
);
4499 local_decls
= DECL_CHAIN (local_decls
))
4500 if (TREE_ADDRESSABLE (local_decls
))
4502 any_addressable_vars
= 1;
4505 if (block
== gimple_block (parallel_stmt
))
4509 if (!any_addressable_vars
)
4510 gimple_omp_return_set_nowait (stmt
);
4516 remove_exit_barriers (struct omp_region
*region
)
4518 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4519 remove_exit_barrier (region
);
4523 region
= region
->inner
;
4524 remove_exit_barriers (region
);
4525 while (region
->next
)
4527 region
= region
->next
;
4528 remove_exit_barriers (region
);
4533 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4534 calls. These can't be declared as const functions, but
4535 within one parallel body they are constant, so they can be
4536 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4537 which are declared const. Similarly for task body, except
4538 that in untied task omp_get_thread_num () can change at any task
4539 scheduling point. */
4542 optimize_omp_library_calls (gimple entry_stmt
)
4545 gimple_stmt_iterator gsi
;
4546 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4547 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4548 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4549 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4550 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4551 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4552 OMP_CLAUSE_UNTIED
) != NULL
);
4554 FOR_EACH_BB_FN (bb
, cfun
)
4555 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4557 gimple call
= gsi_stmt (gsi
);
4560 if (is_gimple_call (call
)
4561 && (decl
= gimple_call_fndecl (call
))
4562 && DECL_EXTERNAL (decl
)
4563 && TREE_PUBLIC (decl
)
4564 && DECL_INITIAL (decl
) == NULL
)
4568 if (DECL_NAME (decl
) == thr_num_id
)
4570 /* In #pragma omp task untied omp_get_thread_num () can change
4571 during the execution of the task region. */
4574 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4576 else if (DECL_NAME (decl
) == num_thr_id
)
4577 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4581 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4582 || gimple_call_num_args (call
) != 0)
4585 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4588 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4589 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4590 TREE_TYPE (TREE_TYPE (built_in
))))
4593 gimple_call_set_fndecl (call
, built_in
);
4598 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4602 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4606 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4607 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4610 if (TREE_CODE (t
) == ADDR_EXPR
)
4611 recompute_tree_invariant_for_addr_expr (t
);
4613 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4617 /* Prepend TO = FROM assignment before *GSI_P. */
4620 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4622 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4623 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4624 true, GSI_SAME_STMT
);
4625 gimple stmt
= gimple_build_assign (to
, from
);
4626 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4627 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4628 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4630 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4631 gimple_regimplify_operands (stmt
, &gsi
);
4635 /* Expand the OpenMP parallel or task directive starting at REGION. */
4638 expand_omp_taskreg (struct omp_region
*region
)
4640 basic_block entry_bb
, exit_bb
, new_bb
;
4641 struct function
*child_cfun
;
4642 tree child_fn
, block
, t
;
4643 gimple_stmt_iterator gsi
;
4644 gimple entry_stmt
, stmt
;
4646 vec
<tree
, va_gc
> *ws_args
;
4648 entry_stmt
= last_stmt (region
->entry
);
4649 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4650 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4652 entry_bb
= region
->entry
;
4653 exit_bb
= region
->exit
;
4655 if (is_combined_parallel (region
))
4656 ws_args
= region
->ws_args
;
4660 if (child_cfun
->cfg
)
4662 /* Due to inlining, it may happen that we have already outlined
4663 the region, in which case all we need to do is make the
4664 sub-graph unreachable and emit the parallel call. */
4665 edge entry_succ_e
, exit_succ_e
;
4666 gimple_stmt_iterator gsi
;
4668 entry_succ_e
= single_succ_edge (entry_bb
);
4670 gsi
= gsi_last_bb (entry_bb
);
4671 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4672 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4673 gsi_remove (&gsi
, true);
4678 exit_succ_e
= single_succ_edge (exit_bb
);
4679 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4681 remove_edge_and_dominated_blocks (entry_succ_e
);
4685 unsigned srcidx
, dstidx
, num
;
4687 /* If the parallel region needs data sent from the parent
4688 function, then the very first statement (except possible
4689 tree profile counter updates) of the parallel body
4690 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4691 &.OMP_DATA_O is passed as an argument to the child function,
4692 we need to replace it with the argument as seen by the child
4695 In most cases, this will end up being the identity assignment
4696 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4697 a function call that has been inlined, the original PARM_DECL
4698 .OMP_DATA_I may have been converted into a different local
4699 variable. In which case, we need to keep the assignment. */
4700 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4702 basic_block entry_succ_bb
= single_succ (entry_bb
);
4703 gimple_stmt_iterator gsi
;
4705 gimple parcopy_stmt
= NULL
;
4707 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4711 gcc_assert (!gsi_end_p (gsi
));
4712 stmt
= gsi_stmt (gsi
);
4713 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4716 if (gimple_num_ops (stmt
) == 2)
4718 tree arg
= gimple_assign_rhs1 (stmt
);
4720 /* We're ignore the subcode because we're
4721 effectively doing a STRIP_NOPS. */
4723 if (TREE_CODE (arg
) == ADDR_EXPR
4724 && TREE_OPERAND (arg
, 0)
4725 == gimple_omp_taskreg_data_arg (entry_stmt
))
4727 parcopy_stmt
= stmt
;
4733 gcc_assert (parcopy_stmt
!= NULL
);
4734 arg
= DECL_ARGUMENTS (child_fn
);
4736 if (!gimple_in_ssa_p (cfun
))
4738 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4739 gsi_remove (&gsi
, true);
4742 /* ?? Is setting the subcode really necessary ?? */
4743 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4744 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4749 /* If we are in ssa form, we must load the value from the default
4750 definition of the argument. That should not be defined now,
4751 since the argument is not used uninitialized. */
4752 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4753 narg
= make_ssa_name (arg
, gimple_build_nop ());
4754 set_ssa_default_def (cfun
, arg
, narg
);
4755 /* ?? Is setting the subcode really necessary ?? */
4756 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4757 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4758 update_stmt (parcopy_stmt
);
4762 /* Declare local variables needed in CHILD_CFUN. */
4763 block
= DECL_INITIAL (child_fn
);
4764 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4765 /* The gimplifier could record temporaries in parallel/task block
4766 rather than in containing function's local_decls chain,
4767 which would mean cgraph missed finalizing them. Do it now. */
4768 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4769 if (TREE_CODE (t
) == VAR_DECL
4771 && !DECL_EXTERNAL (t
))
4772 varpool_finalize_decl (t
);
4773 DECL_SAVED_TREE (child_fn
) = NULL
;
4774 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4775 gimple_set_body (child_fn
, NULL
);
4776 TREE_USED (block
) = 1;
4778 /* Reset DECL_CONTEXT on function arguments. */
4779 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4780 DECL_CONTEXT (t
) = child_fn
;
4782 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4783 so that it can be moved to the child function. */
4784 gsi
= gsi_last_bb (entry_bb
);
4785 stmt
= gsi_stmt (gsi
);
4786 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4787 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4788 gsi_remove (&gsi
, true);
4789 e
= split_block (entry_bb
, stmt
);
4791 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4793 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4796 gsi
= gsi_last_bb (exit_bb
);
4797 gcc_assert (!gsi_end_p (gsi
)
4798 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4799 stmt
= gimple_build_return (NULL
);
4800 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4801 gsi_remove (&gsi
, true);
4804 /* Move the parallel region into CHILD_CFUN. */
4806 if (gimple_in_ssa_p (cfun
))
4808 init_tree_ssa (child_cfun
);
4809 init_ssa_operands (child_cfun
);
4810 child_cfun
->gimple_df
->in_ssa_p
= true;
4814 block
= gimple_block (entry_stmt
);
4816 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4818 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4819 /* When the OMP expansion process cannot guarantee an up-to-date
4820 loop tree arrange for the child function to fixup loops. */
4821 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4822 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4824 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4825 num
= vec_safe_length (child_cfun
->local_decls
);
4826 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4828 t
= (*child_cfun
->local_decls
)[srcidx
];
4829 if (DECL_CONTEXT (t
) == cfun
->decl
)
4831 if (srcidx
!= dstidx
)
4832 (*child_cfun
->local_decls
)[dstidx
] = t
;
4836 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4838 /* Inform the callgraph about the new function. */
4839 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4840 cgraph_add_new_function (child_fn
, true);
4842 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4843 fixed in a following pass. */
4844 push_cfun (child_cfun
);
4846 optimize_omp_library_calls (entry_stmt
);
4847 rebuild_cgraph_edges ();
4849 /* Some EH regions might become dead, see PR34608. If
4850 pass_cleanup_cfg isn't the first pass to happen with the
4851 new child, these dead EH edges might cause problems.
4852 Clean them up now. */
4853 if (flag_exceptions
)
4856 bool changed
= false;
4858 FOR_EACH_BB_FN (bb
, cfun
)
4859 changed
|= gimple_purge_dead_eh_edges (bb
);
4861 cleanup_tree_cfg ();
4863 if (gimple_in_ssa_p (cfun
))
4864 update_ssa (TODO_update_ssa
);
4868 /* Emit a library call to launch the children threads. */
4869 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4870 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4872 expand_task_call (new_bb
, entry_stmt
);
4873 if (gimple_in_ssa_p (cfun
))
4874 update_ssa (TODO_update_ssa_only_virtuals
);
4878 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4879 of the combined collapse > 1 loop constructs, generate code like:
4880 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4885 count3 = (adj + N32 - N31) / STEP3;
4886 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4891 count2 = (adj + N22 - N21) / STEP2;
4892 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4897 count1 = (adj + N12 - N11) / STEP1;
4898 count = count1 * count2 * count3;
4899 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4901 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4902 of the combined loop constructs, just initialize COUNTS array
4903 from the _looptemp_ clauses. */
4905 /* NOTE: It *could* be better to moosh all of the BBs together,
4906 creating one larger BB with all the computation and the unexpected
4907 jump at the end. I.e.
4909 bool zero3, zero2, zero1, zero;
4912 count3 = (N32 - N31) /[cl] STEP3;
4914 count2 = (N22 - N21) /[cl] STEP2;
4916 count1 = (N12 - N11) /[cl] STEP1;
4917 zero = zero3 || zero2 || zero1;
4918 count = count1 * count2 * count3;
4919 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4921 After all, we expect the zero=false, and thus we expect to have to
4922 evaluate all of the comparison expressions, so short-circuiting
4923 oughtn't be a win. Since the condition isn't protecting a
4924 denominator, we're not concerned about divide-by-zero, so we can
4925 fully evaluate count even if a numerator turned out to be wrong.
4927 It seems like putting this all together would create much better
4928 scheduling opportunities, and less pressure on the chip's branch
4932 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4933 basic_block
&entry_bb
, tree
*counts
,
4934 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4935 basic_block
&l2_dom_bb
)
4937 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
4942 /* Collapsed loops need work for expansion into SSA form. */
4943 gcc_assert (!gimple_in_ssa_p (cfun
));
4945 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
4946 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
4948 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4949 isn't supposed to be handled, as the inner loop doesn't
4951 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
4952 OMP_CLAUSE__LOOPTEMP_
);
4953 gcc_assert (innerc
);
4954 for (i
= 0; i
< fd
->collapse
; i
++)
4956 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
4957 OMP_CLAUSE__LOOPTEMP_
);
4958 gcc_assert (innerc
);
4960 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
4962 counts
[0] = NULL_TREE
;
4967 for (i
= 0; i
< fd
->collapse
; i
++)
4969 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
4971 if (SSA_VAR_P (fd
->loop
.n2
)
4972 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
4973 fold_convert (itype
, fd
->loops
[i
].n1
),
4974 fold_convert (itype
, fd
->loops
[i
].n2
)))
4975 == NULL_TREE
|| !integer_onep (t
)))
4978 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
4979 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
4980 true, GSI_SAME_STMT
);
4981 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
4982 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
4983 true, GSI_SAME_STMT
);
4984 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
4985 NULL_TREE
, NULL_TREE
);
4986 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4987 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
4988 expand_omp_regimplify_p
, NULL
, NULL
)
4989 || walk_tree (gimple_cond_rhs_ptr (stmt
),
4990 expand_omp_regimplify_p
, NULL
, NULL
))
4992 *gsi
= gsi_for_stmt (stmt
);
4993 gimple_regimplify_operands (stmt
, gsi
);
4995 e
= split_block (entry_bb
, stmt
);
4996 if (zero_iter_bb
== NULL
)
4998 first_zero_iter
= i
;
4999 zero_iter_bb
= create_empty_bb (entry_bb
);
5001 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5002 *gsi
= gsi_after_labels (zero_iter_bb
);
5003 stmt
= gimple_build_assign (fd
->loop
.n2
,
5004 build_zero_cst (type
));
5005 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5006 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5009 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5010 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5011 e
->flags
= EDGE_TRUE_VALUE
;
5012 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5013 if (l2_dom_bb
== NULL
)
5014 l2_dom_bb
= entry_bb
;
5016 *gsi
= gsi_last_bb (entry_bb
);
5019 if (POINTER_TYPE_P (itype
))
5020 itype
= signed_type_for (itype
);
5021 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5023 t
= fold_build2 (PLUS_EXPR
, itype
,
5024 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5025 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5026 fold_convert (itype
, fd
->loops
[i
].n2
));
5027 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5028 fold_convert (itype
, fd
->loops
[i
].n1
));
5029 /* ?? We could probably use CEIL_DIV_EXPR instead of
5030 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5031 generate the same code in the end because generically we
5032 don't know that the values involved must be negative for
5034 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5035 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5036 fold_build1 (NEGATE_EXPR
, itype
, t
),
5037 fold_build1 (NEGATE_EXPR
, itype
,
5038 fold_convert (itype
,
5039 fd
->loops
[i
].step
)));
5041 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5042 fold_convert (itype
, fd
->loops
[i
].step
));
5043 t
= fold_convert (type
, t
);
5044 if (TREE_CODE (t
) == INTEGER_CST
)
5048 counts
[i
] = create_tmp_reg (type
, ".count");
5049 expand_omp_build_assign (gsi
, counts
[i
], t
);
5051 if (SSA_VAR_P (fd
->loop
.n2
))
5056 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5057 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5063 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5065 V3 = N31 + (T % count3) * STEP3;
5067 V2 = N21 + (T % count2) * STEP2;
5069 V1 = N11 + T * STEP1;
5070 if this loop doesn't have an inner loop construct combined with it.
5071 If it does have an inner loop construct combined with it and the
5072 iteration count isn't known constant, store values from counts array
5073 into its _looptemp_ temporaries instead. */
5076 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5077 tree
*counts
, gimple inner_stmt
, tree startvar
)
5080 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5082 /* If fd->loop.n2 is constant, then no propagation of the counts
5083 is needed, they are constant. */
5084 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5087 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5088 ? gimple_omp_parallel_clauses (inner_stmt
)
5089 : gimple_omp_for_clauses (inner_stmt
);
5090 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5091 isn't supposed to be handled, as the inner loop doesn't
5093 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5094 gcc_assert (innerc
);
5095 for (i
= 0; i
< fd
->collapse
; i
++)
5097 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5098 OMP_CLAUSE__LOOPTEMP_
);
5099 gcc_assert (innerc
);
5102 tree tem
= OMP_CLAUSE_DECL (innerc
);
5103 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5104 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5105 false, GSI_CONTINUE_LINKING
);
5106 gimple stmt
= gimple_build_assign (tem
, t
);
5107 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5113 tree type
= TREE_TYPE (fd
->loop
.v
);
5114 tree tem
= create_tmp_reg (type
, ".tem");
5115 gimple stmt
= gimple_build_assign (tem
, startvar
);
5116 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5118 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5120 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5122 if (POINTER_TYPE_P (vtype
))
5123 itype
= signed_type_for (vtype
);
5125 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5128 t
= fold_convert (itype
, t
);
5129 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5130 fold_convert (itype
, fd
->loops
[i
].step
));
5131 if (POINTER_TYPE_P (vtype
))
5132 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5134 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5135 t
= force_gimple_operand_gsi (gsi
, t
,
5136 DECL_P (fd
->loops
[i
].v
)
5137 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5139 GSI_CONTINUE_LINKING
);
5140 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5141 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5144 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5145 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5146 false, GSI_CONTINUE_LINKING
);
5147 stmt
= gimple_build_assign (tem
, t
);
5148 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5154 /* Helper function for expand_omp_for_*. Generate code like:
5157 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5161 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5168 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5169 basic_block body_bb
)
5171 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5173 gimple_stmt_iterator gsi
;
5179 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5181 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5183 bb
= create_empty_bb (last_bb
);
5185 add_bb_to_loop (bb
, last_bb
->loop_father
);
5186 gsi
= gsi_start_bb (bb
);
5188 if (i
< fd
->collapse
- 1)
5190 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5191 e
->probability
= REG_BR_PROB_BASE
/ 8;
5193 t
= fd
->loops
[i
+ 1].n1
;
5194 t
= force_gimple_operand_gsi (&gsi
, t
,
5195 DECL_P (fd
->loops
[i
+ 1].v
)
5196 && TREE_ADDRESSABLE (fd
->loops
[i
5199 GSI_CONTINUE_LINKING
);
5200 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5201 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5206 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5208 if (POINTER_TYPE_P (vtype
))
5209 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5211 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5212 t
= force_gimple_operand_gsi (&gsi
, t
,
5213 DECL_P (fd
->loops
[i
].v
)
5214 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5215 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5216 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5217 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5221 t
= fd
->loops
[i
].n2
;
5222 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5223 false, GSI_CONTINUE_LINKING
);
5224 tree v
= fd
->loops
[i
].v
;
5225 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5226 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5227 false, GSI_CONTINUE_LINKING
);
5228 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5229 stmt
= gimple_build_cond_empty (t
);
5230 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5231 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5232 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5235 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5243 /* A subroutine of expand_omp_for. Generate code for a parallel
5244 loop with any schedule. Given parameters:
5246 for (V = N1; V cond N2; V += STEP) BODY;
5248 where COND is "<" or ">", we generate pseudocode
5250 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5251 if (more) goto L0; else goto L3;
5258 if (V cond iend) goto L1; else goto L2;
5260 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5263 If this is a combined omp parallel loop, instead of the call to
5264 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5265 If this is gimple_omp_for_combined_p loop, then instead of assigning
5266 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5267 inner GIMPLE_OMP_FOR and V += STEP; and
5268 if (V cond iend) goto L1; else goto L2; are removed.
5270 For collapsed loops, given parameters:
5272 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5273 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5274 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5277 we generate pseudocode
5279 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5284 count3 = (adj + N32 - N31) / STEP3;
5285 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5290 count2 = (adj + N22 - N21) / STEP2;
5291 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5296 count1 = (adj + N12 - N11) / STEP1;
5297 count = count1 * count2 * count3;
5302 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5303 if (more) goto L0; else goto L3;
5307 V3 = N31 + (T % count3) * STEP3;
5309 V2 = N21 + (T % count2) * STEP2;
5311 V1 = N11 + T * STEP1;
5316 if (V < iend) goto L10; else goto L2;
5319 if (V3 cond3 N32) goto L1; else goto L11;
5323 if (V2 cond2 N22) goto L1; else goto L12;
5329 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5335 expand_omp_for_generic (struct omp_region
*region
,
5336 struct omp_for_data
*fd
,
5337 enum built_in_function start_fn
,
5338 enum built_in_function next_fn
,
5341 tree type
, istart0
, iend0
, iend
;
5342 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5343 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5344 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5345 gimple_stmt_iterator gsi
;
5347 bool in_combined_parallel
= is_combined_parallel (region
);
5348 bool broken_loop
= region
->cont
== NULL
;
5350 tree
*counts
= NULL
;
5353 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5354 gcc_assert (fd
->iter_type
== long_integer_type_node
5355 || !in_combined_parallel
);
5357 type
= TREE_TYPE (fd
->loop
.v
);
5358 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5359 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5360 TREE_ADDRESSABLE (istart0
) = 1;
5361 TREE_ADDRESSABLE (iend0
) = 1;
5363 /* See if we need to bias by LLONG_MIN. */
5364 if (fd
->iter_type
== long_long_unsigned_type_node
5365 && TREE_CODE (type
) == INTEGER_TYPE
5366 && !TYPE_UNSIGNED (type
))
5370 if (fd
->loop
.cond_code
== LT_EXPR
)
5373 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5377 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5380 if (TREE_CODE (n1
) != INTEGER_CST
5381 || TREE_CODE (n2
) != INTEGER_CST
5382 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5383 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5386 entry_bb
= region
->entry
;
5387 cont_bb
= region
->cont
;
5389 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5390 gcc_assert (broken_loop
5391 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5392 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5393 l1_bb
= single_succ (l0_bb
);
5396 l2_bb
= create_empty_bb (cont_bb
);
5397 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5398 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5402 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5403 exit_bb
= region
->exit
;
5405 gsi
= gsi_last_bb (entry_bb
);
5407 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5408 if (fd
->collapse
> 1)
5410 int first_zero_iter
= -1;
5411 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5413 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5414 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5415 zero_iter_bb
, first_zero_iter
,
5420 /* Some counts[i] vars might be uninitialized if
5421 some loop has zero iterations. But the body shouldn't
5422 be executed in that case, so just avoid uninit warnings. */
5423 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5424 if (SSA_VAR_P (counts
[i
]))
5425 TREE_NO_WARNING (counts
[i
]) = 1;
5427 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5429 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5430 gsi
= gsi_last_bb (entry_bb
);
5431 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5432 get_immediate_dominator (CDI_DOMINATORS
,
5436 if (in_combined_parallel
)
5438 /* In a combined parallel loop, emit a call to
5439 GOMP_loop_foo_next. */
5440 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5441 build_fold_addr_expr (istart0
),
5442 build_fold_addr_expr (iend0
));
5446 tree t0
, t1
, t2
, t3
, t4
;
5447 /* If this is not a combined parallel loop, emit a call to
5448 GOMP_loop_foo_start in ENTRY_BB. */
5449 t4
= build_fold_addr_expr (iend0
);
5450 t3
= build_fold_addr_expr (istart0
);
5451 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5454 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5456 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5457 OMP_CLAUSE__LOOPTEMP_
);
5458 gcc_assert (innerc
);
5459 t0
= OMP_CLAUSE_DECL (innerc
);
5460 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5461 OMP_CLAUSE__LOOPTEMP_
);
5462 gcc_assert (innerc
);
5463 t1
= OMP_CLAUSE_DECL (innerc
);
5465 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5466 && TYPE_PRECISION (TREE_TYPE (t0
))
5467 != TYPE_PRECISION (fd
->iter_type
))
5469 /* Avoid casting pointers to integer of a different size. */
5470 tree itype
= signed_type_for (type
);
5471 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5472 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5476 t1
= fold_convert (fd
->iter_type
, t1
);
5477 t0
= fold_convert (fd
->iter_type
, t0
);
5481 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5482 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5484 if (fd
->iter_type
== long_integer_type_node
)
5488 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5489 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5490 6, t0
, t1
, t2
, t
, t3
, t4
);
5493 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5494 5, t0
, t1
, t2
, t3
, t4
);
5502 /* The GOMP_loop_ull_*start functions have additional boolean
5503 argument, true for < loops and false for > loops.
5504 In Fortran, the C bool type can be different from
5505 boolean_type_node. */
5506 bfn_decl
= builtin_decl_explicit (start_fn
);
5507 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5508 t5
= build_int_cst (c_bool_type
,
5509 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5512 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5513 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5514 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5517 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5518 6, t5
, t0
, t1
, t2
, t3
, t4
);
5521 if (TREE_TYPE (t
) != boolean_type_node
)
5522 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5523 t
, build_int_cst (TREE_TYPE (t
), 0));
5524 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5525 true, GSI_SAME_STMT
);
5526 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5528 /* Remove the GIMPLE_OMP_FOR statement. */
5529 gsi_remove (&gsi
, true);
5531 /* Iteration setup for sequential loop goes in L0_BB. */
5532 tree startvar
= fd
->loop
.v
;
5533 tree endvar
= NULL_TREE
;
5535 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5537 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5538 && gimple_omp_for_kind (inner_stmt
)
5539 == GF_OMP_FOR_KIND_SIMD
);
5540 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5541 OMP_CLAUSE__LOOPTEMP_
);
5542 gcc_assert (innerc
);
5543 startvar
= OMP_CLAUSE_DECL (innerc
);
5544 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5545 OMP_CLAUSE__LOOPTEMP_
);
5546 gcc_assert (innerc
);
5547 endvar
= OMP_CLAUSE_DECL (innerc
);
5550 gsi
= gsi_start_bb (l0_bb
);
5553 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5554 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5555 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5556 t
= fold_convert (TREE_TYPE (startvar
), t
);
5557 t
= force_gimple_operand_gsi (&gsi
, t
,
5559 && TREE_ADDRESSABLE (startvar
),
5560 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5561 stmt
= gimple_build_assign (startvar
, t
);
5562 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5566 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5567 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5568 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5569 t
= fold_convert (TREE_TYPE (startvar
), t
);
5570 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5571 false, GSI_CONTINUE_LINKING
);
5574 stmt
= gimple_build_assign (endvar
, iend
);
5575 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5577 if (fd
->collapse
> 1)
5578 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5582 /* Code to control the increment and predicate for the sequential
5583 loop goes in the CONT_BB. */
5584 gsi
= gsi_last_bb (cont_bb
);
5585 stmt
= gsi_stmt (gsi
);
5586 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5587 vmain
= gimple_omp_continue_control_use (stmt
);
5588 vback
= gimple_omp_continue_control_def (stmt
);
5590 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5592 if (POINTER_TYPE_P (type
))
5593 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5595 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5596 t
= force_gimple_operand_gsi (&gsi
, t
,
5598 && TREE_ADDRESSABLE (vback
),
5599 NULL_TREE
, true, GSI_SAME_STMT
);
5600 stmt
= gimple_build_assign (vback
, t
);
5601 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5603 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5604 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5606 stmt
= gimple_build_cond_empty (t
);
5607 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5610 /* Remove GIMPLE_OMP_CONTINUE. */
5611 gsi_remove (&gsi
, true);
5613 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5614 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5616 /* Emit code to get the next parallel iteration in L2_BB. */
5617 gsi
= gsi_start_bb (l2_bb
);
5619 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5620 build_fold_addr_expr (istart0
),
5621 build_fold_addr_expr (iend0
));
5622 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5623 false, GSI_CONTINUE_LINKING
);
5624 if (TREE_TYPE (t
) != boolean_type_node
)
5625 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5626 t
, build_int_cst (TREE_TYPE (t
), 0));
5627 stmt
= gimple_build_cond_empty (t
);
5628 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5631 /* Add the loop cleanup function. */
5632 gsi
= gsi_last_bb (exit_bb
);
5633 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5634 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5635 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5636 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5638 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5639 stmt
= gimple_build_call (t
, 0);
5640 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5641 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5642 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5643 gsi_remove (&gsi
, true);
5645 /* Connect the new blocks. */
5646 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5647 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5653 e
= find_edge (cont_bb
, l3_bb
);
5654 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5656 phis
= phi_nodes (l3_bb
);
5657 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5659 gimple phi
= gsi_stmt (gsi
);
5660 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5661 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5665 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5667 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5668 e
= find_edge (cont_bb
, l1_bb
);
5669 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5674 else if (fd
->collapse
> 1)
5677 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5680 e
->flags
= EDGE_TRUE_VALUE
;
5683 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5684 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5688 e
= find_edge (cont_bb
, l2_bb
);
5689 e
->flags
= EDGE_FALLTHRU
;
5691 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5693 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5694 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5695 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5696 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5697 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5698 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5699 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5700 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5702 struct loop
*outer_loop
= alloc_loop ();
5703 outer_loop
->header
= l0_bb
;
5704 outer_loop
->latch
= l2_bb
;
5705 add_loop (outer_loop
, l0_bb
->loop_father
);
5707 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5709 struct loop
*loop
= alloc_loop ();
5710 loop
->header
= l1_bb
;
5711 /* The loop may have multiple latches. */
5712 add_loop (loop
, outer_loop
);
5718 /* A subroutine of expand_omp_for. Generate code for a parallel
5719 loop with static schedule and no specified chunk size. Given
5722 for (V = N1; V cond N2; V += STEP) BODY;
5724 where COND is "<" or ">", we generate pseudocode
5726 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5731 if ((__typeof (V)) -1 > 0 && cond is >)
5732 n = -(adj + N2 - N1) / -STEP;
5734 n = (adj + N2 - N1) / STEP;
5737 if (threadid < tt) goto L3; else goto L4;
5742 s0 = q * threadid + tt;
5745 if (s0 >= e0) goto L2; else goto L0;
5751 if (V cond e) goto L1;
5756 expand_omp_for_static_nochunk (struct omp_region
*region
,
5757 struct omp_for_data
*fd
,
5760 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5761 tree type
, itype
, vmain
, vback
;
5762 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5763 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5765 gimple_stmt_iterator gsi
;
5768 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5769 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5770 bool broken_loop
= region
->cont
== NULL
;
5771 tree
*counts
= NULL
;
5774 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5775 if (POINTER_TYPE_P (type
))
5776 itype
= signed_type_for (type
);
5778 entry_bb
= region
->entry
;
5779 cont_bb
= region
->cont
;
5780 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5781 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5782 gcc_assert (broken_loop
5783 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5784 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5785 body_bb
= single_succ (seq_start_bb
);
5788 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5789 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5791 exit_bb
= region
->exit
;
5793 /* Iteration space partitioning goes in ENTRY_BB. */
5794 gsi
= gsi_last_bb (entry_bb
);
5795 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5797 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5799 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5800 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5803 if (fd
->collapse
> 1)
5805 int first_zero_iter
= -1;
5806 basic_block l2_dom_bb
= NULL
;
5808 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5809 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5810 fin_bb
, first_zero_iter
,
5814 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5815 t
= integer_one_node
;
5817 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5818 fold_convert (type
, fd
->loop
.n1
),
5819 fold_convert (type
, fd
->loop
.n2
));
5820 if (fd
->collapse
== 1
5821 && TYPE_UNSIGNED (type
)
5822 && (t
== NULL_TREE
|| !integer_onep (t
)))
5824 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5825 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5826 true, GSI_SAME_STMT
);
5827 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5828 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5829 true, GSI_SAME_STMT
);
5830 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5831 NULL_TREE
, NULL_TREE
);
5832 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5833 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5834 expand_omp_regimplify_p
, NULL
, NULL
)
5835 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5836 expand_omp_regimplify_p
, NULL
, NULL
))
5838 gsi
= gsi_for_stmt (stmt
);
5839 gimple_regimplify_operands (stmt
, &gsi
);
5841 ep
= split_block (entry_bb
, stmt
);
5842 ep
->flags
= EDGE_TRUE_VALUE
;
5843 entry_bb
= ep
->dest
;
5844 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5845 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5846 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5847 if (gimple_in_ssa_p (cfun
))
5849 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5850 for (gsi
= gsi_start_phis (fin_bb
);
5851 !gsi_end_p (gsi
); gsi_next (&gsi
))
5853 gimple phi
= gsi_stmt (gsi
);
5854 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5855 ep
, UNKNOWN_LOCATION
);
5858 gsi
= gsi_last_bb (entry_bb
);
5861 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5862 t
= fold_convert (itype
, t
);
5863 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5864 true, GSI_SAME_STMT
);
5866 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5867 t
= fold_convert (itype
, t
);
5868 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5869 true, GSI_SAME_STMT
);
5873 step
= fd
->loop
.step
;
5874 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5876 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5877 OMP_CLAUSE__LOOPTEMP_
);
5878 gcc_assert (innerc
);
5879 n1
= OMP_CLAUSE_DECL (innerc
);
5880 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5881 OMP_CLAUSE__LOOPTEMP_
);
5882 gcc_assert (innerc
);
5883 n2
= OMP_CLAUSE_DECL (innerc
);
5885 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5886 true, NULL_TREE
, true, GSI_SAME_STMT
);
5887 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5888 true, NULL_TREE
, true, GSI_SAME_STMT
);
5889 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5890 true, NULL_TREE
, true, GSI_SAME_STMT
);
5892 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5893 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5894 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5895 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5896 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5897 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5898 fold_build1 (NEGATE_EXPR
, itype
, t
),
5899 fold_build1 (NEGATE_EXPR
, itype
, step
));
5901 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5902 t
= fold_convert (itype
, t
);
5903 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5905 q
= create_tmp_reg (itype
, "q");
5906 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5907 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5908 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5910 tt
= create_tmp_reg (itype
, "tt");
5911 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5912 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5913 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5915 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5916 stmt
= gimple_build_cond_empty (t
);
5917 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5919 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5920 gsi
= gsi_last_bb (second_bb
);
5921 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5923 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5925 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5926 build_int_cst (itype
, 1));
5927 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5929 third_bb
= split_block (second_bb
, stmt
)->dest
;
5930 gsi
= gsi_last_bb (third_bb
);
5931 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5933 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
5934 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
5935 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5937 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
5938 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5940 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
5941 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5943 /* Remove the GIMPLE_OMP_FOR statement. */
5944 gsi_remove (&gsi
, true);
5946 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5947 gsi
= gsi_start_bb (seq_start_bb
);
5949 tree startvar
= fd
->loop
.v
;
5950 tree endvar
= NULL_TREE
;
5952 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5954 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5955 ? gimple_omp_parallel_clauses (inner_stmt
)
5956 : gimple_omp_for_clauses (inner_stmt
);
5957 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5958 gcc_assert (innerc
);
5959 startvar
= OMP_CLAUSE_DECL (innerc
);
5960 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5961 OMP_CLAUSE__LOOPTEMP_
);
5962 gcc_assert (innerc
);
5963 endvar
= OMP_CLAUSE_DECL (innerc
);
5965 t
= fold_convert (itype
, s0
);
5966 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5967 if (POINTER_TYPE_P (type
))
5968 t
= fold_build_pointer_plus (n1
, t
);
5970 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5971 t
= fold_convert (TREE_TYPE (startvar
), t
);
5972 t
= force_gimple_operand_gsi (&gsi
, t
,
5974 && TREE_ADDRESSABLE (startvar
),
5975 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5976 stmt
= gimple_build_assign (startvar
, t
);
5977 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5979 t
= fold_convert (itype
, e0
);
5980 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5981 if (POINTER_TYPE_P (type
))
5982 t
= fold_build_pointer_plus (n1
, t
);
5984 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5985 t
= fold_convert (TREE_TYPE (startvar
), t
);
5986 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5987 false, GSI_CONTINUE_LINKING
);
5990 stmt
= gimple_build_assign (endvar
, e
);
5991 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5993 if (fd
->collapse
> 1)
5994 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5998 /* The code controlling the sequential loop replaces the
5999 GIMPLE_OMP_CONTINUE. */
6000 gsi
= gsi_last_bb (cont_bb
);
6001 stmt
= gsi_stmt (gsi
);
6002 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6003 vmain
= gimple_omp_continue_control_use (stmt
);
6004 vback
= gimple_omp_continue_control_def (stmt
);
6006 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6008 if (POINTER_TYPE_P (type
))
6009 t
= fold_build_pointer_plus (vmain
, step
);
6011 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6012 t
= force_gimple_operand_gsi (&gsi
, t
,
6014 && TREE_ADDRESSABLE (vback
),
6015 NULL_TREE
, true, GSI_SAME_STMT
);
6016 stmt
= gimple_build_assign (vback
, t
);
6017 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6019 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6020 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6022 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6025 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6026 gsi_remove (&gsi
, true);
6028 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6029 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6032 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6033 gsi
= gsi_last_bb (exit_bb
);
6034 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6036 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6037 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6039 gsi_remove (&gsi
, true);
6041 /* Connect all the blocks. */
6042 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6043 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6044 ep
= find_edge (entry_bb
, second_bb
);
6045 ep
->flags
= EDGE_TRUE_VALUE
;
6046 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6047 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6048 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6052 ep
= find_edge (cont_bb
, body_bb
);
6053 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6058 else if (fd
->collapse
> 1)
6061 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6064 ep
->flags
= EDGE_TRUE_VALUE
;
6065 find_edge (cont_bb
, fin_bb
)->flags
6066 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6069 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6070 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6071 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6073 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6074 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6075 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6076 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6078 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6080 struct loop
*loop
= alloc_loop ();
6081 loop
->header
= body_bb
;
6082 if (collapse_bb
== NULL
)
6083 loop
->latch
= cont_bb
;
6084 add_loop (loop
, body_bb
->loop_father
);
6089 /* A subroutine of expand_omp_for. Generate code for a parallel
6090 loop with static schedule and a specified chunk size. Given
6093 for (V = N1; V cond N2; V += STEP) BODY;
6095 where COND is "<" or ">", we generate pseudocode
6097 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6102 if ((__typeof (V)) -1 > 0 && cond is >)
6103 n = -(adj + N2 - N1) / -STEP;
6105 n = (adj + N2 - N1) / STEP;
6107 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6108 here so that V is defined
6109 if the loop is not entered
6111 s0 = (trip * nthreads + threadid) * CHUNK;
6112 e0 = min(s0 + CHUNK, n);
6113 if (s0 < n) goto L1; else goto L4;
6120 if (V cond e) goto L2; else goto L3;
6128 expand_omp_for_static_chunk (struct omp_region
*region
,
6129 struct omp_for_data
*fd
, gimple inner_stmt
)
6131 tree n
, s0
, e0
, e
, t
;
6132 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6133 tree type
, itype
, v_main
, v_back
, v_extra
;
6134 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6135 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6136 gimple_stmt_iterator si
;
6139 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6140 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6141 bool broken_loop
= region
->cont
== NULL
;
6142 tree
*counts
= NULL
;
6145 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6146 if (POINTER_TYPE_P (type
))
6147 itype
= signed_type_for (type
);
6149 entry_bb
= region
->entry
;
6150 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6152 iter_part_bb
= se
->dest
;
6153 cont_bb
= region
->cont
;
6154 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6155 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6156 gcc_assert (broken_loop
6157 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6158 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6159 body_bb
= single_succ (seq_start_bb
);
6162 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6163 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6164 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6166 exit_bb
= region
->exit
;
6168 /* Trip and adjustment setup goes in ENTRY_BB. */
6169 si
= gsi_last_bb (entry_bb
);
6170 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6172 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6174 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6175 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6178 if (fd
->collapse
> 1)
6180 int first_zero_iter
= -1;
6181 basic_block l2_dom_bb
= NULL
;
6183 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6184 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6185 fin_bb
, first_zero_iter
,
6189 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6190 t
= integer_one_node
;
6192 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6193 fold_convert (type
, fd
->loop
.n1
),
6194 fold_convert (type
, fd
->loop
.n2
));
6195 if (fd
->collapse
== 1
6196 && TYPE_UNSIGNED (type
)
6197 && (t
== NULL_TREE
|| !integer_onep (t
)))
6199 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6200 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6201 true, GSI_SAME_STMT
);
6202 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6203 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6204 true, GSI_SAME_STMT
);
6205 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6206 NULL_TREE
, NULL_TREE
);
6207 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6208 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6209 expand_omp_regimplify_p
, NULL
, NULL
)
6210 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6211 expand_omp_regimplify_p
, NULL
, NULL
))
6213 si
= gsi_for_stmt (stmt
);
6214 gimple_regimplify_operands (stmt
, &si
);
6216 se
= split_block (entry_bb
, stmt
);
6217 se
->flags
= EDGE_TRUE_VALUE
;
6218 entry_bb
= se
->dest
;
6219 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6220 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6221 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6222 if (gimple_in_ssa_p (cfun
))
6224 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6225 for (si
= gsi_start_phis (fin_bb
);
6226 !gsi_end_p (si
); gsi_next (&si
))
6228 gimple phi
= gsi_stmt (si
);
6229 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6230 se
, UNKNOWN_LOCATION
);
6233 si
= gsi_last_bb (entry_bb
);
6236 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6237 t
= fold_convert (itype
, t
);
6238 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6239 true, GSI_SAME_STMT
);
6241 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6242 t
= fold_convert (itype
, t
);
6243 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6244 true, GSI_SAME_STMT
);
6248 step
= fd
->loop
.step
;
6249 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6251 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6252 OMP_CLAUSE__LOOPTEMP_
);
6253 gcc_assert (innerc
);
6254 n1
= OMP_CLAUSE_DECL (innerc
);
6255 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6256 OMP_CLAUSE__LOOPTEMP_
);
6257 gcc_assert (innerc
);
6258 n2
= OMP_CLAUSE_DECL (innerc
);
6260 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6261 true, NULL_TREE
, true, GSI_SAME_STMT
);
6262 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6263 true, NULL_TREE
, true, GSI_SAME_STMT
);
6264 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6265 true, NULL_TREE
, true, GSI_SAME_STMT
);
6267 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6268 true, NULL_TREE
, true, GSI_SAME_STMT
);
6270 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6271 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6272 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6273 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6274 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6275 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6276 fold_build1 (NEGATE_EXPR
, itype
, t
),
6277 fold_build1 (NEGATE_EXPR
, itype
, step
));
6279 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6280 t
= fold_convert (itype
, t
);
6281 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6282 true, GSI_SAME_STMT
);
6284 trip_var
= create_tmp_reg (itype
, ".trip");
6285 if (gimple_in_ssa_p (cfun
))
6287 trip_init
= make_ssa_name (trip_var
, NULL
);
6288 trip_main
= make_ssa_name (trip_var
, NULL
);
6289 trip_back
= make_ssa_name (trip_var
, NULL
);
6293 trip_init
= trip_var
;
6294 trip_main
= trip_var
;
6295 trip_back
= trip_var
;
6298 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6299 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6301 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6302 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6303 if (POINTER_TYPE_P (type
))
6304 t
= fold_build_pointer_plus (n1
, t
);
6306 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6307 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6308 true, GSI_SAME_STMT
);
6310 /* Remove the GIMPLE_OMP_FOR. */
6311 gsi_remove (&si
, true);
6313 /* Iteration space partitioning goes in ITER_PART_BB. */
6314 si
= gsi_last_bb (iter_part_bb
);
6316 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6317 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6318 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6319 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6320 false, GSI_CONTINUE_LINKING
);
6322 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6323 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6324 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6325 false, GSI_CONTINUE_LINKING
);
6327 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6328 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6330 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6331 si
= gsi_start_bb (seq_start_bb
);
6333 tree startvar
= fd
->loop
.v
;
6334 tree endvar
= NULL_TREE
;
6336 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6338 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6339 ? gimple_omp_parallel_clauses (inner_stmt
)
6340 : gimple_omp_for_clauses (inner_stmt
);
6341 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6342 gcc_assert (innerc
);
6343 startvar
= OMP_CLAUSE_DECL (innerc
);
6344 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6345 OMP_CLAUSE__LOOPTEMP_
);
6346 gcc_assert (innerc
);
6347 endvar
= OMP_CLAUSE_DECL (innerc
);
6350 t
= fold_convert (itype
, s0
);
6351 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6352 if (POINTER_TYPE_P (type
))
6353 t
= fold_build_pointer_plus (n1
, t
);
6355 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6356 t
= fold_convert (TREE_TYPE (startvar
), t
);
6357 t
= force_gimple_operand_gsi (&si
, t
,
6359 && TREE_ADDRESSABLE (startvar
),
6360 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6361 stmt
= gimple_build_assign (startvar
, t
);
6362 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6364 t
= fold_convert (itype
, e0
);
6365 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6366 if (POINTER_TYPE_P (type
))
6367 t
= fold_build_pointer_plus (n1
, t
);
6369 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6370 t
= fold_convert (TREE_TYPE (startvar
), t
);
6371 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6372 false, GSI_CONTINUE_LINKING
);
6375 stmt
= gimple_build_assign (endvar
, e
);
6376 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6378 if (fd
->collapse
> 1)
6379 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6383 /* The code controlling the sequential loop goes in CONT_BB,
6384 replacing the GIMPLE_OMP_CONTINUE. */
6385 si
= gsi_last_bb (cont_bb
);
6386 stmt
= gsi_stmt (si
);
6387 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6388 v_main
= gimple_omp_continue_control_use (stmt
);
6389 v_back
= gimple_omp_continue_control_def (stmt
);
6391 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6393 if (POINTER_TYPE_P (type
))
6394 t
= fold_build_pointer_plus (v_main
, step
);
6396 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6397 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6398 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6399 true, GSI_SAME_STMT
);
6400 stmt
= gimple_build_assign (v_back
, t
);
6401 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6403 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6404 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6406 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6409 /* Remove GIMPLE_OMP_CONTINUE. */
6410 gsi_remove (&si
, true);
6412 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6413 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6415 /* Trip update code goes into TRIP_UPDATE_BB. */
6416 si
= gsi_start_bb (trip_update_bb
);
6418 t
= build_int_cst (itype
, 1);
6419 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6420 stmt
= gimple_build_assign (trip_back
, t
);
6421 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6424 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6425 si
= gsi_last_bb (exit_bb
);
6426 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6428 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6429 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6431 gsi_remove (&si
, true);
6433 /* Connect the new blocks. */
6434 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6435 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6439 se
= find_edge (cont_bb
, body_bb
);
6440 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6445 else if (fd
->collapse
> 1)
6448 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6451 se
->flags
= EDGE_TRUE_VALUE
;
6452 find_edge (cont_bb
, trip_update_bb
)->flags
6453 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6455 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6458 if (gimple_in_ssa_p (cfun
))
6460 gimple_stmt_iterator psi
;
6463 edge_var_map_vector
*head
;
6467 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6469 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6470 remove arguments of the phi nodes in fin_bb. We need to create
6471 appropriate phi nodes in iter_part_bb instead. */
6472 se
= single_pred_edge (fin_bb
);
6473 re
= single_succ_edge (trip_update_bb
);
6474 head
= redirect_edge_var_map_vector (re
);
6475 ene
= single_succ_edge (entry_bb
);
6477 psi
= gsi_start_phis (fin_bb
);
6478 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6479 gsi_next (&psi
), ++i
)
6482 source_location locus
;
6484 phi
= gsi_stmt (psi
);
6485 t
= gimple_phi_result (phi
);
6486 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6487 nphi
= create_phi_node (t
, iter_part_bb
);
6489 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6490 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6492 /* A special case -- fd->loop.v is not yet computed in
6493 iter_part_bb, we need to use v_extra instead. */
6494 if (t
== fd
->loop
.v
)
6496 add_phi_arg (nphi
, t
, ene
, locus
);
6497 locus
= redirect_edge_var_map_location (vm
);
6498 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6500 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6501 redirect_edge_var_map_clear (re
);
6504 psi
= gsi_start_phis (fin_bb
);
6505 if (gsi_end_p (psi
))
6507 remove_phi_node (&psi
, false);
6510 /* Make phi node for trip. */
6511 phi
= create_phi_node (trip_main
, iter_part_bb
);
6512 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6514 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6519 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6520 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6521 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6522 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6523 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6524 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6525 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6526 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6527 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6531 struct loop
*trip_loop
= alloc_loop ();
6532 trip_loop
->header
= iter_part_bb
;
6533 trip_loop
->latch
= trip_update_bb
;
6534 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6536 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6538 struct loop
*loop
= alloc_loop ();
6539 loop
->header
= body_bb
;
6540 if (collapse_bb
== NULL
)
6541 loop
->latch
= cont_bb
;
6542 add_loop (loop
, trip_loop
);
6548 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6549 loop. Given parameters:
6551 for (V = N1; V cond N2; V += STEP) BODY;
6553 where COND is "<" or ">", we generate pseudocode
6561 if (V cond N2) goto L0; else goto L2;
6564 For collapsed loops, given parameters:
6566 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6567 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6568 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6571 we generate pseudocode
6577 count3 = (adj + N32 - N31) / STEP3;
6582 count2 = (adj + N22 - N21) / STEP2;
6587 count1 = (adj + N12 - N11) / STEP1;
6588 count = count1 * count2 * count3;
6598 V2 += (V3 cond3 N32) ? 0 : STEP2;
6599 V3 = (V3 cond3 N32) ? V3 : N31;
6600 V1 += (V2 cond2 N22) ? 0 : STEP1;
6601 V2 = (V2 cond2 N22) ? V2 : N21;
6603 if (V < count) goto L0; else goto L2;
6609 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6612 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6613 gimple_stmt_iterator gsi
;
6615 bool broken_loop
= region
->cont
== NULL
;
6617 tree
*counts
= NULL
;
6619 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6620 OMP_CLAUSE_SAFELEN
);
6621 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6622 OMP_CLAUSE__SIMDUID_
);
6625 type
= TREE_TYPE (fd
->loop
.v
);
6626 entry_bb
= region
->entry
;
6627 cont_bb
= region
->cont
;
6628 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6629 gcc_assert (broken_loop
6630 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6631 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6634 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6635 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6636 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6637 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6641 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6642 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6643 l2_bb
= single_succ (l1_bb
);
6645 exit_bb
= region
->exit
;
6648 gsi
= gsi_last_bb (entry_bb
);
6650 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6651 /* Not needed in SSA form right now. */
6652 gcc_assert (!gimple_in_ssa_p (cfun
));
6653 if (fd
->collapse
> 1)
6655 int first_zero_iter
= -1;
6656 basic_block zero_iter_bb
= l2_bb
;
6658 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6659 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6660 zero_iter_bb
, first_zero_iter
,
6663 if (l2_dom_bb
== NULL
)
6668 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6670 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6671 OMP_CLAUSE__LOOPTEMP_
);
6672 gcc_assert (innerc
);
6673 n1
= OMP_CLAUSE_DECL (innerc
);
6674 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6675 OMP_CLAUSE__LOOPTEMP_
);
6676 gcc_assert (innerc
);
6677 n2
= OMP_CLAUSE_DECL (innerc
);
6678 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6679 fold_convert (type
, n1
));
6680 if (fd
->collapse
> 1)
6683 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6689 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6690 fold_convert (type
, fd
->loop
.n1
));
6691 if (fd
->collapse
> 1)
6692 for (i
= 0; i
< fd
->collapse
; i
++)
6694 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6695 if (POINTER_TYPE_P (itype
))
6696 itype
= signed_type_for (itype
);
6697 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6698 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6702 /* Remove the GIMPLE_OMP_FOR statement. */
6703 gsi_remove (&gsi
, true);
6707 /* Code to control the increment goes in the CONT_BB. */
6708 gsi
= gsi_last_bb (cont_bb
);
6709 stmt
= gsi_stmt (gsi
);
6710 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6712 if (POINTER_TYPE_P (type
))
6713 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6715 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6716 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6718 if (fd
->collapse
> 1)
6720 i
= fd
->collapse
- 1;
6721 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6723 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6724 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6728 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6730 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6733 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6735 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6737 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6738 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6739 if (POINTER_TYPE_P (itype2
))
6740 itype2
= signed_type_for (itype2
);
6741 t
= build3 (COND_EXPR
, itype2
,
6742 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6744 fold_convert (itype
, fd
->loops
[i
].n2
)),
6745 build_int_cst (itype2
, 0),
6746 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6747 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6748 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6750 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6751 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6753 t
= build3 (COND_EXPR
, itype
,
6754 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6756 fold_convert (itype
, fd
->loops
[i
].n2
)),
6758 fold_convert (itype
, fd
->loops
[i
].n1
));
6759 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6763 /* Remove GIMPLE_OMP_CONTINUE. */
6764 gsi_remove (&gsi
, true);
6767 /* Emit the condition in L1_BB. */
6768 gsi
= gsi_start_bb (l1_bb
);
6770 t
= fold_convert (type
, n2
);
6771 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6772 false, GSI_CONTINUE_LINKING
);
6773 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6774 stmt
= gimple_build_cond_empty (t
);
6775 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6776 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6778 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6781 gsi
= gsi_for_stmt (stmt
);
6782 gimple_regimplify_operands (stmt
, &gsi
);
6785 /* Remove GIMPLE_OMP_RETURN. */
6786 gsi
= gsi_last_bb (exit_bb
);
6787 gsi_remove (&gsi
, true);
6789 /* Connect the new blocks. */
6790 remove_edge (FALLTHRU_EDGE (entry_bb
));
6794 remove_edge (BRANCH_EDGE (entry_bb
));
6795 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6797 e
= BRANCH_EDGE (l1_bb
);
6798 ne
= FALLTHRU_EDGE (l1_bb
);
6799 e
->flags
= EDGE_TRUE_VALUE
;
6803 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6805 ne
= single_succ_edge (l1_bb
);
6806 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6809 ne
->flags
= EDGE_FALSE_VALUE
;
6810 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6811 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6813 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6814 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6815 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6819 struct loop
*loop
= alloc_loop ();
6820 loop
->header
= l1_bb
;
6821 loop
->latch
= cont_bb
;
6822 add_loop (loop
, l1_bb
->loop_father
);
6823 if (safelen
== NULL_TREE
)
6824 loop
->safelen
= INT_MAX
;
6827 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6828 if (!tree_fits_uhwi_p (safelen
)
6829 || tree_to_uhwi (safelen
) > INT_MAX
)
6830 loop
->safelen
= INT_MAX
;
6832 loop
->safelen
= tree_to_uhwi (safelen
);
6833 if (loop
->safelen
== 1)
6838 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6839 cfun
->has_simduid_loops
= true;
6841 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6843 if ((flag_tree_loop_vectorize
6844 || (!global_options_set
.x_flag_tree_loop_vectorize
6845 && !global_options_set
.x_flag_tree_vectorize
))
6846 && loop
->safelen
> 1)
6848 loop
->force_vect
= true;
6849 cfun
->has_force_vect_loops
= true;
6855 /* Expand the OpenMP loop defined by REGION. */
6858 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6860 struct omp_for_data fd
;
6861 struct omp_for_data_loop
*loops
;
6864 = (struct omp_for_data_loop
*)
6865 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6866 * sizeof (struct omp_for_data_loop
));
6867 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6868 region
->sched_kind
= fd
.sched_kind
;
6870 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6871 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6872 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6875 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6876 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6877 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6880 /* If there isn't a continue then this is a degerate case where
6881 the introduction of abnormal edges during lowering will prevent
6882 original loops from being detected. Fix that up. */
6883 loops_state_set (LOOPS_NEED_FIXUP
);
6885 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_KIND_SIMD
)
6886 expand_omp_simd (region
, &fd
);
6887 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6888 && !fd
.have_ordered
)
6890 if (fd
.chunk_size
== NULL
)
6891 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6893 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6897 int fn_index
, start_ix
, next_ix
;
6899 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6900 == GF_OMP_FOR_KIND_FOR
);
6901 if (fd
.chunk_size
== NULL
6902 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6903 fd
.chunk_size
= integer_zero_node
;
6904 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6905 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6906 ? 3 : fd
.sched_kind
;
6907 fn_index
+= fd
.have_ordered
* 4;
6908 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6909 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6910 if (fd
.iter_type
== long_long_unsigned_type_node
)
6912 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6913 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6914 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6915 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6917 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6918 (enum built_in_function
) next_ix
, inner_stmt
);
6921 if (gimple_in_ssa_p (cfun
))
6922 update_ssa (TODO_update_ssa_only_virtuals
);
6926 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6928 v = GOMP_sections_start (n);
6945 v = GOMP_sections_next ();
6950 If this is a combined parallel sections, replace the call to
6951 GOMP_sections_start with call to GOMP_sections_next. */
6954 expand_omp_sections (struct omp_region
*region
)
6956 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
6958 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
6959 gimple_stmt_iterator si
, switch_si
;
6960 gimple sections_stmt
, stmt
, cont
;
6963 struct omp_region
*inner
;
6965 bool exit_reachable
= region
->cont
!= NULL
;
6967 gcc_assert (region
->exit
!= NULL
);
6968 entry_bb
= region
->entry
;
6969 l0_bb
= single_succ (entry_bb
);
6970 l1_bb
= region
->cont
;
6971 l2_bb
= region
->exit
;
6972 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
6973 l2
= gimple_block_label (l2_bb
);
6976 /* This can happen if there are reductions. */
6977 len
= EDGE_COUNT (l0_bb
->succs
);
6978 gcc_assert (len
> 0);
6979 e
= EDGE_SUCC (l0_bb
, len
- 1);
6980 si
= gsi_last_bb (e
->dest
);
6983 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6984 l2
= gimple_block_label (e
->dest
);
6986 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
6988 si
= gsi_last_bb (e
->dest
);
6990 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6992 l2
= gimple_block_label (e
->dest
);
6998 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7000 default_bb
= create_empty_bb (l0_bb
);
7002 /* We will build a switch() with enough cases for all the
7003 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7004 and a default case to abort if something goes wrong. */
7005 len
= EDGE_COUNT (l0_bb
->succs
);
7007 /* Use vec::quick_push on label_vec throughout, since we know the size
7009 auto_vec
<tree
> label_vec (len
);
7011 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7012 GIMPLE_OMP_SECTIONS statement. */
7013 si
= gsi_last_bb (entry_bb
);
7014 sections_stmt
= gsi_stmt (si
);
7015 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7016 vin
= gimple_omp_sections_control (sections_stmt
);
7017 if (!is_combined_parallel (region
))
7019 /* If we are not inside a combined parallel+sections region,
7020 call GOMP_sections_start. */
7021 t
= build_int_cst (unsigned_type_node
, len
- 1);
7022 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7023 stmt
= gimple_build_call (u
, 1, t
);
7027 /* Otherwise, call GOMP_sections_next. */
7028 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7029 stmt
= gimple_build_call (u
, 0);
7031 gimple_call_set_lhs (stmt
, vin
);
7032 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7033 gsi_remove (&si
, true);
7035 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7037 switch_si
= gsi_last_bb (l0_bb
);
7038 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7041 cont
= last_stmt (l1_bb
);
7042 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7043 vmain
= gimple_omp_continue_control_use (cont
);
7044 vnext
= gimple_omp_continue_control_def (cont
);
7052 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7053 label_vec
.quick_push (t
);
7056 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7057 for (inner
= region
->inner
, casei
= 1;
7059 inner
= inner
->next
, i
++, casei
++)
7061 basic_block s_entry_bb
, s_exit_bb
;
7063 /* Skip optional reduction region. */
7064 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7071 s_entry_bb
= inner
->entry
;
7072 s_exit_bb
= inner
->exit
;
7074 t
= gimple_block_label (s_entry_bb
);
7075 u
= build_int_cst (unsigned_type_node
, casei
);
7076 u
= build_case_label (u
, NULL
, t
);
7077 label_vec
.quick_push (u
);
7079 si
= gsi_last_bb (s_entry_bb
);
7080 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7081 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7082 gsi_remove (&si
, true);
7083 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7085 if (s_exit_bb
== NULL
)
7088 si
= gsi_last_bb (s_exit_bb
);
7089 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7090 gsi_remove (&si
, true);
7092 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7095 /* Error handling code goes in DEFAULT_BB. */
7096 t
= gimple_block_label (default_bb
);
7097 u
= build_case_label (NULL
, NULL
, t
);
7098 make_edge (l0_bb
, default_bb
, 0);
7100 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7102 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7103 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7104 gsi_remove (&switch_si
, true);
7106 si
= gsi_start_bb (default_bb
);
7107 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7108 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7114 /* Code to get the next section goes in L1_BB. */
7115 si
= gsi_last_bb (l1_bb
);
7116 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7118 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7119 stmt
= gimple_build_call (bfn_decl
, 0);
7120 gimple_call_set_lhs (stmt
, vnext
);
7121 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7122 gsi_remove (&si
, true);
7124 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7127 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7128 si
= gsi_last_bb (l2_bb
);
7129 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7130 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7131 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7132 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7134 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7135 stmt
= gimple_build_call (t
, 0);
7136 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7137 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7138 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7139 gsi_remove (&si
, true);
7141 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7145 /* Expand code for an OpenMP single directive. We've already expanded
7146 much of the code, here we simply place the GOMP_barrier call. */
7149 expand_omp_single (struct omp_region
*region
)
7151 basic_block entry_bb
, exit_bb
;
7152 gimple_stmt_iterator si
;
7154 entry_bb
= region
->entry
;
7155 exit_bb
= region
->exit
;
7157 si
= gsi_last_bb (entry_bb
);
7158 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7159 gsi_remove (&si
, true);
7160 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7162 si
= gsi_last_bb (exit_bb
);
7163 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7165 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7166 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7168 gsi_remove (&si
, true);
7169 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7173 /* Generic expansion for OpenMP synchronization directives: master,
7174 ordered and critical. All we need to do here is remove the entry
7175 and exit markers for REGION. */
7178 expand_omp_synch (struct omp_region
*region
)
7180 basic_block entry_bb
, exit_bb
;
7181 gimple_stmt_iterator si
;
7183 entry_bb
= region
->entry
;
7184 exit_bb
= region
->exit
;
7186 si
= gsi_last_bb (entry_bb
);
7187 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7188 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7189 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7190 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7191 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7192 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7193 gsi_remove (&si
, true);
7194 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7198 si
= gsi_last_bb (exit_bb
);
7199 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7200 gsi_remove (&si
, true);
7201 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7205 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7206 operation as a normal volatile load. */
7209 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7210 tree loaded_val
, int index
)
7212 enum built_in_function tmpbase
;
7213 gimple_stmt_iterator gsi
;
7214 basic_block store_bb
;
7217 tree decl
, call
, type
, itype
;
7219 gsi
= gsi_last_bb (load_bb
);
7220 stmt
= gsi_stmt (gsi
);
7221 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7222 loc
= gimple_location (stmt
);
7224 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7225 is smaller than word size, then expand_atomic_load assumes that the load
7226 is atomic. We could avoid the builtin entirely in this case. */
7228 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7229 decl
= builtin_decl_explicit (tmpbase
);
7230 if (decl
== NULL_TREE
)
7233 type
= TREE_TYPE (loaded_val
);
7234 itype
= TREE_TYPE (TREE_TYPE (decl
));
7236 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7237 build_int_cst (NULL
,
7238 gimple_omp_atomic_seq_cst_p (stmt
)
7240 : MEMMODEL_RELAXED
));
7241 if (!useless_type_conversion_p (type
, itype
))
7242 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7243 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7245 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7246 gsi_remove (&gsi
, true);
7248 store_bb
= single_succ (load_bb
);
7249 gsi
= gsi_last_bb (store_bb
);
7250 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7251 gsi_remove (&gsi
, true);
7253 if (gimple_in_ssa_p (cfun
))
7254 update_ssa (TODO_update_ssa_no_phi
);
7259 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7260 operation as a normal volatile store. */
7263 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7264 tree loaded_val
, tree stored_val
, int index
)
7266 enum built_in_function tmpbase
;
7267 gimple_stmt_iterator gsi
;
7268 basic_block store_bb
= single_succ (load_bb
);
7271 tree decl
, call
, type
, itype
;
7272 enum machine_mode imode
;
7275 gsi
= gsi_last_bb (load_bb
);
7276 stmt
= gsi_stmt (gsi
);
7277 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7279 /* If the load value is needed, then this isn't a store but an exchange. */
7280 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7282 gsi
= gsi_last_bb (store_bb
);
7283 stmt
= gsi_stmt (gsi
);
7284 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7285 loc
= gimple_location (stmt
);
7287 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7288 is smaller than word size, then expand_atomic_store assumes that the store
7289 is atomic. We could avoid the builtin entirely in this case. */
7291 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7292 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7293 decl
= builtin_decl_explicit (tmpbase
);
7294 if (decl
== NULL_TREE
)
7297 type
= TREE_TYPE (stored_val
);
7299 /* Dig out the type of the function's second argument. */
7300 itype
= TREE_TYPE (decl
);
7301 itype
= TYPE_ARG_TYPES (itype
);
7302 itype
= TREE_CHAIN (itype
);
7303 itype
= TREE_VALUE (itype
);
7304 imode
= TYPE_MODE (itype
);
7306 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7309 if (!useless_type_conversion_p (itype
, type
))
7310 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7311 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7312 build_int_cst (NULL
,
7313 gimple_omp_atomic_seq_cst_p (stmt
)
7315 : MEMMODEL_RELAXED
));
7318 if (!useless_type_conversion_p (type
, itype
))
7319 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7320 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7323 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7324 gsi_remove (&gsi
, true);
7326 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7327 gsi
= gsi_last_bb (load_bb
);
7328 gsi_remove (&gsi
, true);
7330 if (gimple_in_ssa_p (cfun
))
7331 update_ssa (TODO_update_ssa_no_phi
);
7336 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7337 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7338 size of the data type, and thus usable to find the index of the builtin
7339 decl. Returns false if the expression is not of the proper form. */
7342 expand_omp_atomic_fetch_op (basic_block load_bb
,
7343 tree addr
, tree loaded_val
,
7344 tree stored_val
, int index
)
7346 enum built_in_function oldbase
, newbase
, tmpbase
;
7347 tree decl
, itype
, call
;
7349 basic_block store_bb
= single_succ (load_bb
);
7350 gimple_stmt_iterator gsi
;
7353 enum tree_code code
;
7354 bool need_old
, need_new
;
7355 enum machine_mode imode
;
7358 /* We expect to find the following sequences:
7361 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7364 val = tmp OP something; (or: something OP tmp)
7365 GIMPLE_OMP_STORE (val)
7367 ???FIXME: Allow a more flexible sequence.
7368 Perhaps use data flow to pick the statements.
7372 gsi
= gsi_after_labels (store_bb
);
7373 stmt
= gsi_stmt (gsi
);
7374 loc
= gimple_location (stmt
);
7375 if (!is_gimple_assign (stmt
))
7378 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7380 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7381 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7382 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7383 gcc_checking_assert (!need_old
|| !need_new
);
7385 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7388 /* Check for one of the supported fetch-op operations. */
7389 code
= gimple_assign_rhs_code (stmt
);
7393 case POINTER_PLUS_EXPR
:
7394 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7395 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7398 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7399 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7402 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7403 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7406 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7407 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7410 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7411 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7417 /* Make sure the expression is of the proper form. */
7418 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7419 rhs
= gimple_assign_rhs2 (stmt
);
7420 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7421 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7422 rhs
= gimple_assign_rhs1 (stmt
);
7426 tmpbase
= ((enum built_in_function
)
7427 ((need_new
? newbase
: oldbase
) + index
+ 1));
7428 decl
= builtin_decl_explicit (tmpbase
);
7429 if (decl
== NULL_TREE
)
7431 itype
= TREE_TYPE (TREE_TYPE (decl
));
7432 imode
= TYPE_MODE (itype
);
7434 /* We could test all of the various optabs involved, but the fact of the
7435 matter is that (with the exception of i486 vs i586 and xadd) all targets
7436 that support any atomic operaton optab also implements compare-and-swap.
7437 Let optabs.c take care of expanding any compare-and-swap loop. */
7438 if (!can_compare_and_swap_p (imode
, true))
7441 gsi
= gsi_last_bb (load_bb
);
7442 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7444 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7445 It only requires that the operation happen atomically. Thus we can
7446 use the RELAXED memory model. */
7447 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7448 fold_convert_loc (loc
, itype
, rhs
),
7449 build_int_cst (NULL
,
7450 seq_cst
? MEMMODEL_SEQ_CST
7451 : MEMMODEL_RELAXED
));
7453 if (need_old
|| need_new
)
7455 lhs
= need_old
? loaded_val
: stored_val
;
7456 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7457 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7460 call
= fold_convert_loc (loc
, void_type_node
, call
);
7461 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7462 gsi_remove (&gsi
, true);
7464 gsi
= gsi_last_bb (store_bb
);
7465 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7466 gsi_remove (&gsi
, true);
7467 gsi
= gsi_last_bb (store_bb
);
7468 gsi_remove (&gsi
, true);
7470 if (gimple_in_ssa_p (cfun
))
7471 update_ssa (TODO_update_ssa_no_phi
);
7476 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7480 newval = rhs; // with oldval replacing *addr in rhs
7481 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7482 if (oldval != newval)
7485 INDEX is log2 of the size of the data type, and thus usable to find the
7486 index of the builtin decl. */
7489 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7490 tree addr
, tree loaded_val
, tree stored_val
,
7493 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7494 tree type
, itype
, cmpxchg
, iaddr
;
7495 gimple_stmt_iterator si
;
7496 basic_block loop_header
= single_succ (load_bb
);
7499 enum built_in_function fncode
;
7501 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7502 order to use the RELAXED memory model effectively. */
7503 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7505 cmpxchg
= builtin_decl_explicit (fncode
);
7506 if (cmpxchg
== NULL_TREE
)
7508 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7509 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7511 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7514 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7515 si
= gsi_last_bb (load_bb
);
7516 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7518 /* For floating-point values, we'll need to view-convert them to integers
7519 so that we can perform the atomic compare and swap. Simplify the
7520 following code by always setting up the "i"ntegral variables. */
7521 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7525 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7528 = force_gimple_operand_gsi (&si
,
7529 fold_convert (TREE_TYPE (iaddr
), addr
),
7530 false, NULL_TREE
, true, GSI_SAME_STMT
);
7531 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7532 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7533 loadedi
= create_tmp_var (itype
, NULL
);
7534 if (gimple_in_ssa_p (cfun
))
7535 loadedi
= make_ssa_name (loadedi
, NULL
);
7540 loadedi
= loaded_val
;
7544 = force_gimple_operand_gsi (&si
,
7545 build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)),
7547 build_int_cst (TREE_TYPE (iaddr
), 0)),
7548 true, NULL_TREE
, true, GSI_SAME_STMT
);
7550 /* Move the value to the LOADEDI temporary. */
7551 if (gimple_in_ssa_p (cfun
))
7553 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7554 phi
= create_phi_node (loadedi
, loop_header
);
7555 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7559 gsi_insert_before (&si
,
7560 gimple_build_assign (loadedi
, initial
),
7562 if (loadedi
!= loaded_val
)
7564 gimple_stmt_iterator gsi2
;
7567 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7568 gsi2
= gsi_start_bb (loop_header
);
7569 if (gimple_in_ssa_p (cfun
))
7572 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7573 true, GSI_SAME_STMT
);
7574 stmt
= gimple_build_assign (loaded_val
, x
);
7575 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7579 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7580 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7581 true, GSI_SAME_STMT
);
7584 gsi_remove (&si
, true);
7586 si
= gsi_last_bb (store_bb
);
7587 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7590 storedi
= stored_val
;
7593 force_gimple_operand_gsi (&si
,
7594 build1 (VIEW_CONVERT_EXPR
, itype
,
7595 stored_val
), true, NULL_TREE
, true,
7598 /* Build the compare&swap statement. */
7599 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7600 new_storedi
= force_gimple_operand_gsi (&si
,
7601 fold_convert (TREE_TYPE (loadedi
),
7604 true, GSI_SAME_STMT
);
7606 if (gimple_in_ssa_p (cfun
))
7610 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7611 stmt
= gimple_build_assign (old_vali
, loadedi
);
7612 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7614 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7615 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7618 /* Note that we always perform the comparison as an integer, even for
7619 floating point. This allows the atomic operation to properly
7620 succeed even with NaNs and -0.0. */
7621 stmt
= gimple_build_cond_empty
7622 (build2 (NE_EXPR
, boolean_type_node
,
7623 new_storedi
, old_vali
));
7624 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7627 e
= single_succ_edge (store_bb
);
7628 e
->flags
&= ~EDGE_FALLTHRU
;
7629 e
->flags
|= EDGE_FALSE_VALUE
;
7631 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7633 /* Copy the new value to loadedi (we already did that before the condition
7634 if we are not in SSA). */
7635 if (gimple_in_ssa_p (cfun
))
7637 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7638 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7641 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7642 gsi_remove (&si
, true);
7644 struct loop
*loop
= alloc_loop ();
7645 loop
->header
= loop_header
;
7646 loop
->latch
= store_bb
;
7647 add_loop (loop
, loop_header
->loop_father
);
7649 if (gimple_in_ssa_p (cfun
))
7650 update_ssa (TODO_update_ssa_no_phi
);
7655 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7657 GOMP_atomic_start ();
7661 The result is not globally atomic, but works so long as all parallel
7662 references are within #pragma omp atomic directives. According to
7663 responses received from omp@openmp.org, appears to be within spec.
7664 Which makes sense, since that's how several other compilers handle
7665 this situation as well.
7666 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7667 expanding. STORED_VAL is the operand of the matching
7668 GIMPLE_OMP_ATOMIC_STORE.
7671 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7675 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7680 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7681 tree addr
, tree loaded_val
, tree stored_val
)
7683 gimple_stmt_iterator si
;
7687 si
= gsi_last_bb (load_bb
);
7688 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7690 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7691 t
= build_call_expr (t
, 0);
7692 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7694 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7695 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7696 gsi_remove (&si
, true);
7698 si
= gsi_last_bb (store_bb
);
7699 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7701 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7703 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7705 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7706 t
= build_call_expr (t
, 0);
7707 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7708 gsi_remove (&si
, true);
7710 if (gimple_in_ssa_p (cfun
))
7711 update_ssa (TODO_update_ssa_no_phi
);
7715 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7716 using expand_omp_atomic_fetch_op. If it failed, we try to
7717 call expand_omp_atomic_pipeline, and if it fails too, the
7718 ultimate fallback is wrapping the operation in a mutex
7719 (expand_omp_atomic_mutex). REGION is the atomic region built
7720 by build_omp_regions_1(). */
7723 expand_omp_atomic (struct omp_region
*region
)
7725 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7726 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7727 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7728 tree addr
= gimple_omp_atomic_load_rhs (load
);
7729 tree stored_val
= gimple_omp_atomic_store_val (store
);
7730 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7731 HOST_WIDE_INT index
;
7733 /* Make sure the type is one of the supported sizes. */
7734 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7735 index
= exact_log2 (index
);
7736 if (index
>= 0 && index
<= 4)
7738 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7740 /* __sync builtins require strict data alignment. */
7741 if (exact_log2 (align
) >= index
)
7744 if (loaded_val
== stored_val
7745 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7746 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7747 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7748 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7752 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7753 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7754 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7755 && store_bb
== single_succ (load_bb
)
7756 && first_stmt (store_bb
) == store
7757 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7761 /* When possible, use specialized atomic update functions. */
7762 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7763 && store_bb
== single_succ (load_bb
)
7764 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7765 loaded_val
, stored_val
, index
))
7768 /* If we don't have specialized __sync builtins, try and implement
7769 as a compare and swap loop. */
7770 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7771 loaded_val
, stored_val
, index
))
7776 /* The ultimate fallback is wrapping the operation in a mutex. */
7777 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7781 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7784 expand_omp_target (struct omp_region
*region
)
7786 basic_block entry_bb
, exit_bb
, new_bb
;
7787 struct function
*child_cfun
= NULL
;
7788 tree child_fn
= NULL_TREE
, block
, t
;
7789 gimple_stmt_iterator gsi
;
7790 gimple entry_stmt
, stmt
;
7793 entry_stmt
= last_stmt (region
->entry
);
7794 new_bb
= region
->entry
;
7795 int kind
= gimple_omp_target_kind (entry_stmt
);
7796 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7798 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7799 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7802 entry_bb
= region
->entry
;
7803 exit_bb
= region
->exit
;
7805 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7807 unsigned srcidx
, dstidx
, num
;
7809 /* If the target region needs data sent from the parent
7810 function, then the very first statement (except possible
7811 tree profile counter updates) of the parallel body
7812 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7813 &.OMP_DATA_O is passed as an argument to the child function,
7814 we need to replace it with the argument as seen by the child
7817 In most cases, this will end up being the identity assignment
7818 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7819 a function call that has been inlined, the original PARM_DECL
7820 .OMP_DATA_I may have been converted into a different local
7821 variable. In which case, we need to keep the assignment. */
7822 if (gimple_omp_target_data_arg (entry_stmt
))
7824 basic_block entry_succ_bb
= single_succ (entry_bb
);
7825 gimple_stmt_iterator gsi
;
7827 gimple tgtcopy_stmt
= NULL
;
7829 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7831 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7833 gcc_assert (!gsi_end_p (gsi
));
7834 stmt
= gsi_stmt (gsi
);
7835 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7838 if (gimple_num_ops (stmt
) == 2)
7840 tree arg
= gimple_assign_rhs1 (stmt
);
7842 /* We're ignoring the subcode because we're
7843 effectively doing a STRIP_NOPS. */
7845 if (TREE_CODE (arg
) == ADDR_EXPR
7846 && TREE_OPERAND (arg
, 0) == sender
)
7848 tgtcopy_stmt
= stmt
;
7854 gcc_assert (tgtcopy_stmt
!= NULL
);
7855 arg
= DECL_ARGUMENTS (child_fn
);
7857 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7858 gsi_remove (&gsi
, true);
7861 /* Declare local variables needed in CHILD_CFUN. */
7862 block
= DECL_INITIAL (child_fn
);
7863 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7864 /* The gimplifier could record temporaries in target block
7865 rather than in containing function's local_decls chain,
7866 which would mean cgraph missed finalizing them. Do it now. */
7867 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7868 if (TREE_CODE (t
) == VAR_DECL
7870 && !DECL_EXTERNAL (t
))
7871 varpool_finalize_decl (t
);
7872 DECL_SAVED_TREE (child_fn
) = NULL
;
7873 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7874 gimple_set_body (child_fn
, NULL
);
7875 TREE_USED (block
) = 1;
7877 /* Reset DECL_CONTEXT on function arguments. */
7878 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7879 DECL_CONTEXT (t
) = child_fn
;
7881 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7882 so that it can be moved to the child function. */
7883 gsi
= gsi_last_bb (entry_bb
);
7884 stmt
= gsi_stmt (gsi
);
7885 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7886 && gimple_omp_target_kind (stmt
)
7887 == GF_OMP_TARGET_KIND_REGION
);
7888 gsi_remove (&gsi
, true);
7889 e
= split_block (entry_bb
, stmt
);
7891 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7893 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7896 gsi
= gsi_last_bb (exit_bb
);
7897 gcc_assert (!gsi_end_p (gsi
)
7898 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7899 stmt
= gimple_build_return (NULL
);
7900 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7901 gsi_remove (&gsi
, true);
7904 /* Move the target region into CHILD_CFUN. */
7906 block
= gimple_block (entry_stmt
);
7908 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7910 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7911 /* When the OMP expansion process cannot guarantee an up-to-date
7912 loop tree arrange for the child function to fixup loops. */
7913 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7914 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7916 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7917 num
= vec_safe_length (child_cfun
->local_decls
);
7918 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7920 t
= (*child_cfun
->local_decls
)[srcidx
];
7921 if (DECL_CONTEXT (t
) == cfun
->decl
)
7923 if (srcidx
!= dstidx
)
7924 (*child_cfun
->local_decls
)[dstidx
] = t
;
7928 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7930 /* Inform the callgraph about the new function. */
7931 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
7932 cgraph_add_new_function (child_fn
, true);
7934 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7935 fixed in a following pass. */
7936 push_cfun (child_cfun
);
7937 rebuild_cgraph_edges ();
7939 /* Some EH regions might become dead, see PR34608. If
7940 pass_cleanup_cfg isn't the first pass to happen with the
7941 new child, these dead EH edges might cause problems.
7942 Clean them up now. */
7943 if (flag_exceptions
)
7946 bool changed
= false;
7948 FOR_EACH_BB_FN (bb
, cfun
)
7949 changed
|= gimple_purge_dead_eh_edges (bb
);
7951 cleanup_tree_cfg ();
7956 /* Emit a library call to launch the target region, or do data
7958 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
7959 enum built_in_function start_ix
;
7960 location_t clause_loc
;
7962 clauses
= gimple_omp_target_clauses (entry_stmt
);
7964 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7965 start_ix
= BUILT_IN_GOMP_TARGET
;
7966 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
7967 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
7969 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
7971 /* By default, the value of DEVICE is -1 (let runtime library choose)
7972 and there is no conditional. */
7974 device
= build_int_cst (integer_type_node
, -1);
7976 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
7978 cond
= OMP_CLAUSE_IF_EXPR (c
);
7980 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
7983 device
= OMP_CLAUSE_DEVICE_ID (c
);
7984 clause_loc
= OMP_CLAUSE_LOCATION (c
);
7987 clause_loc
= gimple_location (entry_stmt
);
7989 /* Ensure 'device' is of the correct type. */
7990 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
7992 /* If we found the clause 'if (cond)', build
7993 (cond ? device : -2). */
7996 cond
= gimple_boolify (cond
);
7998 basic_block cond_bb
, then_bb
, else_bb
;
8002 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8003 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8005 gsi
= gsi_last_bb (new_bb
);
8007 e
= split_block (new_bb
, gsi_stmt (gsi
));
8010 e
= split_block (new_bb
, NULL
);
8015 then_bb
= create_empty_bb (cond_bb
);
8016 else_bb
= create_empty_bb (then_bb
);
8017 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8018 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8020 stmt
= gimple_build_cond_empty (cond
);
8021 gsi
= gsi_last_bb (cond_bb
);
8022 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8024 gsi
= gsi_start_bb (then_bb
);
8025 stmt
= gimple_build_assign (tmp_var
, device
);
8026 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8028 gsi
= gsi_start_bb (else_bb
);
8029 stmt
= gimple_build_assign (tmp_var
,
8030 build_int_cst (integer_type_node
, -2));
8031 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8033 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8034 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8037 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8038 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8040 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8041 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8046 gsi
= gsi_last_bb (new_bb
);
8047 t
= gimple_omp_target_data_arg (entry_stmt
);
8050 t1
= size_zero_node
;
8051 t2
= build_zero_cst (ptr_type_node
);
8057 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8058 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8059 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8060 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8061 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8065 /* FIXME: This will be address of
8066 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8067 symbol, as soon as the linker plugin is able to create it for us. */
8068 tree openmp_target
= build_zero_cst (ptr_type_node
);
8069 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8071 tree fnaddr
= build_fold_addr_expr (child_fn
);
8072 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8073 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8076 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8077 device
, openmp_target
, t1
, t2
, t3
, t4
);
8078 gimple_set_location (g
, gimple_location (entry_stmt
));
8079 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8080 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8083 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8084 gsi_remove (&gsi
, true);
8086 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8088 gsi
= gsi_last_bb (region
->exit
);
8090 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8091 gsi_remove (&gsi
, true);
8096 /* Expand the parallel region tree rooted at REGION. Expansion
8097 proceeds in depth-first order. Innermost regions are expanded
8098 first. This way, parallel regions that require a new function to
8099 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8100 internal dependencies in their body. */
8103 expand_omp (struct omp_region
*region
)
8107 location_t saved_location
;
8108 gimple inner_stmt
= NULL
;
8110 /* First, determine whether this is a combined parallel+workshare
8112 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8113 determine_parallel_type (region
);
8115 if (region
->type
== GIMPLE_OMP_FOR
8116 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8117 inner_stmt
= last_stmt (region
->inner
->entry
);
8120 expand_omp (region
->inner
);
8122 saved_location
= input_location
;
8123 if (gimple_has_location (last_stmt (region
->entry
)))
8124 input_location
= gimple_location (last_stmt (region
->entry
));
8126 switch (region
->type
)
8128 case GIMPLE_OMP_PARALLEL
:
8129 case GIMPLE_OMP_TASK
:
8130 expand_omp_taskreg (region
);
8133 case GIMPLE_OMP_FOR
:
8134 expand_omp_for (region
, inner_stmt
);
8137 case GIMPLE_OMP_SECTIONS
:
8138 expand_omp_sections (region
);
8141 case GIMPLE_OMP_SECTION
:
8142 /* Individual omp sections are handled together with their
8143 parent GIMPLE_OMP_SECTIONS region. */
8146 case GIMPLE_OMP_SINGLE
:
8147 expand_omp_single (region
);
8150 case GIMPLE_OMP_MASTER
:
8151 case GIMPLE_OMP_TASKGROUP
:
8152 case GIMPLE_OMP_ORDERED
:
8153 case GIMPLE_OMP_CRITICAL
:
8154 case GIMPLE_OMP_TEAMS
:
8155 expand_omp_synch (region
);
8158 case GIMPLE_OMP_ATOMIC_LOAD
:
8159 expand_omp_atomic (region
);
8162 case GIMPLE_OMP_TARGET
:
8163 expand_omp_target (region
);
8170 input_location
= saved_location
;
8171 region
= region
->next
;
8176 /* Helper for build_omp_regions. Scan the dominator tree starting at
8177 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8178 true, the function ends once a single tree is built (otherwise, whole
8179 forest of OMP constructs may be built). */
8182 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8185 gimple_stmt_iterator gsi
;
8189 gsi
= gsi_last_bb (bb
);
8190 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8192 struct omp_region
*region
;
8193 enum gimple_code code
;
8195 stmt
= gsi_stmt (gsi
);
8196 code
= gimple_code (stmt
);
8197 if (code
== GIMPLE_OMP_RETURN
)
8199 /* STMT is the return point out of region PARENT. Mark it
8200 as the exit point and make PARENT the immediately
8201 enclosing region. */
8202 gcc_assert (parent
);
8205 parent
= parent
->outer
;
8207 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8209 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8210 GIMPLE_OMP_RETURN, but matches with
8211 GIMPLE_OMP_ATOMIC_LOAD. */
8212 gcc_assert (parent
);
8213 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8216 parent
= parent
->outer
;
8219 else if (code
== GIMPLE_OMP_CONTINUE
)
8221 gcc_assert (parent
);
8224 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8226 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8227 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8230 else if (code
== GIMPLE_OMP_TARGET
8231 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8232 new_omp_region (bb
, code
, parent
);
8235 /* Otherwise, this directive becomes the parent for a new
8237 region
= new_omp_region (bb
, code
, parent
);
8242 if (single_tree
&& !parent
)
8245 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8247 son
= next_dom_son (CDI_DOMINATORS
, son
))
8248 build_omp_regions_1 (son
, parent
, single_tree
);
8251 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8255 build_omp_regions_root (basic_block root
)
8257 gcc_assert (root_omp_region
== NULL
);
8258 build_omp_regions_1 (root
, NULL
, true);
8259 gcc_assert (root_omp_region
!= NULL
);
8262 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8265 omp_expand_local (basic_block head
)
8267 build_omp_regions_root (head
);
8268 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8270 fprintf (dump_file
, "\nOMP region tree\n\n");
8271 dump_omp_region (dump_file
, root_omp_region
, 0);
8272 fprintf (dump_file
, "\n");
8275 remove_exit_barriers (root_omp_region
);
8276 expand_omp (root_omp_region
);
8278 free_omp_regions ();
8281 /* Scan the CFG and build a tree of OMP regions. Return the root of
8282 the OMP region tree. */
8285 build_omp_regions (void)
8287 gcc_assert (root_omp_region
== NULL
);
8288 calculate_dominance_info (CDI_DOMINATORS
);
8289 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8292 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8295 execute_expand_omp (void)
8297 build_omp_regions ();
8299 if (!root_omp_region
)
8304 fprintf (dump_file
, "\nOMP region tree\n\n");
8305 dump_omp_region (dump_file
, root_omp_region
, 0);
8306 fprintf (dump_file
, "\n");
8309 remove_exit_barriers (root_omp_region
);
8311 expand_omp (root_omp_region
);
8313 cleanup_tree_cfg ();
8315 free_omp_regions ();
8320 /* OMP expansion -- the default pass, run before creation of SSA form. */
8323 gate_expand_omp (void)
8325 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8326 || flag_enable_cilkplus
!= 0) && !seen_error ());
8331 const pass_data pass_data_expand_omp
=
8333 GIMPLE_PASS
, /* type */
8334 "ompexp", /* name */
8335 OPTGROUP_NONE
, /* optinfo_flags */
8336 true, /* has_gate */
8337 true, /* has_execute */
8338 TV_NONE
, /* tv_id */
8339 PROP_gimple_any
, /* properties_required */
8340 0, /* properties_provided */
8341 0, /* properties_destroyed */
8342 0, /* todo_flags_start */
8343 0, /* todo_flags_finish */
8346 class pass_expand_omp
: public gimple_opt_pass
8349 pass_expand_omp (gcc::context
*ctxt
)
8350 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8353 /* opt_pass methods: */
8354 bool gate () { return gate_expand_omp (); }
8355 unsigned int execute () { return execute_expand_omp (); }
8357 }; // class pass_expand_omp
8362 make_pass_expand_omp (gcc::context
*ctxt
)
8364 return new pass_expand_omp (ctxt
);
8367 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8369 /* If ctx is a worksharing context inside of a cancellable parallel
8370 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8371 and conditional branch to parallel's cancel_label to handle
8372 cancellation in the implicit barrier. */
8375 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8377 gimple omp_return
= gimple_seq_last_stmt (*body
);
8378 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8379 if (gimple_omp_return_nowait_p (omp_return
))
8382 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8383 && ctx
->outer
->cancellable
)
8385 tree lhs
= create_tmp_var (boolean_type_node
, NULL
);
8386 gimple_omp_return_set_lhs (omp_return
, lhs
);
8387 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8388 gimple g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
8389 ctx
->outer
->cancel_label
, fallthru_label
);
8390 gimple_seq_add_stmt (body
, g
);
8391 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8395 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8396 CTX is the enclosing OMP context for the current statement. */
8399 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8401 tree block
, control
;
8402 gimple_stmt_iterator tgsi
;
8403 gimple stmt
, new_stmt
, bind
, t
;
8404 gimple_seq ilist
, dlist
, olist
, new_body
;
8406 stmt
= gsi_stmt (*gsi_p
);
8408 push_gimplify_context ();
8412 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8413 &ilist
, &dlist
, ctx
, NULL
);
8415 new_body
= gimple_omp_body (stmt
);
8416 gimple_omp_set_body (stmt
, NULL
);
8417 tgsi
= gsi_start (new_body
);
8418 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8423 sec_start
= gsi_stmt (tgsi
);
8424 sctx
= maybe_lookup_ctx (sec_start
);
8427 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8428 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8429 GSI_CONTINUE_LINKING
);
8430 gimple_omp_set_body (sec_start
, NULL
);
8432 if (gsi_one_before_end_p (tgsi
))
8434 gimple_seq l
= NULL
;
8435 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8437 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8438 gimple_omp_section_set_last (sec_start
);
8441 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8442 GSI_CONTINUE_LINKING
);
8445 block
= make_node (BLOCK
);
8446 bind
= gimple_build_bind (NULL
, new_body
, block
);
8449 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8451 block
= make_node (BLOCK
);
8452 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8453 gsi_replace (gsi_p
, new_stmt
, true);
8455 pop_gimplify_context (new_stmt
);
8456 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8457 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8458 if (BLOCK_VARS (block
))
8459 TREE_USED (block
) = 1;
8462 gimple_seq_add_seq (&new_body
, ilist
);
8463 gimple_seq_add_stmt (&new_body
, stmt
);
8464 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8465 gimple_seq_add_stmt (&new_body
, bind
);
8467 control
= create_tmp_var (unsigned_type_node
, ".section");
8468 t
= gimple_build_omp_continue (control
, control
);
8469 gimple_omp_sections_set_control (stmt
, control
);
8470 gimple_seq_add_stmt (&new_body
, t
);
8472 gimple_seq_add_seq (&new_body
, olist
);
8473 if (ctx
->cancellable
)
8474 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8475 gimple_seq_add_seq (&new_body
, dlist
);
8477 new_body
= maybe_catch_exception (new_body
);
8479 t
= gimple_build_omp_return
8480 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8481 OMP_CLAUSE_NOWAIT
));
8482 gimple_seq_add_stmt (&new_body
, t
);
8483 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8485 gimple_bind_set_body (new_stmt
, new_body
);
8489 /* A subroutine of lower_omp_single. Expand the simple form of
8490 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8492 if (GOMP_single_start ())
8494 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8496 FIXME. It may be better to delay expanding the logic of this until
8497 pass_expand_omp. The expanded logic may make the job more difficult
8498 to a synchronization analysis pass. */
8501 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8503 location_t loc
= gimple_location (single_stmt
);
8504 tree tlabel
= create_artificial_label (loc
);
8505 tree flabel
= create_artificial_label (loc
);
8509 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8510 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8511 call
= gimple_build_call (decl
, 0);
8512 gimple_call_set_lhs (call
, lhs
);
8513 gimple_seq_add_stmt (pre_p
, call
);
8515 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8516 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8519 gimple_seq_add_stmt (pre_p
, cond
);
8520 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8521 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8522 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8526 /* A subroutine of lower_omp_single. Expand the simple form of
8527 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8529 #pragma omp single copyprivate (a, b, c)
8531 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8534 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8540 GOMP_single_copy_end (©out);
8551 FIXME. It may be better to delay expanding the logic of this until
8552 pass_expand_omp. The expanded logic may make the job more difficult
8553 to a synchronization analysis pass. */
8556 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8558 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8559 gimple_seq copyin_seq
;
8560 location_t loc
= gimple_location (single_stmt
);
8562 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8564 ptr_type
= build_pointer_type (ctx
->record_type
);
8565 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8567 l0
= create_artificial_label (loc
);
8568 l1
= create_artificial_label (loc
);
8569 l2
= create_artificial_label (loc
);
8571 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8572 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8573 t
= fold_convert_loc (loc
, ptr_type
, t
);
8574 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8576 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8577 build_int_cst (ptr_type
, 0));
8578 t
= build3 (COND_EXPR
, void_type_node
, t
,
8579 build_and_jump (&l0
), build_and_jump (&l1
));
8580 gimplify_and_add (t
, pre_p
);
8582 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8584 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8587 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8590 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8591 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8592 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8593 gimplify_and_add (t
, pre_p
);
8595 t
= build_and_jump (&l2
);
8596 gimplify_and_add (t
, pre_p
);
8598 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8600 gimple_seq_add_seq (pre_p
, copyin_seq
);
8602 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8606 /* Expand code for an OpenMP single directive. */
8609 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8612 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8613 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8615 push_gimplify_context ();
8617 block
= make_node (BLOCK
);
8618 bind
= gimple_build_bind (NULL
, NULL
, block
);
8619 gsi_replace (gsi_p
, bind
, true);
8622 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8623 &bind_body
, &dlist
, ctx
, NULL
);
8624 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8626 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8628 if (ctx
->record_type
)
8629 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8631 lower_omp_single_simple (single_stmt
, &bind_body
);
8633 gimple_omp_set_body (single_stmt
, NULL
);
8635 gimple_seq_add_seq (&bind_body
, dlist
);
8637 bind_body
= maybe_catch_exception (bind_body
);
8639 t
= gimple_build_omp_return
8640 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8641 OMP_CLAUSE_NOWAIT
));
8642 gimple_seq_add_stmt (&bind_body_tail
, t
);
8643 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8644 if (ctx
->record_type
)
8646 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8647 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8648 TREE_THIS_VOLATILE (clobber
) = 1;
8649 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8650 clobber
), GSI_SAME_STMT
);
8652 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8653 gimple_bind_set_body (bind
, bind_body
);
8655 pop_gimplify_context (bind
);
8657 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8658 BLOCK_VARS (block
) = ctx
->block_vars
;
8659 if (BLOCK_VARS (block
))
8660 TREE_USED (block
) = 1;
8664 /* Expand code for an OpenMP master directive. */
8667 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8669 tree block
, lab
= NULL
, x
, bfn_decl
;
8670 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8671 location_t loc
= gimple_location (stmt
);
8674 push_gimplify_context ();
8676 block
= make_node (BLOCK
);
8677 bind
= gimple_build_bind (NULL
, NULL
, block
);
8678 gsi_replace (gsi_p
, bind
, true);
8679 gimple_bind_add_stmt (bind
, stmt
);
8681 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8682 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8683 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8684 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8686 gimplify_and_add (x
, &tseq
);
8687 gimple_bind_add_seq (bind
, tseq
);
8689 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8690 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8691 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8692 gimple_omp_set_body (stmt
, NULL
);
8694 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8696 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8698 pop_gimplify_context (bind
);
8700 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8701 BLOCK_VARS (block
) = ctx
->block_vars
;
8705 /* Expand code for an OpenMP taskgroup directive. */
8708 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8710 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8711 tree block
= make_node (BLOCK
);
8713 bind
= gimple_build_bind (NULL
, NULL
, block
);
8714 gsi_replace (gsi_p
, bind
, true);
8715 gimple_bind_add_stmt (bind
, stmt
);
8717 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8719 gimple_bind_add_stmt (bind
, x
);
8721 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8722 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8723 gimple_omp_set_body (stmt
, NULL
);
8725 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8727 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8728 BLOCK_VARS (block
) = ctx
->block_vars
;
8732 /* Expand code for an OpenMP ordered directive. */
8735 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8738 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8740 push_gimplify_context ();
8742 block
= make_node (BLOCK
);
8743 bind
= gimple_build_bind (NULL
, NULL
, block
);
8744 gsi_replace (gsi_p
, bind
, true);
8745 gimple_bind_add_stmt (bind
, stmt
);
8747 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8749 gimple_bind_add_stmt (bind
, x
);
8751 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8752 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8753 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8754 gimple_omp_set_body (stmt
, NULL
);
8756 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8757 gimple_bind_add_stmt (bind
, x
);
8759 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8761 pop_gimplify_context (bind
);
8763 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8764 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8768 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8769 substitution of a couple of function calls. But in the NAMED case,
8770 requires that languages coordinate a symbol name. It is therefore
8771 best put here in common code. */
8773 static GTY((param1_is (tree
), param2_is (tree
)))
8774 splay_tree critical_name_mutexes
;
8777 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8780 tree name
, lock
, unlock
;
8781 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8782 location_t loc
= gimple_location (stmt
);
8785 name
= gimple_omp_critical_name (stmt
);
8791 if (!critical_name_mutexes
)
8792 critical_name_mutexes
8793 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8794 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8795 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8797 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8802 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8804 new_str
= ACONCAT ((".gomp_critical_user_",
8805 IDENTIFIER_POINTER (name
), NULL
));
8806 DECL_NAME (decl
) = get_identifier (new_str
);
8807 TREE_PUBLIC (decl
) = 1;
8808 TREE_STATIC (decl
) = 1;
8809 DECL_COMMON (decl
) = 1;
8810 DECL_ARTIFICIAL (decl
) = 1;
8811 DECL_IGNORED_P (decl
) = 1;
8812 varpool_finalize_decl (decl
);
8814 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8815 (splay_tree_value
) decl
);
8818 decl
= (tree
) n
->value
;
8820 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8821 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8823 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8824 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8825 build_fold_addr_expr_loc (loc
, decl
));
8829 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8830 lock
= build_call_expr_loc (loc
, lock
, 0);
8832 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8833 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8836 push_gimplify_context ();
8838 block
= make_node (BLOCK
);
8839 bind
= gimple_build_bind (NULL
, NULL
, block
);
8840 gsi_replace (gsi_p
, bind
, true);
8841 gimple_bind_add_stmt (bind
, stmt
);
8843 tbody
= gimple_bind_body (bind
);
8844 gimplify_and_add (lock
, &tbody
);
8845 gimple_bind_set_body (bind
, tbody
);
8847 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8848 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8849 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8850 gimple_omp_set_body (stmt
, NULL
);
8852 tbody
= gimple_bind_body (bind
);
8853 gimplify_and_add (unlock
, &tbody
);
8854 gimple_bind_set_body (bind
, tbody
);
8856 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8858 pop_gimplify_context (bind
);
8859 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8860 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8864 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8865 for a lastprivate clause. Given a loop control predicate of (V
8866 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8867 is appended to *DLIST, iterator initialization is appended to
8871 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8872 gimple_seq
*dlist
, struct omp_context
*ctx
)
8874 tree clauses
, cond
, vinit
;
8875 enum tree_code cond_code
;
8878 cond_code
= fd
->loop
.cond_code
;
8879 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8881 /* When possible, use a strict equality expression. This can let VRP
8882 type optimizations deduce the value and remove a copy. */
8883 if (tree_fits_shwi_p (fd
->loop
.step
))
8885 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
8886 if (step
== 1 || step
== -1)
8887 cond_code
= EQ_EXPR
;
8890 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8892 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8894 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8895 if (!gimple_seq_empty_p (stmts
))
8897 gimple_seq_add_seq (&stmts
, *dlist
);
8900 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8901 vinit
= fd
->loop
.n1
;
8902 if (cond_code
== EQ_EXPR
8903 && tree_fits_shwi_p (fd
->loop
.n2
)
8904 && ! integer_zerop (fd
->loop
.n2
))
8905 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8907 vinit
= unshare_expr (vinit
);
8909 /* Initialize the iterator variable, so that threads that don't execute
8910 any iterations don't execute the lastprivate clauses by accident. */
8911 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
8916 /* Lower code for an OpenMP loop directive. */
8919 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8922 struct omp_for_data fd
, *fdp
= NULL
;
8923 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
8924 gimple_seq omp_for_body
, body
, dlist
;
8927 push_gimplify_context ();
8929 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
8931 block
= make_node (BLOCK
);
8932 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8933 /* Replace at gsi right away, so that 'stmt' is no member
8934 of a sequence anymore as we're going to add to to a different
8936 gsi_replace (gsi_p
, new_stmt
, true);
8938 /* Move declaration of temporaries in the loop body before we make
8940 omp_for_body
= gimple_omp_body (stmt
);
8941 if (!gimple_seq_empty_p (omp_for_body
)
8942 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
8944 tree vars
= gimple_bind_vars (gimple_seq_first_stmt (omp_for_body
));
8945 gimple_bind_append_vars (new_stmt
, vars
);
8948 if (gimple_omp_for_combined_into_p (stmt
))
8950 extract_omp_for_data (stmt
, &fd
, NULL
);
8953 /* We need two temporaries with fd.loop.v type (istart/iend)
8954 and then (fd.collapse - 1) temporaries with the same
8955 type for count2 ... countN-1 vars if not constant. */
8957 tree type
= fd
.iter_type
;
8959 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
8960 count
+= fd
.collapse
- 1;
8961 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
8962 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
8966 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
8967 OMP_CLAUSE__LOOPTEMP_
);
8968 for (i
= 0; i
< count
; i
++)
8973 gcc_assert (outerc
);
8974 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
8975 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
8976 OMP_CLAUSE__LOOPTEMP_
);
8979 temp
= create_tmp_var (type
, NULL
);
8980 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
8981 OMP_CLAUSE_DECL (*pc
) = temp
;
8982 pc
= &OMP_CLAUSE_CHAIN (*pc
);
8987 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
8990 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
8992 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
8994 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8996 /* Lower the header expressions. At this point, we can assume that
8997 the header is of the form:
8999 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9001 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9002 using the .omp_data_s mapping, if needed. */
9003 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9005 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9006 if (!is_gimple_min_invariant (*rhs_p
))
9007 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9009 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9010 if (!is_gimple_min_invariant (*rhs_p
))
9011 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9013 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9014 if (!is_gimple_min_invariant (*rhs_p
))
9015 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9018 /* Once lowered, extract the bounds and clauses. */
9019 extract_omp_for_data (stmt
, &fd
, NULL
);
9021 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9023 gimple_seq_add_stmt (&body
, stmt
);
9024 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9026 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9029 /* After the loop, add exit clauses. */
9030 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9032 if (ctx
->cancellable
)
9033 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9035 gimple_seq_add_seq (&body
, dlist
);
9037 body
= maybe_catch_exception (body
);
9039 /* Region exit marker goes at the end of the loop body. */
9040 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9041 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9042 pop_gimplify_context (new_stmt
);
9044 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9045 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9046 if (BLOCK_VARS (block
))
9047 TREE_USED (block
) = 1;
9049 gimple_bind_set_body (new_stmt
, body
);
9050 gimple_omp_set_body (stmt
, NULL
);
9051 gimple_omp_for_set_pre_body (stmt
, NULL
);
9054 /* Callback for walk_stmts. Check if the current statement only contains
9055 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
9058 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9059 bool *handled_ops_p
,
9060 struct walk_stmt_info
*wi
)
9062 int *info
= (int *) wi
->info
;
9063 gimple stmt
= gsi_stmt (*gsi_p
);
9065 *handled_ops_p
= true;
9066 switch (gimple_code (stmt
))
9070 case GIMPLE_OMP_FOR
:
9071 case GIMPLE_OMP_SECTIONS
:
9072 *info
= *info
== 0 ? 1 : -1;
9081 struct omp_taskcopy_context
9083 /* This field must be at the beginning, as we do "inheritance": Some
9084 callback functions for tree-inline.c (e.g., omp_copy_decl)
9085 receive a copy_body_data pointer that is up-casted to an
9086 omp_context pointer. */
9092 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9094 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9096 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9097 return create_tmp_var (TREE_TYPE (var
), NULL
);
9103 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9105 tree name
, new_fields
= NULL
, type
, f
;
9107 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9108 name
= DECL_NAME (TYPE_NAME (orig_type
));
9109 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9110 TYPE_DECL
, name
, type
);
9111 TYPE_NAME (type
) = name
;
9113 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9115 tree new_f
= copy_node (f
);
9116 DECL_CONTEXT (new_f
) = type
;
9117 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9118 TREE_CHAIN (new_f
) = new_fields
;
9119 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9120 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9121 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9124 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9126 TYPE_FIELDS (type
) = nreverse (new_fields
);
9131 /* Create task copyfn. */
9134 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9136 struct function
*child_cfun
;
9137 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9138 tree record_type
, srecord_type
, bind
, list
;
9139 bool record_needs_remap
= false, srecord_needs_remap
= false;
9141 struct omp_taskcopy_context tcctx
;
9142 location_t loc
= gimple_location (task_stmt
);
9144 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9145 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9146 gcc_assert (child_cfun
->cfg
== NULL
);
9147 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9149 /* Reset DECL_CONTEXT on function arguments. */
9150 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9151 DECL_CONTEXT (t
) = child_fn
;
9153 /* Populate the function. */
9154 push_gimplify_context ();
9155 push_cfun (child_cfun
);
9157 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9158 TREE_SIDE_EFFECTS (bind
) = 1;
9160 DECL_SAVED_TREE (child_fn
) = bind
;
9161 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9163 /* Remap src and dst argument types if needed. */
9164 record_type
= ctx
->record_type
;
9165 srecord_type
= ctx
->srecord_type
;
9166 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9167 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9169 record_needs_remap
= true;
9172 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9173 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9175 srecord_needs_remap
= true;
9179 if (record_needs_remap
|| srecord_needs_remap
)
9181 memset (&tcctx
, '\0', sizeof (tcctx
));
9182 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9183 tcctx
.cb
.dst_fn
= child_fn
;
9184 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9185 gcc_checking_assert (tcctx
.cb
.src_node
);
9186 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9187 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9188 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9189 tcctx
.cb
.eh_lp_nr
= 0;
9190 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9191 tcctx
.cb
.decl_map
= pointer_map_create ();
9194 if (record_needs_remap
)
9195 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9196 if (srecord_needs_remap
)
9197 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9200 tcctx
.cb
.decl_map
= NULL
;
9202 arg
= DECL_ARGUMENTS (child_fn
);
9203 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9204 sarg
= DECL_CHAIN (arg
);
9205 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9207 /* First pass: initialize temporaries used in record_type and srecord_type
9208 sizes and field offsets. */
9209 if (tcctx
.cb
.decl_map
)
9210 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9211 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9215 decl
= OMP_CLAUSE_DECL (c
);
9216 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9219 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9220 sf
= (tree
) n
->value
;
9221 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9222 src
= build_simple_mem_ref_loc (loc
, sarg
);
9223 src
= omp_build_component_ref (src
, sf
);
9224 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9225 append_to_statement_list (t
, &list
);
9228 /* Second pass: copy shared var pointers and copy construct non-VLA
9229 firstprivate vars. */
9230 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9231 switch (OMP_CLAUSE_CODE (c
))
9233 case OMP_CLAUSE_SHARED
:
9234 decl
= OMP_CLAUSE_DECL (c
);
9235 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9238 f
= (tree
) n
->value
;
9239 if (tcctx
.cb
.decl_map
)
9240 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9241 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9242 sf
= (tree
) n
->value
;
9243 if (tcctx
.cb
.decl_map
)
9244 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9245 src
= build_simple_mem_ref_loc (loc
, sarg
);
9246 src
= omp_build_component_ref (src
, sf
);
9247 dst
= build_simple_mem_ref_loc (loc
, arg
);
9248 dst
= omp_build_component_ref (dst
, f
);
9249 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9250 append_to_statement_list (t
, &list
);
9252 case OMP_CLAUSE_FIRSTPRIVATE
:
9253 decl
= OMP_CLAUSE_DECL (c
);
9254 if (is_variable_sized (decl
))
9256 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9259 f
= (tree
) n
->value
;
9260 if (tcctx
.cb
.decl_map
)
9261 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9262 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9265 sf
= (tree
) n
->value
;
9266 if (tcctx
.cb
.decl_map
)
9267 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9268 src
= build_simple_mem_ref_loc (loc
, sarg
);
9269 src
= omp_build_component_ref (src
, sf
);
9270 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9271 src
= build_simple_mem_ref_loc (loc
, src
);
9275 dst
= build_simple_mem_ref_loc (loc
, arg
);
9276 dst
= omp_build_component_ref (dst
, f
);
9277 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9278 append_to_statement_list (t
, &list
);
9280 case OMP_CLAUSE_PRIVATE
:
9281 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9283 decl
= OMP_CLAUSE_DECL (c
);
9284 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9285 f
= (tree
) n
->value
;
9286 if (tcctx
.cb
.decl_map
)
9287 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9288 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9291 sf
= (tree
) n
->value
;
9292 if (tcctx
.cb
.decl_map
)
9293 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9294 src
= build_simple_mem_ref_loc (loc
, sarg
);
9295 src
= omp_build_component_ref (src
, sf
);
9296 if (use_pointer_for_field (decl
, NULL
))
9297 src
= build_simple_mem_ref_loc (loc
, src
);
9301 dst
= build_simple_mem_ref_loc (loc
, arg
);
9302 dst
= omp_build_component_ref (dst
, f
);
9303 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9304 append_to_statement_list (t
, &list
);
9310 /* Last pass: handle VLA firstprivates. */
9311 if (tcctx
.cb
.decl_map
)
9312 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9313 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9317 decl
= OMP_CLAUSE_DECL (c
);
9318 if (!is_variable_sized (decl
))
9320 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9323 f
= (tree
) n
->value
;
9324 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9325 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9326 ind
= DECL_VALUE_EXPR (decl
);
9327 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9328 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9329 n
= splay_tree_lookup (ctx
->sfield_map
,
9330 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9331 sf
= (tree
) n
->value
;
9332 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9333 src
= build_simple_mem_ref_loc (loc
, sarg
);
9334 src
= omp_build_component_ref (src
, sf
);
9335 src
= build_simple_mem_ref_loc (loc
, src
);
9336 dst
= build_simple_mem_ref_loc (loc
, arg
);
9337 dst
= omp_build_component_ref (dst
, f
);
9338 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9339 append_to_statement_list (t
, &list
);
9340 n
= splay_tree_lookup (ctx
->field_map
,
9341 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9342 df
= (tree
) n
->value
;
9343 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9344 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9345 ptr
= omp_build_component_ref (ptr
, df
);
9346 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9347 build_fold_addr_expr_loc (loc
, dst
));
9348 append_to_statement_list (t
, &list
);
9351 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9352 append_to_statement_list (t
, &list
);
9354 if (tcctx
.cb
.decl_map
)
9355 pointer_map_destroy (tcctx
.cb
.decl_map
);
9356 pop_gimplify_context (NULL
);
9357 BIND_EXPR_BODY (bind
) = list
;
9362 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9366 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9368 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9370 gcc_assert (clauses
);
9371 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9372 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9373 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9375 case OMP_CLAUSE_DEPEND_IN
:
9378 case OMP_CLAUSE_DEPEND_OUT
:
9379 case OMP_CLAUSE_DEPEND_INOUT
:
9385 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9386 tree array
= create_tmp_var (type
, NULL
);
9387 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9389 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9390 gimple_seq_add_stmt (iseq
, g
);
9391 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9393 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9394 gimple_seq_add_stmt (iseq
, g
);
9395 for (i
= 0; i
< 2; i
++)
9397 if ((i
? n_in
: n_out
) == 0)
9399 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9400 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9401 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9403 tree t
= OMP_CLAUSE_DECL (c
);
9404 t
= fold_convert (ptr_type_node
, t
);
9405 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9406 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9407 NULL_TREE
, NULL_TREE
);
9408 g
= gimple_build_assign (r
, t
);
9409 gimple_seq_add_stmt (iseq
, g
);
9412 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9413 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9414 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9415 OMP_CLAUSE_CHAIN (c
) = *p
;
9417 tree clobber
= build_constructor (type
, NULL
);
9418 TREE_THIS_VOLATILE (clobber
) = 1;
9419 g
= gimple_build_assign (array
, clobber
);
9420 gimple_seq_add_stmt (oseq
, g
);
9423 /* Lower the OpenMP parallel or task directive in the current statement
9424 in GSI_P. CTX holds context information for the directive. */
9427 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9431 gimple stmt
= gsi_stmt (*gsi_p
);
9432 gimple par_bind
, bind
, dep_bind
= NULL
;
9433 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9434 location_t loc
= gimple_location (stmt
);
9436 clauses
= gimple_omp_taskreg_clauses (stmt
);
9437 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9438 par_body
= gimple_bind_body (par_bind
);
9439 child_fn
= ctx
->cb
.dst_fn
;
9440 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9441 && !gimple_omp_parallel_combined_p (stmt
))
9443 struct walk_stmt_info wi
;
9446 memset (&wi
, 0, sizeof (wi
));
9449 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9451 gimple_omp_parallel_set_combined_p (stmt
, true);
9453 gimple_seq dep_ilist
= NULL
;
9454 gimple_seq dep_olist
= NULL
;
9455 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9456 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9458 push_gimplify_context ();
9459 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9460 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9463 if (ctx
->srecord_type
)
9464 create_task_copyfn (stmt
, ctx
);
9466 push_gimplify_context ();
9471 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9472 lower_omp (&par_body
, ctx
);
9473 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9474 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9476 /* Declare all the variables created by mapping and the variables
9477 declared in the scope of the parallel body. */
9478 record_vars_into (ctx
->block_vars
, child_fn
);
9479 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9481 if (ctx
->record_type
)
9484 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9485 : ctx
->record_type
, ".omp_data_o");
9486 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9487 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9488 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9493 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9494 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9496 if (ctx
->record_type
)
9498 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9499 TREE_THIS_VOLATILE (clobber
) = 1;
9500 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9504 /* Once all the expansions are done, sequence all the different
9505 fragments inside gimple_omp_body. */
9509 if (ctx
->record_type
)
9511 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9512 /* fixup_child_record_type might have changed receiver_decl's type. */
9513 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9514 gimple_seq_add_stmt (&new_body
,
9515 gimple_build_assign (ctx
->receiver_decl
, t
));
9518 gimple_seq_add_seq (&new_body
, par_ilist
);
9519 gimple_seq_add_seq (&new_body
, par_body
);
9520 gimple_seq_add_seq (&new_body
, par_rlist
);
9521 if (ctx
->cancellable
)
9522 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9523 gimple_seq_add_seq (&new_body
, par_olist
);
9524 new_body
= maybe_catch_exception (new_body
);
9525 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9526 gimple_omp_set_body (stmt
, new_body
);
9528 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9529 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9530 gimple_bind_add_seq (bind
, ilist
);
9531 gimple_bind_add_stmt (bind
, stmt
);
9532 gimple_bind_add_seq (bind
, olist
);
9534 pop_gimplify_context (NULL
);
9538 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9539 gimple_bind_add_stmt (dep_bind
, bind
);
9540 gimple_bind_add_seq (dep_bind
, dep_olist
);
9541 pop_gimplify_context (dep_bind
);
9545 /* Lower the OpenMP target directive in the current statement
9546 in GSI_P. CTX holds context information for the directive. */
9549 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9552 tree child_fn
, t
, c
;
9553 gimple stmt
= gsi_stmt (*gsi_p
);
9554 gimple tgt_bind
= NULL
, bind
;
9555 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9556 location_t loc
= gimple_location (stmt
);
9557 int kind
= gimple_omp_target_kind (stmt
);
9558 unsigned int map_cnt
= 0;
9560 clauses
= gimple_omp_target_clauses (stmt
);
9561 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9563 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9564 tgt_body
= gimple_bind_body (tgt_bind
);
9566 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9567 tgt_body
= gimple_omp_body (stmt
);
9568 child_fn
= ctx
->cb
.dst_fn
;
9570 push_gimplify_context ();
9572 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9573 switch (OMP_CLAUSE_CODE (c
))
9579 case OMP_CLAUSE_MAP
:
9581 case OMP_CLAUSE_FROM
:
9582 var
= OMP_CLAUSE_DECL (c
);
9585 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9586 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9592 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9594 tree var2
= DECL_VALUE_EXPR (var
);
9595 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9596 var2
= TREE_OPERAND (var2
, 0);
9597 gcc_assert (DECL_P (var2
));
9601 if (!maybe_lookup_field (var
, ctx
))
9604 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9606 x
= build_receiver_ref (var
, true, ctx
);
9607 tree new_var
= lookup_decl (var
, ctx
);
9608 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9609 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9610 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9611 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9612 x
= build_simple_mem_ref (x
);
9613 SET_DECL_VALUE_EXPR (new_var
, x
);
9614 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9619 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9621 target_nesting_level
++;
9622 lower_omp (&tgt_body
, ctx
);
9623 target_nesting_level
--;
9625 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9626 lower_omp (&tgt_body
, ctx
);
9628 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9630 /* Declare all the variables created by mapping and the variables
9631 declared in the scope of the target body. */
9632 record_vars_into (ctx
->block_vars
, child_fn
);
9633 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9638 if (ctx
->record_type
)
9641 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9642 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9643 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9644 t
= make_tree_vec (3);
9645 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9647 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9649 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9650 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9651 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9653 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9656 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9657 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9658 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9659 gimple_omp_target_set_data_arg (stmt
, t
);
9661 vec
<constructor_elt
, va_gc
> *vsize
;
9662 vec
<constructor_elt
, va_gc
> *vkind
;
9663 vec_alloc (vsize
, map_cnt
);
9664 vec_alloc (vkind
, map_cnt
);
9665 unsigned int map_idx
= 0;
9667 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9668 switch (OMP_CLAUSE_CODE (c
))
9674 case OMP_CLAUSE_MAP
:
9676 case OMP_CLAUSE_FROM
:
9678 ovar
= OMP_CLAUSE_DECL (c
);
9681 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9682 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9684 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9685 == get_base_address (ovar
));
9686 nc
= OMP_CLAUSE_CHAIN (c
);
9687 ovar
= OMP_CLAUSE_DECL (nc
);
9691 tree x
= build_sender_ref (ovar
, ctx
);
9693 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9694 gimplify_assign (x
, v
, &ilist
);
9700 if (DECL_SIZE (ovar
)
9701 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9703 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9704 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9705 ovar2
= TREE_OPERAND (ovar2
, 0);
9706 gcc_assert (DECL_P (ovar2
));
9709 if (!maybe_lookup_field (ovar
, ctx
))
9715 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9716 tree x
= build_sender_ref (ovar
, ctx
);
9717 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9718 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9719 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9720 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9722 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9724 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9725 mark_addressable (avar
);
9726 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9727 avar
= build_fold_addr_expr (avar
);
9728 gimplify_assign (x
, avar
, &ilist
);
9730 else if (is_gimple_reg (var
))
9732 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9733 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9734 mark_addressable (avar
);
9735 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9736 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9737 gimplify_assign (avar
, var
, &ilist
);
9738 avar
= build_fold_addr_expr (avar
);
9739 gimplify_assign (x
, avar
, &ilist
);
9740 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9741 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9742 && !TYPE_READONLY (TREE_TYPE (var
)))
9744 x
= build_sender_ref (ovar
, ctx
);
9745 x
= build_simple_mem_ref (x
);
9746 gimplify_assign (var
, x
, &olist
);
9751 var
= build_fold_addr_expr (var
);
9752 gimplify_assign (x
, var
, &ilist
);
9755 tree s
= OMP_CLAUSE_SIZE (c
);
9757 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9758 s
= fold_convert (size_type_node
, s
);
9759 tree purpose
= size_int (map_idx
++);
9760 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9761 if (TREE_CODE (s
) != INTEGER_CST
)
9762 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9764 unsigned char tkind
= 0;
9765 switch (OMP_CLAUSE_CODE (c
))
9767 case OMP_CLAUSE_MAP
:
9768 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9771 tkind
= OMP_CLAUSE_MAP_TO
;
9773 case OMP_CLAUSE_FROM
:
9774 tkind
= OMP_CLAUSE_MAP_FROM
;
9779 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9780 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9781 talign
= DECL_ALIGN_UNIT (ovar
);
9782 talign
= ceil_log2 (talign
);
9783 tkind
|= talign
<< 3;
9784 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9785 build_int_cst (unsigned_char_type_node
,
9791 gcc_assert (map_idx
== map_cnt
);
9793 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9794 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9795 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9796 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9797 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9799 gimple_seq initlist
= NULL
;
9800 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9801 TREE_VEC_ELT (t
, 1)),
9802 &initlist
, true, NULL_TREE
);
9803 gimple_seq_add_seq (&ilist
, initlist
);
9806 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9807 TREE_THIS_VOLATILE (clobber
) = 1;
9808 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9812 /* Once all the expansions are done, sequence all the different
9813 fragments inside gimple_omp_body. */
9817 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9819 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9820 /* fixup_child_record_type might have changed receiver_decl's type. */
9821 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9822 gimple_seq_add_stmt (&new_body
,
9823 gimple_build_assign (ctx
->receiver_decl
, t
));
9826 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9828 gimple_seq_add_seq (&new_body
, tgt_body
);
9829 new_body
= maybe_catch_exception (new_body
);
9831 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9832 new_body
= tgt_body
;
9833 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9835 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9836 gimple_omp_set_body (stmt
, new_body
);
9839 bind
= gimple_build_bind (NULL
, NULL
,
9840 tgt_bind
? gimple_bind_block (tgt_bind
)
9842 gsi_replace (gsi_p
, bind
, true);
9843 gimple_bind_add_seq (bind
, ilist
);
9844 gimple_bind_add_stmt (bind
, stmt
);
9845 gimple_bind_add_seq (bind
, olist
);
9847 pop_gimplify_context (NULL
);
9850 /* Expand code for an OpenMP teams directive. */
9853 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9855 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9856 push_gimplify_context ();
9858 tree block
= make_node (BLOCK
);
9859 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9860 gsi_replace (gsi_p
, bind
, true);
9861 gimple_seq bind_body
= NULL
;
9862 gimple_seq dlist
= NULL
;
9863 gimple_seq olist
= NULL
;
9865 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9866 OMP_CLAUSE_NUM_TEAMS
);
9867 if (num_teams
== NULL_TREE
)
9868 num_teams
= build_int_cst (unsigned_type_node
, 0);
9871 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9872 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9873 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9875 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9876 OMP_CLAUSE_THREAD_LIMIT
);
9877 if (thread_limit
== NULL_TREE
)
9878 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9881 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9882 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9883 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9887 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9888 &bind_body
, &dlist
, ctx
, NULL
);
9889 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9890 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9891 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9893 location_t loc
= gimple_location (teams_stmt
);
9894 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9895 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9896 gimple_set_location (call
, loc
);
9897 gimple_seq_add_stmt (&bind_body
, call
);
9899 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
9900 gimple_omp_set_body (teams_stmt
, NULL
);
9901 gimple_seq_add_seq (&bind_body
, olist
);
9902 gimple_seq_add_seq (&bind_body
, dlist
);
9903 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
9904 gimple_bind_set_body (bind
, bind_body
);
9906 pop_gimplify_context (bind
);
9908 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9909 BLOCK_VARS (block
) = ctx
->block_vars
;
9910 if (BLOCK_VARS (block
))
9911 TREE_USED (block
) = 1;
9915 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9916 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9917 of OpenMP context, but with task_shared_vars set. */
9920 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
9925 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9926 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
9929 if (task_shared_vars
9931 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
9934 /* If a global variable has been privatized, TREE_CONSTANT on
9935 ADDR_EXPR might be wrong. */
9936 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
9937 recompute_tree_invariant_for_addr_expr (t
);
9939 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
9944 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9946 gimple stmt
= gsi_stmt (*gsi_p
);
9947 struct walk_stmt_info wi
;
9949 if (gimple_has_location (stmt
))
9950 input_location
= gimple_location (stmt
);
9952 if (task_shared_vars
)
9953 memset (&wi
, '\0', sizeof (wi
));
9955 /* If we have issued syntax errors, avoid doing any heavy lifting.
9956 Just replace the OpenMP directives with a NOP to avoid
9957 confusing RTL expansion. */
9958 if (seen_error () && is_gimple_omp (stmt
))
9960 gsi_replace (gsi_p
, gimple_build_nop (), true);
9964 switch (gimple_code (stmt
))
9967 if ((ctx
|| task_shared_vars
)
9968 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
9969 ctx
? NULL
: &wi
, NULL
)
9970 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
9971 ctx
? NULL
: &wi
, NULL
)))
9972 gimple_regimplify_operands (stmt
, gsi_p
);
9975 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
9977 case GIMPLE_EH_FILTER
:
9978 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
9981 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
9982 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
9984 case GIMPLE_TRANSACTION
:
9985 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
9988 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
9990 case GIMPLE_OMP_PARALLEL
:
9991 case GIMPLE_OMP_TASK
:
9992 ctx
= maybe_lookup_ctx (stmt
);
9994 if (ctx
->cancellable
)
9995 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9996 lower_omp_taskreg (gsi_p
, ctx
);
9998 case GIMPLE_OMP_FOR
:
9999 ctx
= maybe_lookup_ctx (stmt
);
10001 if (ctx
->cancellable
)
10002 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10003 lower_omp_for (gsi_p
, ctx
);
10005 case GIMPLE_OMP_SECTIONS
:
10006 ctx
= maybe_lookup_ctx (stmt
);
10008 if (ctx
->cancellable
)
10009 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10010 lower_omp_sections (gsi_p
, ctx
);
10012 case GIMPLE_OMP_SINGLE
:
10013 ctx
= maybe_lookup_ctx (stmt
);
10015 lower_omp_single (gsi_p
, ctx
);
10017 case GIMPLE_OMP_MASTER
:
10018 ctx
= maybe_lookup_ctx (stmt
);
10020 lower_omp_master (gsi_p
, ctx
);
10022 case GIMPLE_OMP_TASKGROUP
:
10023 ctx
= maybe_lookup_ctx (stmt
);
10025 lower_omp_taskgroup (gsi_p
, ctx
);
10027 case GIMPLE_OMP_ORDERED
:
10028 ctx
= maybe_lookup_ctx (stmt
);
10030 lower_omp_ordered (gsi_p
, ctx
);
10032 case GIMPLE_OMP_CRITICAL
:
10033 ctx
= maybe_lookup_ctx (stmt
);
10035 lower_omp_critical (gsi_p
, ctx
);
10037 case GIMPLE_OMP_ATOMIC_LOAD
:
10038 if ((ctx
|| task_shared_vars
)
10039 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10040 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10041 gimple_regimplify_operands (stmt
, gsi_p
);
10043 case GIMPLE_OMP_TARGET
:
10044 ctx
= maybe_lookup_ctx (stmt
);
10046 lower_omp_target (gsi_p
, ctx
);
10048 case GIMPLE_OMP_TEAMS
:
10049 ctx
= maybe_lookup_ctx (stmt
);
10051 lower_omp_teams (gsi_p
, ctx
);
10055 fndecl
= gimple_call_fndecl (stmt
);
10057 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10058 switch (DECL_FUNCTION_CODE (fndecl
))
10060 case BUILT_IN_GOMP_BARRIER
:
10064 case BUILT_IN_GOMP_CANCEL
:
10065 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10068 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10069 cctx
= cctx
->outer
;
10070 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10071 if (!cctx
->cancellable
)
10073 if (DECL_FUNCTION_CODE (fndecl
)
10074 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10076 stmt
= gimple_build_nop ();
10077 gsi_replace (gsi_p
, stmt
, false);
10082 lhs
= create_tmp_var (boolean_type_node
, NULL
);
10083 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10085 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10086 gimple_call_set_fndecl (stmt
, fndecl
);
10087 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10089 gimple_call_set_lhs (stmt
, lhs
);
10090 tree fallthru_label
;
10091 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10093 g
= gimple_build_label (fallthru_label
);
10094 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10095 g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
10096 cctx
->cancel_label
, fallthru_label
);
10097 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10104 if ((ctx
|| task_shared_vars
)
10105 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10107 gimple_regimplify_operands (stmt
, gsi_p
);
10113 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10115 location_t saved_location
= input_location
;
10116 gimple_stmt_iterator gsi
;
10117 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10118 lower_omp_1 (&gsi
, ctx
);
10119 /* Inside target region we haven't called fold_stmt during gimplification,
10120 because it can break code by adding decl references that weren't in the
10121 source. Call fold_stmt now. */
10122 if (target_nesting_level
)
10123 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10125 input_location
= saved_location
;
10128 /* Main entry point. */
10130 static unsigned int
10131 execute_lower_omp (void)
10135 /* This pass always runs, to provide PROP_gimple_lomp.
10136 But there is nothing to do unless -fopenmp is given. */
10137 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_enable_cilkplus
== 0)
10140 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10141 delete_omp_context
);
10143 body
= gimple_body (current_function_decl
);
10144 scan_omp (&body
, NULL
);
10145 gcc_assert (taskreg_nesting_level
== 0);
10147 if (all_contexts
->root
)
10149 if (task_shared_vars
)
10150 push_gimplify_context ();
10151 lower_omp (&body
, NULL
);
10152 if (task_shared_vars
)
10153 pop_gimplify_context (NULL
);
10158 splay_tree_delete (all_contexts
);
10159 all_contexts
= NULL
;
10161 BITMAP_FREE (task_shared_vars
);
10167 const pass_data pass_data_lower_omp
=
10169 GIMPLE_PASS
, /* type */
10170 "omplower", /* name */
10171 OPTGROUP_NONE
, /* optinfo_flags */
10172 false, /* has_gate */
10173 true, /* has_execute */
10174 TV_NONE
, /* tv_id */
10175 PROP_gimple_any
, /* properties_required */
10176 PROP_gimple_lomp
, /* properties_provided */
10177 0, /* properties_destroyed */
10178 0, /* todo_flags_start */
10179 0, /* todo_flags_finish */
10182 class pass_lower_omp
: public gimple_opt_pass
10185 pass_lower_omp (gcc::context
*ctxt
)
10186 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10189 /* opt_pass methods: */
10190 unsigned int execute () { return execute_lower_omp (); }
10192 }; // class pass_lower_omp
10194 } // anon namespace
10197 make_pass_lower_omp (gcc::context
*ctxt
)
10199 return new pass_lower_omp (ctxt
);
10202 /* The following is a utility to diagnose OpenMP structured block violations.
10203 It is not part of the "omplower" pass, as that's invoked too late. It
10204 should be invoked by the respective front ends after gimplification. */
10206 static splay_tree all_labels
;
10208 /* Check for mismatched contexts and generate an error if needed. Return
10209 true if an error is detected. */
10212 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10213 gimple branch_ctx
, gimple label_ctx
)
10215 if (label_ctx
== branch_ctx
)
10220 Previously we kept track of the label's entire context in diagnose_sb_[12]
10221 so we could traverse it and issue a correct "exit" or "enter" error
10222 message upon a structured block violation.
10224 We built the context by building a list with tree_cons'ing, but there is
10225 no easy counterpart in gimple tuples. It seems like far too much work
10226 for issuing exit/enter error messages. If someone really misses the
10227 distinct error message... patches welcome.
10231 /* Try to avoid confusing the user by producing and error message
10232 with correct "exit" or "enter" verbiage. We prefer "exit"
10233 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10234 if (branch_ctx
== NULL
)
10240 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10245 label_ctx
= TREE_CHAIN (label_ctx
);
10250 error ("invalid exit from OpenMP structured block");
10252 error ("invalid entry to OpenMP structured block");
10255 bool cilkplus_block
= false;
10256 if (flag_enable_cilkplus
)
10259 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10260 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10261 || (gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10262 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10263 cilkplus_block
= true;
10266 /* If it's obvious we have an invalid entry, be specific about the error. */
10267 if (branch_ctx
== NULL
)
10269 if (cilkplus_block
)
10270 error ("invalid entry to Cilk Plus structured block");
10272 error ("invalid entry to OpenMP structured block");
10276 /* Otherwise, be vague and lazy, but efficient. */
10277 if (cilkplus_block
)
10278 error ("invalid branch to/from a Cilk Plus structured block");
10280 error ("invalid branch to/from an OpenMP structured block");
10283 gsi_replace (gsi_p
, gimple_build_nop (), false);
10287 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10288 where each label is found. */
10291 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10292 struct walk_stmt_info
*wi
)
10294 gimple context
= (gimple
) wi
->info
;
10295 gimple inner_context
;
10296 gimple stmt
= gsi_stmt (*gsi_p
);
10298 *handled_ops_p
= true;
10300 switch (gimple_code (stmt
))
10304 case GIMPLE_OMP_PARALLEL
:
10305 case GIMPLE_OMP_TASK
:
10306 case GIMPLE_OMP_SECTIONS
:
10307 case GIMPLE_OMP_SINGLE
:
10308 case GIMPLE_OMP_SECTION
:
10309 case GIMPLE_OMP_MASTER
:
10310 case GIMPLE_OMP_ORDERED
:
10311 case GIMPLE_OMP_CRITICAL
:
10312 case GIMPLE_OMP_TARGET
:
10313 case GIMPLE_OMP_TEAMS
:
10314 case GIMPLE_OMP_TASKGROUP
:
10315 /* The minimal context here is just the current OMP construct. */
10316 inner_context
= stmt
;
10317 wi
->info
= inner_context
;
10318 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10319 wi
->info
= context
;
10322 case GIMPLE_OMP_FOR
:
10323 inner_context
= stmt
;
10324 wi
->info
= inner_context
;
10325 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10327 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10328 diagnose_sb_1
, NULL
, wi
);
10329 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10330 wi
->info
= context
;
10334 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10335 (splay_tree_value
) context
);
10345 /* Pass 2: Check each branch and see if its context differs from that of
10346 the destination label's context. */
10349 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10350 struct walk_stmt_info
*wi
)
10352 gimple context
= (gimple
) wi
->info
;
10354 gimple stmt
= gsi_stmt (*gsi_p
);
10356 *handled_ops_p
= true;
10358 switch (gimple_code (stmt
))
10362 case GIMPLE_OMP_PARALLEL
:
10363 case GIMPLE_OMP_TASK
:
10364 case GIMPLE_OMP_SECTIONS
:
10365 case GIMPLE_OMP_SINGLE
:
10366 case GIMPLE_OMP_SECTION
:
10367 case GIMPLE_OMP_MASTER
:
10368 case GIMPLE_OMP_ORDERED
:
10369 case GIMPLE_OMP_CRITICAL
:
10370 case GIMPLE_OMP_TARGET
:
10371 case GIMPLE_OMP_TEAMS
:
10372 case GIMPLE_OMP_TASKGROUP
:
10374 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10375 wi
->info
= context
;
10378 case GIMPLE_OMP_FOR
:
10380 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10382 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10383 diagnose_sb_2
, NULL
, wi
);
10384 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10385 wi
->info
= context
;
10390 tree lab
= gimple_cond_true_label (stmt
);
10393 n
= splay_tree_lookup (all_labels
,
10394 (splay_tree_key
) lab
);
10395 diagnose_sb_0 (gsi_p
, context
,
10396 n
? (gimple
) n
->value
: NULL
);
10398 lab
= gimple_cond_false_label (stmt
);
10401 n
= splay_tree_lookup (all_labels
,
10402 (splay_tree_key
) lab
);
10403 diagnose_sb_0 (gsi_p
, context
,
10404 n
? (gimple
) n
->value
: NULL
);
10411 tree lab
= gimple_goto_dest (stmt
);
10412 if (TREE_CODE (lab
) != LABEL_DECL
)
10415 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10416 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10420 case GIMPLE_SWITCH
:
10423 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10425 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10426 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10427 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10433 case GIMPLE_RETURN
:
10434 diagnose_sb_0 (gsi_p
, context
, NULL
);
10444 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10447 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
)
10449 gimple last
= last_stmt (bb
);
10450 enum gimple_code code
= gimple_code (last
);
10451 struct omp_region
*cur_region
= *region
;
10452 bool fallthru
= false;
10456 case GIMPLE_OMP_PARALLEL
:
10457 case GIMPLE_OMP_TASK
:
10458 case GIMPLE_OMP_FOR
:
10459 case GIMPLE_OMP_SINGLE
:
10460 case GIMPLE_OMP_TEAMS
:
10461 case GIMPLE_OMP_MASTER
:
10462 case GIMPLE_OMP_TASKGROUP
:
10463 case GIMPLE_OMP_ORDERED
:
10464 case GIMPLE_OMP_CRITICAL
:
10465 case GIMPLE_OMP_SECTION
:
10466 cur_region
= new_omp_region (bb
, code
, cur_region
);
10470 case GIMPLE_OMP_TARGET
:
10471 cur_region
= new_omp_region (bb
, code
, cur_region
);
10473 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10474 cur_region
= cur_region
->outer
;
10477 case GIMPLE_OMP_SECTIONS
:
10478 cur_region
= new_omp_region (bb
, code
, cur_region
);
10482 case GIMPLE_OMP_SECTIONS_SWITCH
:
10486 case GIMPLE_OMP_ATOMIC_LOAD
:
10487 case GIMPLE_OMP_ATOMIC_STORE
:
10491 case GIMPLE_OMP_RETURN
:
10492 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10493 somewhere other than the next block. This will be
10495 cur_region
->exit
= bb
;
10496 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10497 cur_region
= cur_region
->outer
;
10500 case GIMPLE_OMP_CONTINUE
:
10501 cur_region
->cont
= bb
;
10502 switch (cur_region
->type
)
10504 case GIMPLE_OMP_FOR
:
10505 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10506 succs edges as abnormal to prevent splitting
10508 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10509 /* Make the loopback edge. */
10510 make_edge (bb
, single_succ (cur_region
->entry
),
10513 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10514 corresponds to the case that the body of the loop
10515 is not executed at all. */
10516 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10517 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10521 case GIMPLE_OMP_SECTIONS
:
10522 /* Wire up the edges into and out of the nested sections. */
10524 basic_block switch_bb
= single_succ (cur_region
->entry
);
10526 struct omp_region
*i
;
10527 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10529 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10530 make_edge (switch_bb
, i
->entry
, 0);
10531 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10534 /* Make the loopback edge to the block with
10535 GIMPLE_OMP_SECTIONS_SWITCH. */
10536 make_edge (bb
, switch_bb
, 0);
10538 /* Make the edge from the switch to exit. */
10539 make_edge (switch_bb
, bb
->next_bb
, 0);
10545 gcc_unreachable ();
10550 gcc_unreachable ();
10553 if (*region
!= cur_region
)
10554 *region
= cur_region
;
10559 static unsigned int
10560 diagnose_omp_structured_block_errors (void)
10562 struct walk_stmt_info wi
;
10563 gimple_seq body
= gimple_body (current_function_decl
);
10565 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10567 memset (&wi
, 0, sizeof (wi
));
10568 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10570 memset (&wi
, 0, sizeof (wi
));
10571 wi
.want_locations
= true;
10572 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10574 gimple_set_body (current_function_decl
, body
);
10576 splay_tree_delete (all_labels
);
10583 gate_diagnose_omp_blocks (void)
10585 return flag_openmp
|| flag_enable_cilkplus
;
10590 const pass_data pass_data_diagnose_omp_blocks
=
10592 GIMPLE_PASS
, /* type */
10593 "*diagnose_omp_blocks", /* name */
10594 OPTGROUP_NONE
, /* optinfo_flags */
10595 true, /* has_gate */
10596 true, /* has_execute */
10597 TV_NONE
, /* tv_id */
10598 PROP_gimple_any
, /* properties_required */
10599 0, /* properties_provided */
10600 0, /* properties_destroyed */
10601 0, /* todo_flags_start */
10602 0, /* todo_flags_finish */
10605 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10608 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10609 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10612 /* opt_pass methods: */
10613 bool gate () { return gate_diagnose_omp_blocks (); }
10614 unsigned int execute () {
10615 return diagnose_omp_structured_block_errors ();
10618 }; // class pass_diagnose_omp_blocks
10620 } // anon namespace
10623 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10625 return new pass_diagnose_omp_blocks (ctxt
);
10628 /* SIMD clone supporting code. */
10630 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10631 of arguments to reserve space for. */
10633 static struct cgraph_simd_clone
*
10634 simd_clone_struct_alloc (int nargs
)
10636 struct cgraph_simd_clone
*clone_info
;
10637 size_t len
= (sizeof (struct cgraph_simd_clone
)
10638 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
10639 clone_info
= (struct cgraph_simd_clone
*)
10640 ggc_internal_cleared_alloc_stat (len PASS_MEM_STAT
);
10644 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10647 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
10648 struct cgraph_simd_clone
*from
)
10650 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
10651 + from
->nargs
* sizeof (struct cgraph_simd_clone_arg
)));
10654 /* Return vector of parameter types of function FNDECL. This uses
10655 TYPE_ARG_TYPES if available, otherwise falls back to types of
10656 DECL_ARGUMENTS types. */
10659 simd_clone_vector_of_formal_parm_types (tree fndecl
)
10661 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
10662 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
10663 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
10666 FOR_EACH_VEC_ELT (args
, i
, arg
)
10667 args
[i
] = TREE_TYPE (args
[i
]);
10671 /* Given a simd function in NODE, extract the simd specific
10672 information from the OMP clauses passed in CLAUSES, and return
10673 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10674 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10675 otherwise set to FALSE. */
10677 static struct cgraph_simd_clone
*
10678 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
10679 bool *inbranch_specified
)
10681 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
10684 *inbranch_specified
= false;
10686 n
= args
.length ();
10687 if (n
> 0 && args
.last () == void_type_node
)
10690 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10691 be cloned have a distinctive artificial label in addition to "omp
10694 = (flag_enable_cilkplus
10695 && lookup_attribute ("cilk plus elemental",
10696 DECL_ATTRIBUTES (node
->decl
)));
10698 /* Allocate one more than needed just in case this is an in-branch
10699 clone which will require a mask argument. */
10700 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
10701 clone_info
->nargs
= n
;
10702 clone_info
->cilk_elemental
= cilk_clone
;
10709 clauses
= TREE_VALUE (clauses
);
10710 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
10713 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
10715 switch (OMP_CLAUSE_CODE (t
))
10717 case OMP_CLAUSE_INBRANCH
:
10718 clone_info
->inbranch
= 1;
10719 *inbranch_specified
= true;
10721 case OMP_CLAUSE_NOTINBRANCH
:
10722 clone_info
->inbranch
= 0;
10723 *inbranch_specified
= true;
10725 case OMP_CLAUSE_SIMDLEN
:
10726 clone_info
->simdlen
10727 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
10729 case OMP_CLAUSE_LINEAR
:
10731 tree decl
= OMP_CLAUSE_DECL (t
);
10732 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
10733 int argno
= TREE_INT_CST_LOW (decl
);
10734 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
10736 clone_info
->args
[argno
].arg_type
10737 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
10738 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10739 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
10740 && clone_info
->args
[argno
].linear_step
< n
);
10744 if (POINTER_TYPE_P (args
[argno
]))
10745 step
= fold_convert (ssizetype
, step
);
10746 if (!tree_fits_shwi_p (step
))
10748 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10749 "ignoring large linear step");
10753 else if (integer_zerop (step
))
10755 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10756 "ignoring zero linear step");
10762 clone_info
->args
[argno
].arg_type
10763 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
10764 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10769 case OMP_CLAUSE_UNIFORM
:
10771 tree decl
= OMP_CLAUSE_DECL (t
);
10772 int argno
= tree_to_uhwi (decl
);
10773 clone_info
->args
[argno
].arg_type
10774 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
10777 case OMP_CLAUSE_ALIGNED
:
10779 tree decl
= OMP_CLAUSE_DECL (t
);
10780 int argno
= tree_to_uhwi (decl
);
10781 clone_info
->args
[argno
].alignment
10782 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
10793 /* Given a SIMD clone in NODE, calculate the characteristic data
10794 type and return the coresponding type. The characteristic data
10795 type is computed as described in the Intel Vector ABI. */
10798 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
10799 struct cgraph_simd_clone
*clone_info
)
10801 tree type
= integer_type_node
;
10802 tree fndecl
= node
->decl
;
10804 /* a) For non-void function, the characteristic data type is the
10806 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
10807 type
= TREE_TYPE (TREE_TYPE (fndecl
));
10809 /* b) If the function has any non-uniform, non-linear parameters,
10810 then the characteristic data type is the type of the first
10814 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
10815 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
10816 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
10824 /* c) If the characteristic data type determined by a) or b) above
10825 is struct, union, or class type which is pass-by-value (except
10826 for the type that maps to the built-in complex data type), the
10827 characteristic data type is int. */
10828 if (RECORD_OR_UNION_TYPE_P (type
)
10829 && !aggregate_value_p (type
, NULL
)
10830 && TREE_CODE (type
) != COMPLEX_TYPE
)
10831 return integer_type_node
;
10833 /* d) If none of the above three classes is applicable, the
10834 characteristic data type is int. */
10838 /* e) For Intel Xeon Phi native and offload compilation, if the
10839 resulting characteristic data type is 8-bit or 16-bit integer
10840 data type, the characteristic data type is int. */
10841 /* Well, we don't handle Xeon Phi yet. */
10845 simd_clone_mangle (struct cgraph_node
*node
,
10846 struct cgraph_simd_clone
*clone_info
)
10848 char vecsize_mangle
= clone_info
->vecsize_mangle
;
10849 char mask
= clone_info
->inbranch
? 'M' : 'N';
10850 unsigned int simdlen
= clone_info
->simdlen
;
10854 gcc_assert (vecsize_mangle
&& simdlen
);
10856 pp_string (&pp
, "_ZGV");
10857 pp_character (&pp
, vecsize_mangle
);
10858 pp_character (&pp
, mask
);
10859 pp_decimal_int (&pp
, simdlen
);
10861 for (n
= 0; n
< clone_info
->nargs
; ++n
)
10863 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
10865 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
10866 pp_character (&pp
, 'u');
10867 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
10869 gcc_assert (arg
.linear_step
!= 0);
10870 pp_character (&pp
, 'l');
10871 if (arg
.linear_step
> 1)
10872 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
10873 else if (arg
.linear_step
< 0)
10875 pp_character (&pp
, 'n');
10876 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
10880 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
10882 pp_character (&pp
, 's');
10883 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
10886 pp_character (&pp
, 'v');
10889 pp_character (&pp
, 'a');
10890 pp_decimal_int (&pp
, arg
.alignment
);
10894 pp_underscore (&pp
);
10896 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
10897 const char *str
= pp_formatted_text (&pp
);
10899 /* If there already is a SIMD clone with the same mangled name, don't
10900 add another one. This can happen e.g. for
10901 #pragma omp declare simd
10902 #pragma omp declare simd simdlen(8)
10903 int foo (int, int);
10904 if the simdlen is assumed to be 8 for the first one, etc. */
10905 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
10906 clone
= clone
->simdclone
->next_clone
)
10907 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
10911 return get_identifier (str
);
10914 /* Create a simd clone of OLD_NODE and return it. */
10916 static struct cgraph_node
*
10917 simd_clone_create (struct cgraph_node
*old_node
)
10919 struct cgraph_node
*new_node
;
10920 if (old_node
->definition
)
10922 if (!cgraph_function_with_gimple_body_p (old_node
))
10924 cgraph_get_body (old_node
);
10925 new_node
= cgraph_function_versioning (old_node
, vNULL
, NULL
, NULL
,
10926 false, NULL
, NULL
, "simdclone");
10930 tree old_decl
= old_node
->decl
;
10931 tree new_decl
= copy_node (old_node
->decl
);
10932 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
10933 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
10934 SET_DECL_RTL (new_decl
, NULL
);
10935 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
10936 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
10938 = cgraph_copy_node_for_versioning (old_node
, new_decl
, vNULL
, NULL
);
10939 cgraph_call_function_insertion_hooks (new_node
);
10941 if (new_node
== NULL
)
10944 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
10946 /* The function cgraph_function_versioning () will force the new
10947 symbol local. Undo this, and inherit external visability from
10949 new_node
->local
.local
= old_node
->local
.local
;
10950 new_node
->externally_visible
= old_node
->externally_visible
;
10955 /* Adjust the return type of the given function to its appropriate
10956 vector counterpart. Returns a simd array to be used throughout the
10957 function as a return value. */
10960 simd_clone_adjust_return_type (struct cgraph_node
*node
)
10962 tree fndecl
= node
->decl
;
10963 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
10964 unsigned int veclen
;
10967 /* Adjust the function return type. */
10968 if (orig_rettype
== void_type_node
)
10970 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
10971 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
10972 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
10973 veclen
= node
->simdclone
->vecsize_int
;
10975 veclen
= node
->simdclone
->vecsize_float
;
10976 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
10977 if (veclen
> node
->simdclone
->simdlen
)
10978 veclen
= node
->simdclone
->simdlen
;
10979 if (veclen
== node
->simdclone
->simdlen
)
10980 TREE_TYPE (TREE_TYPE (fndecl
))
10981 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
10982 node
->simdclone
->simdlen
);
10985 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
10986 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
10987 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
10989 if (!node
->definition
)
10992 t
= DECL_RESULT (fndecl
);
10993 /* Adjust the DECL_RESULT. */
10994 gcc_assert (TREE_TYPE (t
) != void_type_node
);
10995 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
10998 tree atype
= build_array_type_nelts (orig_rettype
,
10999 node
->simdclone
->simdlen
);
11000 if (veclen
!= node
->simdclone
->simdlen
)
11001 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11003 /* Set up a SIMD array to use as the return value. */
11004 tree retval
= create_tmp_var_raw (atype
, "retval");
11005 gimple_add_tmp_var (retval
);
11009 /* Each vector argument has a corresponding array to be used locally
11010 as part of the eventual loop. Create such temporary array and
11013 PREFIX is the prefix to be used for the temporary.
11015 TYPE is the inner element type.
11017 SIMDLEN is the number of elements. */
11020 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11022 tree atype
= build_array_type_nelts (type
, simdlen
);
11023 tree avar
= create_tmp_var_raw (atype
, prefix
);
11024 gimple_add_tmp_var (avar
);
11028 /* Modify the function argument types to their corresponding vector
11029 counterparts if appropriate. Also, create one array for each simd
11030 argument to be used locally when using the function arguments as
11033 NODE is the function whose arguments are to be adjusted.
11035 Returns an adjustment vector that will be filled describing how the
11036 argument types will be adjusted. */
11038 static ipa_parm_adjustment_vec
11039 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11042 ipa_parm_adjustment_vec adjustments
;
11044 if (node
->definition
)
11045 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11047 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11048 adjustments
.create (args
.length ());
11049 unsigned i
, j
, veclen
;
11050 struct ipa_parm_adjustment adj
;
11051 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11053 memset (&adj
, 0, sizeof (adj
));
11054 tree parm
= args
[i
];
11055 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11056 adj
.base_index
= i
;
11059 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11060 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11062 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11064 /* No adjustment necessary for scalar arguments. */
11065 adj
.op
= IPA_PARM_OP_COPY
;
11069 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11070 veclen
= node
->simdclone
->vecsize_int
;
11072 veclen
= node
->simdclone
->vecsize_float
;
11073 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11074 if (veclen
> node
->simdclone
->simdlen
)
11075 veclen
= node
->simdclone
->simdlen
;
11076 adj
.arg_prefix
= "simd";
11077 adj
.type
= build_vector_type (parm_type
, veclen
);
11078 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11079 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11081 adjustments
.safe_push (adj
);
11084 memset (&adj
, 0, sizeof (adj
));
11085 adj
.op
= IPA_PARM_OP_NEW
;
11086 adj
.arg_prefix
= "simd";
11087 adj
.base_index
= i
;
11088 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11092 if (node
->definition
)
11093 node
->simdclone
->args
[i
].simd_array
11094 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11095 parm_type
, node
->simdclone
->simdlen
);
11097 adjustments
.safe_push (adj
);
11100 if (node
->simdclone
->inbranch
)
11103 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11106 memset (&adj
, 0, sizeof (adj
));
11107 adj
.op
= IPA_PARM_OP_NEW
;
11108 adj
.arg_prefix
= "mask";
11110 adj
.base_index
= i
;
11111 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11112 veclen
= node
->simdclone
->vecsize_int
;
11114 veclen
= node
->simdclone
->vecsize_float
;
11115 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11116 if (veclen
> node
->simdclone
->simdlen
)
11117 veclen
= node
->simdclone
->simdlen
;
11118 adj
.type
= build_vector_type (base_type
, veclen
);
11119 adjustments
.safe_push (adj
);
11121 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11122 adjustments
.safe_push (adj
);
11124 /* We have previously allocated one extra entry for the mask. Use
11126 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11128 if (node
->definition
)
11130 sc
->args
[i
].orig_arg
11131 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11132 sc
->args
[i
].simd_array
11133 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11135 sc
->args
[i
].orig_type
= base_type
;
11136 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11139 if (node
->definition
)
11140 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11143 tree new_arg_types
= NULL_TREE
, new_reversed
;
11144 bool last_parm_void
= false;
11145 if (args
.length () > 0 && args
.last () == void_type_node
)
11146 last_parm_void
= true;
11148 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11149 j
= adjustments
.length ();
11150 for (i
= 0; i
< j
; i
++)
11152 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11154 if (adj
->op
== IPA_PARM_OP_COPY
)
11155 ptype
= args
[adj
->base_index
];
11158 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11160 new_reversed
= nreverse (new_arg_types
);
11161 if (last_parm_void
)
11164 TREE_CHAIN (new_arg_types
) = void_list_node
;
11166 new_reversed
= void_list_node
;
11169 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11170 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11171 TREE_TYPE (node
->decl
) = new_type
;
11173 adjustments
.release ();
11176 return adjustments
;
11179 /* Initialize and copy the function arguments in NODE to their
11180 corresponding local simd arrays. Returns a fresh gimple_seq with
11181 the instruction sequence generated. */
11184 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11185 ipa_parm_adjustment_vec adjustments
)
11187 gimple_seq seq
= NULL
;
11188 unsigned i
= 0, j
= 0, k
;
11190 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11192 arg
= DECL_CHAIN (arg
), i
++, j
++)
11194 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11197 node
->simdclone
->args
[i
].vector_arg
= arg
;
11199 tree array
= node
->simdclone
->args
[i
].simd_array
;
11200 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11202 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11203 tree ptr
= build_fold_addr_expr (array
);
11204 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11205 build_int_cst (ptype
, 0));
11206 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11207 gimplify_and_add (t
, &seq
);
11211 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11212 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11213 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11215 tree ptr
= build_fold_addr_expr (array
);
11219 arg
= DECL_CHAIN (arg
);
11223 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11224 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11225 build_int_cst (ptype
, k
* elemsize
));
11226 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11227 gimplify_and_add (t
, &seq
);
11234 /* Callback info for ipa_simd_modify_stmt_ops below. */
11236 struct modify_stmt_info
{
11237 ipa_parm_adjustment_vec adjustments
;
11239 /* True if the parent statement was modified by
11240 ipa_simd_modify_stmt_ops. */
11244 /* Callback for walk_gimple_op.
11246 Adjust operands from a given statement as specified in the
11247 adjustments vector in the callback data. */
11250 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11252 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11253 if (!SSA_VAR_P (*tp
))
11255 /* Make sure we treat subtrees as a RHS. This makes sure that
11256 when examining the `*foo' in *foo=x, the `foo' get treated as
11258 wi
->is_lhs
= false;
11259 wi
->val_only
= true;
11261 *walk_subtrees
= 0;
11264 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11265 struct ipa_parm_adjustment
*cand
11266 = ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11271 tree repl
= make_ssa_name (TREE_TYPE (t
), NULL
);
11274 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11277 stmt
= gimple_build_assign (unshare_expr (cand
->new_decl
), repl
);
11278 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
11279 SSA_NAME_DEF_STMT (repl
) = info
->stmt
;
11283 /* You'd think we could skip the extra SSA variable when
11284 wi->val_only=true, but we may have `*var' which will get
11285 replaced into `*var_array[iter]' and will likely be something
11287 stmt
= gimple_build_assign (repl
, unshare_expr (cand
->new_decl
));
11288 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11291 if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11293 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11299 info
->modified
= true;
11300 wi
->is_lhs
= false;
11301 wi
->val_only
= true;
11305 /* Traverse the function body and perform all modifications as
11306 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11307 modified such that the replacement/reduction value will now be an
11308 offset into the corresponding simd_array.
11310 This function will replace all function argument uses with their
11311 corresponding simd array elements, and ajust the return values
11315 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11316 ipa_parm_adjustment_vec adjustments
,
11317 tree retval_array
, tree iter
)
11322 /* Re-use the adjustments array, but this time use it to replace
11323 every function argument use to an offset into the corresponding
11325 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11327 if (!node
->simdclone
->args
[i
].vector_arg
)
11330 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11331 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11332 adjustments
[j
].new_decl
11333 = build4 (ARRAY_REF
,
11335 node
->simdclone
->args
[i
].simd_array
,
11337 NULL_TREE
, NULL_TREE
);
11338 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11339 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11340 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11343 struct modify_stmt_info info
;
11344 info
.adjustments
= adjustments
;
11346 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11348 gimple_stmt_iterator gsi
;
11350 gsi
= gsi_start_bb (bb
);
11351 while (!gsi_end_p (gsi
))
11353 gimple stmt
= gsi_stmt (gsi
);
11355 struct walk_stmt_info wi
;
11357 memset (&wi
, 0, sizeof (wi
));
11358 info
.modified
= false;
11360 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11362 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11364 tree retval
= gimple_return_retval (stmt
);
11367 gsi_remove (&gsi
, true);
11371 /* Replace `return foo' with `retval_array[iter] = foo'. */
11372 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11373 retval_array
, iter
, NULL
, NULL
);
11374 stmt
= gimple_build_assign (ref
, retval
);
11375 gsi_replace (&gsi
, stmt
, true);
11376 info
.modified
= true;
11381 update_stmt (stmt
);
11382 if (maybe_clean_eh_stmt (stmt
))
11383 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11390 /* Adjust the argument types in NODE to their appropriate vector
11394 simd_clone_adjust (struct cgraph_node
*node
)
11396 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11398 targetm
.simd_clone
.adjust (node
);
11400 tree retval
= simd_clone_adjust_return_type (node
);
11401 ipa_parm_adjustment_vec adjustments
11402 = simd_clone_adjust_argument_types (node
);
11404 push_gimplify_context ();
11406 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11408 /* Adjust all uses of vector arguments accordingly. Adjust all
11409 return values accordingly. */
11410 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11411 tree iter1
= make_ssa_name (iter
, NULL
);
11412 tree iter2
= make_ssa_name (iter
, NULL
);
11413 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11415 /* Initialize the iteration variable. */
11416 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11417 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11418 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11419 /* Insert the SIMD array and iv initialization at function
11421 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11423 pop_gimplify_context (NULL
);
11425 /* Create a new BB right before the original exit BB, to hold the
11426 iteration increment and the condition/branch. */
11427 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11428 basic_block incr_bb
= create_empty_bb (orig_exit
);
11429 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11430 flag. Set it now to be a FALLTHRU_EDGE. */
11431 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11432 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11433 for (unsigned i
= 0;
11434 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
11436 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
11437 redirect_edge_succ (e
, incr_bb
);
11439 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
11440 e
->probability
= REG_BR_PROB_BASE
;
11441 gsi
= gsi_last_bb (incr_bb
);
11442 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
11443 build_int_cst (unsigned_type_node
,
11445 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11447 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11448 struct loop
*loop
= alloc_loop ();
11449 cfun
->has_force_vect_loops
= true;
11450 loop
->safelen
= node
->simdclone
->simdlen
;
11451 loop
->force_vect
= true;
11452 loop
->header
= body_bb
;
11453 add_bb_to_loop (incr_bb
, loop
);
11455 /* Branch around the body if the mask applies. */
11456 if (node
->simdclone
->inbranch
)
11458 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
11460 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
11461 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
11462 tree aref
= build4 (ARRAY_REF
,
11463 TREE_TYPE (TREE_TYPE (mask_array
)),
11466 g
= gimple_build_assign (mask
, aref
);
11467 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11468 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
11469 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
11471 aref
= build1 (VIEW_CONVERT_EXPR
,
11472 build_nonstandard_integer_type (bitsize
, 0), mask
);
11473 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
11474 g
= gimple_build_assign (mask
, aref
);
11475 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11478 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
11480 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11481 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
11482 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
11485 /* Generate the condition. */
11486 g
= gimple_build_cond (LT_EXPR
,
11488 build_int_cst (unsigned_type_node
,
11489 node
->simdclone
->simdlen
),
11491 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11492 e
= split_block (incr_bb
, gsi_stmt (gsi
));
11493 basic_block latch_bb
= e
->dest
;
11494 basic_block new_exit_bb
= e
->dest
;
11495 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
11496 loop
->latch
= latch_bb
;
11498 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
11500 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
11501 /* The successor of incr_bb is already pointing to latch_bb; just
11503 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11504 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
11506 gimple phi
= create_phi_node (iter1
, body_bb
);
11507 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
11508 edge latch_edge
= single_succ_edge (latch_bb
);
11509 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
11511 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11513 /* Generate the new return. */
11514 gsi
= gsi_last_bb (new_exit_bb
);
11516 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
11517 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
11518 retval
= TREE_OPERAND (retval
, 0);
11521 retval
= build1 (VIEW_CONVERT_EXPR
,
11522 TREE_TYPE (TREE_TYPE (node
->decl
)),
11524 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
11525 false, GSI_CONTINUE_LINKING
);
11527 g
= gimple_build_return (retval
);
11528 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11530 /* Handle aligned clauses by replacing default defs of the aligned
11531 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11532 lhs. Handle linear by adding PHIs. */
11533 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
11534 if (node
->simdclone
->args
[i
].alignment
11535 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
11536 && (node
->simdclone
->args
[i
].alignment
11537 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
11538 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
11541 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
11542 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11543 tree def
= ssa_default_def (cfun
, orig_arg
);
11544 if (!has_zero_uses (def
))
11546 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
11547 gimple_seq seq
= NULL
;
11548 bool need_cvt
= false;
11550 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
11552 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
11555 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
11556 gimple_call_set_lhs (g
, t
);
11557 gimple_seq_add_stmt_without_update (&seq
, g
);
11560 t
= make_ssa_name (orig_arg
, NULL
);
11561 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
11562 gimple_call_lhs (g
),
11564 gimple_seq_add_stmt_without_update (&seq
, g
);
11566 gsi_insert_seq_on_edge_immediate
11567 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
11569 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11570 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
11572 cgraph_create_edge (node
, cgraph_get_create_node (fn
),
11573 call
, entry_bb
->count
, freq
);
11575 imm_use_iterator iter
;
11576 use_operand_p use_p
;
11578 tree repl
= gimple_get_lhs (g
);
11579 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11580 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
11583 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11584 SET_USE (use_p
, repl
);
11587 else if (node
->simdclone
->args
[i
].arg_type
11588 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11590 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11591 tree def
= ssa_default_def (cfun
, orig_arg
);
11592 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11593 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
11594 if (!has_zero_uses (def
))
11596 iter1
= make_ssa_name (orig_arg
, NULL
);
11597 iter2
= make_ssa_name (orig_arg
, NULL
);
11598 phi
= create_phi_node (iter1
, body_bb
);
11599 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
11600 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11601 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11602 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
11603 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11604 ? TREE_TYPE (orig_arg
) : sizetype
;
11606 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
11607 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
11608 gsi
= gsi_last_bb (incr_bb
);
11609 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11611 imm_use_iterator iter
;
11612 use_operand_p use_p
;
11614 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11615 if (use_stmt
== phi
)
11618 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11619 SET_USE (use_p
, iter1
);
11623 calculate_dominance_info (CDI_DOMINATORS
);
11624 add_loop (loop
, loop
->header
->loop_father
);
11625 update_ssa (TODO_update_ssa
);
11630 /* If the function in NODE is tagged as an elemental SIMD function,
11631 create the appropriate SIMD clones. */
11634 expand_simd_clones (struct cgraph_node
*node
)
11636 tree attr
= lookup_attribute ("omp declare simd",
11637 DECL_ATTRIBUTES (node
->decl
));
11638 if (attr
== NULL_TREE
11639 || node
->global
.inlined_to
11640 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
11644 #pragma omp declare simd
11646 in C, there we don't know the argument types at all. */
11647 if (!node
->definition
11648 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
11653 /* Start with parsing the "omp declare simd" attribute(s). */
11654 bool inbranch_clause_specified
;
11655 struct cgraph_simd_clone
*clone_info
11656 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
11657 &inbranch_clause_specified
);
11658 if (clone_info
== NULL
)
11661 int orig_simdlen
= clone_info
->simdlen
;
11662 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
11663 /* The target can return 0 (no simd clones should be created),
11664 1 (just one ISA of simd clones should be created) or higher
11665 count of ISA variants. In that case, clone_info is initialized
11666 for the first ISA variant. */
11668 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
11673 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11674 also create one inbranch and one !inbranch clone of it. */
11675 for (int i
= 0; i
< count
* 2; i
++)
11677 struct cgraph_simd_clone
*clone
= clone_info
;
11678 if (inbranch_clause_specified
&& (i
& 1) != 0)
11683 clone
= simd_clone_struct_alloc (clone_info
->nargs
11684 - clone_info
->inbranch
11686 simd_clone_struct_copy (clone
, clone_info
);
11687 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11688 and simd_clone_adjust_argument_types did to the first
11690 clone
->nargs
-= clone_info
->inbranch
;
11691 clone
->simdlen
= orig_simdlen
;
11692 /* And call the target hook again to get the right ISA. */
11693 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
11697 clone
->inbranch
= 1;
11700 /* simd_clone_mangle might fail if such a clone has been created
11702 tree id
= simd_clone_mangle (node
, clone
);
11703 if (id
== NULL_TREE
)
11706 /* Only when we are sure we want to create the clone actually
11707 clone the function (or definitions) or create another
11708 extern FUNCTION_DECL (for prototypes without definitions). */
11709 struct cgraph_node
*n
= simd_clone_create (node
);
11713 n
->simdclone
= clone
;
11714 clone
->origin
= node
;
11715 clone
->next_clone
= NULL
;
11716 if (node
->simd_clones
== NULL
)
11718 clone
->prev_clone
= n
;
11719 node
->simd_clones
= n
;
11723 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
11724 clone
->prev_clone
->simdclone
->next_clone
= n
;
11725 node
->simd_clones
->simdclone
->prev_clone
= n
;
11727 change_decl_assembler_name (n
->decl
, id
);
11728 /* And finally adjust the return type, parameters and for
11729 definitions also function body. */
11730 if (node
->definition
)
11731 simd_clone_adjust (n
);
11734 simd_clone_adjust_return_type (n
);
11735 simd_clone_adjust_argument_types (n
);
11739 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
11742 /* Entry point for IPA simd clone creation pass. */
11744 static unsigned int
11745 ipa_omp_simd_clone (void)
11747 struct cgraph_node
*node
;
11748 FOR_EACH_FUNCTION (node
)
11749 expand_simd_clones (node
);
11755 const pass_data pass_data_omp_simd_clone
=
11757 SIMPLE_IPA_PASS
, /* type */
11758 "simdclone", /* name */
11759 OPTGROUP_NONE
, /* optinfo_flags */
11760 true, /* has_gate */
11761 true, /* has_execute */
11762 TV_NONE
, /* tv_id */
11763 ( PROP_ssa
| PROP_cfg
), /* properties_required */
11764 0, /* properties_provided */
11765 0, /* properties_destroyed */
11766 0, /* todo_flags_start */
11767 0, /* todo_flags_finish */
11770 class pass_omp_simd_clone
: public simple_ipa_opt_pass
11773 pass_omp_simd_clone(gcc::context
*ctxt
)
11774 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
11777 /* opt_pass methods: */
11778 bool gate () { return ((flag_openmp
|| flag_openmp_simd
11779 || flag_enable_cilkplus
|| (in_lto_p
&& !flag_wpa
))
11780 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
11782 unsigned int execute () { return ipa_omp_simd_clone (); }
11785 } // anon namespace
11787 simple_ipa_opt_pass
*
11788 make_pass_omp_simd_clone (gcc::context
*ctxt
)
11790 return new pass_omp_simd_clone (ctxt
);
11793 #include "gt-omp-low.h"