1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
207 static vec
<omp_context
*> taskreg_contexts
;
209 static void scan_omp (gimple_seq
*, omp_context
*);
210 static tree
scan_omp_1_op (tree
*, int *, void *);
212 #define WALK_SUBSTMTS \
216 case GIMPLE_EH_FILTER: \
217 case GIMPLE_TRANSACTION: \
218 /* The sub-statements for these should be walked. */ \
219 *handled_ops_p = false; \
222 /* Convenience function for calling scan_omp_1_op on tree operands. */
225 scan_omp_op (tree
*tp
, omp_context
*ctx
)
227 struct walk_stmt_info wi
;
229 memset (&wi
, 0, sizeof (wi
));
231 wi
.want_locations
= true;
233 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
236 static void lower_omp (gimple_seq
*, omp_context
*);
237 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
238 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
240 /* Find an OpenMP clause of type KIND within CLAUSES. */
243 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
245 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
246 if (OMP_CLAUSE_CODE (clauses
) == kind
)
252 /* Return true if CTX is for an omp parallel. */
255 is_parallel_ctx (omp_context
*ctx
)
257 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
261 /* Return true if CTX is for an omp task. */
264 is_task_ctx (omp_context
*ctx
)
266 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
270 /* Return true if CTX is for an omp parallel or omp task. */
273 is_taskreg_ctx (omp_context
*ctx
)
275 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
276 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
280 /* Return true if REGION is a combined parallel+workshare region. */
283 is_combined_parallel (struct omp_region
*region
)
285 return region
->is_combined_parallel
;
289 /* Extract the header elements of parallel loop FOR_STMT and store
293 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
294 struct omp_for_data_loop
*loops
)
296 tree t
, var
, *collapse_iter
, *collapse_count
;
297 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
298 struct omp_for_data_loop
*loop
;
300 struct omp_for_data_loop dummy_loop
;
301 location_t loc
= gimple_location (for_stmt
);
302 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_KIND_SIMD
;
303 bool distribute
= gimple_omp_for_kind (for_stmt
)
304 == GF_OMP_FOR_KIND_DISTRIBUTE
;
306 fd
->for_stmt
= for_stmt
;
308 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
309 if (fd
->collapse
> 1)
312 fd
->loops
= &fd
->loop
;
314 fd
->have_nowait
= distribute
|| simd
;
315 fd
->have_ordered
= false;
316 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
317 fd
->chunk_size
= NULL_TREE
;
318 collapse_iter
= NULL
;
319 collapse_count
= NULL
;
321 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
322 switch (OMP_CLAUSE_CODE (t
))
324 case OMP_CLAUSE_NOWAIT
:
325 fd
->have_nowait
= true;
327 case OMP_CLAUSE_ORDERED
:
328 fd
->have_ordered
= true;
330 case OMP_CLAUSE_SCHEDULE
:
331 gcc_assert (!distribute
);
332 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
333 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
335 case OMP_CLAUSE_DIST_SCHEDULE
:
336 gcc_assert (distribute
);
337 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
339 case OMP_CLAUSE_COLLAPSE
:
340 if (fd
->collapse
> 1)
342 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
343 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
349 /* FIXME: for now map schedule(auto) to schedule(static).
350 There should be analysis to determine whether all iterations
351 are approximately the same amount of work (then schedule(static)
352 is best) or if it varies (then schedule(dynamic,N) is better). */
353 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
355 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
356 gcc_assert (fd
->chunk_size
== NULL
);
358 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
359 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
360 gcc_assert (fd
->chunk_size
== NULL
);
361 else if (fd
->chunk_size
== NULL
)
363 /* We only need to compute a default chunk size for ordered
364 static loops and dynamic loops. */
365 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
367 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
368 ? integer_zero_node
: integer_one_node
;
371 for (i
= 0; i
< fd
->collapse
; i
++)
373 if (fd
->collapse
== 1)
375 else if (loops
!= NULL
)
380 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
381 gcc_assert (SSA_VAR_P (loop
->v
));
382 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
383 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
384 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
385 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
387 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
388 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
389 switch (loop
->cond_code
)
395 gcc_assert (gimple_omp_for_kind (for_stmt
)
396 == GF_OMP_FOR_KIND_CILKSIMD
);
399 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
400 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
402 loop
->n2
= fold_build2_loc (loc
,
403 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
404 build_int_cst (TREE_TYPE (loop
->n2
), 1));
405 loop
->cond_code
= LT_EXPR
;
408 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
409 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
411 loop
->n2
= fold_build2_loc (loc
,
412 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
413 build_int_cst (TREE_TYPE (loop
->n2
), 1));
414 loop
->cond_code
= GT_EXPR
;
420 t
= gimple_omp_for_incr (for_stmt
, i
);
421 gcc_assert (TREE_OPERAND (t
, 0) == var
);
422 switch (TREE_CODE (t
))
425 loop
->step
= TREE_OPERAND (t
, 1);
427 case POINTER_PLUS_EXPR
:
428 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
431 loop
->step
= TREE_OPERAND (t
, 1);
432 loop
->step
= fold_build1_loc (loc
,
433 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
441 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
442 && !fd
->have_ordered
))
444 if (fd
->collapse
== 1)
445 iter_type
= TREE_TYPE (loop
->v
);
447 || TYPE_PRECISION (iter_type
)
448 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
450 = build_nonstandard_integer_type
451 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
453 else if (iter_type
!= long_long_unsigned_type_node
)
455 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
456 iter_type
= long_long_unsigned_type_node
;
457 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
458 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
459 >= TYPE_PRECISION (iter_type
))
463 if (loop
->cond_code
== LT_EXPR
)
464 n
= fold_build2_loc (loc
,
465 PLUS_EXPR
, TREE_TYPE (loop
->v
),
466 loop
->n2
, loop
->step
);
469 if (TREE_CODE (n
) != INTEGER_CST
470 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
471 iter_type
= long_long_unsigned_type_node
;
473 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
474 > TYPE_PRECISION (iter_type
))
478 if (loop
->cond_code
== LT_EXPR
)
481 n2
= fold_build2_loc (loc
,
482 PLUS_EXPR
, TREE_TYPE (loop
->v
),
483 loop
->n2
, loop
->step
);
487 n1
= fold_build2_loc (loc
,
488 MINUS_EXPR
, TREE_TYPE (loop
->v
),
489 loop
->n2
, loop
->step
);
492 if (TREE_CODE (n1
) != INTEGER_CST
493 || TREE_CODE (n2
) != INTEGER_CST
494 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
495 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
496 iter_type
= long_long_unsigned_type_node
;
500 if (collapse_count
&& *collapse_count
== NULL
)
502 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
503 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
504 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
505 if (t
&& integer_zerop (t
))
506 count
= build_zero_cst (long_long_unsigned_type_node
);
507 else if ((i
== 0 || count
!= NULL_TREE
)
508 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
509 && TREE_CONSTANT (loop
->n1
)
510 && TREE_CONSTANT (loop
->n2
)
511 && TREE_CODE (loop
->step
) == INTEGER_CST
)
513 tree itype
= TREE_TYPE (loop
->v
);
515 if (POINTER_TYPE_P (itype
))
516 itype
= signed_type_for (itype
);
517 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
518 t
= fold_build2_loc (loc
,
520 fold_convert_loc (loc
, itype
, loop
->step
), t
);
521 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
522 fold_convert_loc (loc
, itype
, loop
->n2
));
523 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
524 fold_convert_loc (loc
, itype
, loop
->n1
));
525 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
526 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
527 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
528 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
529 fold_convert_loc (loc
, itype
,
532 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
533 fold_convert_loc (loc
, itype
, loop
->step
));
534 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
535 if (count
!= NULL_TREE
)
536 count
= fold_build2_loc (loc
,
537 MULT_EXPR
, long_long_unsigned_type_node
,
541 if (TREE_CODE (count
) != INTEGER_CST
)
544 else if (count
&& !integer_zerop (count
))
551 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
552 || fd
->have_ordered
))
554 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
555 iter_type
= long_long_unsigned_type_node
;
557 iter_type
= long_integer_type_node
;
559 else if (collapse_iter
&& *collapse_iter
!= NULL
)
560 iter_type
= TREE_TYPE (*collapse_iter
);
561 fd
->iter_type
= iter_type
;
562 if (collapse_iter
&& *collapse_iter
== NULL
)
563 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
564 if (collapse_count
&& *collapse_count
== NULL
)
567 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
569 *collapse_count
= create_tmp_var (iter_type
, ".count");
572 if (fd
->collapse
> 1)
574 fd
->loop
.v
= *collapse_iter
;
575 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
576 fd
->loop
.n2
= *collapse_count
;
577 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
578 fd
->loop
.cond_code
= LT_EXPR
;
583 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
584 is the immediate dominator of PAR_ENTRY_BB, return true if there
585 are no data dependencies that would prevent expanding the parallel
586 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
588 When expanding a combined parallel+workshare region, the call to
589 the child function may need additional arguments in the case of
590 GIMPLE_OMP_FOR regions. In some cases, these arguments are
591 computed out of variables passed in from the parent to the child
592 via 'struct .omp_data_s'. For instance:
594 #pragma omp parallel for schedule (guided, i * 4)
599 # BLOCK 2 (PAR_ENTRY_BB)
601 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
603 # BLOCK 3 (WS_ENTRY_BB)
604 .omp_data_i = &.omp_data_o;
605 D.1667 = .omp_data_i->i;
607 #pragma omp for schedule (guided, D.1598)
609 When we outline the parallel region, the call to the child function
610 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
611 that value is computed *after* the call site. So, in principle we
612 cannot do the transformation.
614 To see whether the code in WS_ENTRY_BB blocks the combined
615 parallel+workshare call, we collect all the variables used in the
616 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
617 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
620 FIXME. If we had the SSA form built at this point, we could merely
621 hoist the code in block 3 into block 2 and be done with it. But at
622 this point we don't have dataflow information and though we could
623 hack something up here, it is really not worth the aggravation. */
626 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
628 struct omp_for_data fd
;
629 gimple ws_stmt
= last_stmt (ws_entry_bb
);
631 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
634 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
636 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
638 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
640 if (fd
.iter_type
!= long_integer_type_node
)
643 /* FIXME. We give up too easily here. If any of these arguments
644 are not constants, they will likely involve variables that have
645 been mapped into fields of .omp_data_s for sharing with the child
646 function. With appropriate data flow, it would be possible to
648 if (!is_gimple_min_invariant (fd
.loop
.n1
)
649 || !is_gimple_min_invariant (fd
.loop
.n2
)
650 || !is_gimple_min_invariant (fd
.loop
.step
)
651 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
658 /* Collect additional arguments needed to emit a combined
659 parallel+workshare call. WS_STMT is the workshare directive being
662 static vec
<tree
, va_gc
> *
663 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
666 location_t loc
= gimple_location (ws_stmt
);
667 vec
<tree
, va_gc
> *ws_args
;
669 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
671 struct omp_for_data fd
;
674 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
678 if (gimple_omp_for_combined_into_p (ws_stmt
))
681 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
682 OMP_CLAUSE__LOOPTEMP_
);
684 n1
= OMP_CLAUSE_DECL (innerc
);
685 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
686 OMP_CLAUSE__LOOPTEMP_
);
688 n2
= OMP_CLAUSE_DECL (innerc
);
691 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
693 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
694 ws_args
->quick_push (t
);
696 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
697 ws_args
->quick_push (t
);
699 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
700 ws_args
->quick_push (t
);
704 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
705 ws_args
->quick_push (t
);
710 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
712 /* Number of sections is equal to the number of edges from the
713 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
714 the exit of the sections region. */
715 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
716 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
717 vec_alloc (ws_args
, 1);
718 ws_args
->quick_push (t
);
726 /* Discover whether REGION is a combined parallel+workshare region. */
729 determine_parallel_type (struct omp_region
*region
)
731 basic_block par_entry_bb
, par_exit_bb
;
732 basic_block ws_entry_bb
, ws_exit_bb
;
734 if (region
== NULL
|| region
->inner
== NULL
735 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
736 || region
->inner
->cont
== NULL
)
739 /* We only support parallel+for and parallel+sections. */
740 if (region
->type
!= GIMPLE_OMP_PARALLEL
741 || (region
->inner
->type
!= GIMPLE_OMP_FOR
742 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
745 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
746 WS_EXIT_BB -> PAR_EXIT_BB. */
747 par_entry_bb
= region
->entry
;
748 par_exit_bb
= region
->exit
;
749 ws_entry_bb
= region
->inner
->entry
;
750 ws_exit_bb
= region
->inner
->exit
;
752 if (single_succ (par_entry_bb
) == ws_entry_bb
753 && single_succ (ws_exit_bb
) == par_exit_bb
754 && workshare_safe_to_combine_p (ws_entry_bb
)
755 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
756 || (last_and_only_stmt (ws_entry_bb
)
757 && last_and_only_stmt (par_exit_bb
))))
759 gimple par_stmt
= last_stmt (par_entry_bb
);
760 gimple ws_stmt
= last_stmt (ws_entry_bb
);
762 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
764 /* If this is a combined parallel loop, we need to determine
765 whether or not to use the combined library calls. There
766 are two cases where we do not apply the transformation:
767 static loops and any kind of ordered loop. In the first
768 case, we already open code the loop so there is no need
769 to do anything else. In the latter case, the combined
770 parallel loop call would still need extra synchronization
771 to implement ordered semantics, so there would not be any
772 gain in using the combined call. */
773 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
774 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
776 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
777 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
779 region
->is_combined_parallel
= false;
780 region
->inner
->is_combined_parallel
= false;
785 region
->is_combined_parallel
= true;
786 region
->inner
->is_combined_parallel
= true;
787 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
792 /* Return true if EXPR is variable sized. */
795 is_variable_sized (const_tree expr
)
797 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
800 /* Return true if DECL is a reference type. */
803 is_reference (tree decl
)
805 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
808 /* Lookup variables in the decl or field splay trees. The "maybe" form
809 allows for the variable form to not have been entered, otherwise we
810 assert that the variable must have been entered. */
813 lookup_decl (tree var
, omp_context
*ctx
)
816 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
821 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
824 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
825 return n
? *n
: NULL_TREE
;
829 lookup_field (tree var
, omp_context
*ctx
)
832 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
833 return (tree
) n
->value
;
837 lookup_sfield (tree var
, omp_context
*ctx
)
840 n
= splay_tree_lookup (ctx
->sfield_map
841 ? ctx
->sfield_map
: ctx
->field_map
,
842 (splay_tree_key
) var
);
843 return (tree
) n
->value
;
847 maybe_lookup_field (tree var
, omp_context
*ctx
)
850 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
851 return n
? (tree
) n
->value
: NULL_TREE
;
854 /* Return true if DECL should be copied by pointer. SHARED_CTX is
855 the parallel context if DECL is to be shared. */
858 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
860 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
863 /* We can only use copy-in/copy-out semantics for shared variables
864 when we know the value is not accessible from an outer scope. */
867 /* ??? Trivially accessible from anywhere. But why would we even
868 be passing an address in this case? Should we simply assert
869 this to be false, or should we have a cleanup pass that removes
870 these from the list of mappings? */
871 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
874 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
875 without analyzing the expression whether or not its location
876 is accessible to anyone else. In the case of nested parallel
877 regions it certainly may be. */
878 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
881 /* Do not use copy-in/copy-out for variables that have their
883 if (TREE_ADDRESSABLE (decl
))
886 /* lower_send_shared_vars only uses copy-in, but not copy-out
888 if (TREE_READONLY (decl
)
889 || ((TREE_CODE (decl
) == RESULT_DECL
890 || TREE_CODE (decl
) == PARM_DECL
)
891 && DECL_BY_REFERENCE (decl
)))
894 /* Disallow copy-in/out in nested parallel if
895 decl is shared in outer parallel, otherwise
896 each thread could store the shared variable
897 in its own copy-in location, making the
898 variable no longer really shared. */
899 if (shared_ctx
->is_nested
)
903 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
904 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
911 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
912 c
; c
= OMP_CLAUSE_CHAIN (c
))
913 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
914 && OMP_CLAUSE_DECL (c
) == decl
)
918 goto maybe_mark_addressable_and_ret
;
922 /* For tasks avoid using copy-in/out. As tasks can be
923 deferred or executed in different thread, when GOMP_task
924 returns, the task hasn't necessarily terminated. */
925 if (is_task_ctx (shared_ctx
))
928 maybe_mark_addressable_and_ret
:
929 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
930 if (is_gimple_reg (outer
))
932 /* Taking address of OUTER in lower_send_shared_vars
933 might need regimplification of everything that uses the
935 if (!task_shared_vars
)
936 task_shared_vars
= BITMAP_ALLOC (NULL
);
937 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
938 TREE_ADDRESSABLE (outer
) = 1;
947 /* Construct a new automatic decl similar to VAR. */
950 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
952 tree copy
= copy_var_decl (var
, name
, type
);
954 DECL_CONTEXT (copy
) = current_function_decl
;
955 DECL_CHAIN (copy
) = ctx
->block_vars
;
956 ctx
->block_vars
= copy
;
962 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
964 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
967 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
970 omp_build_component_ref (tree obj
, tree field
)
972 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
973 if (TREE_THIS_VOLATILE (field
))
974 TREE_THIS_VOLATILE (ret
) |= 1;
975 if (TREE_READONLY (field
))
976 TREE_READONLY (ret
) |= 1;
980 /* Build tree nodes to access the field for VAR on the receiver side. */
983 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
985 tree x
, field
= lookup_field (var
, ctx
);
987 /* If the receiver record type was remapped in the child function,
988 remap the field into the new record type. */
989 x
= maybe_lookup_field (field
, ctx
);
993 x
= build_simple_mem_ref (ctx
->receiver_decl
);
994 x
= omp_build_component_ref (x
, field
);
996 x
= build_simple_mem_ref (x
);
1001 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1002 of a parallel, this is a component reference; for workshare constructs
1003 this is some variable. */
1006 build_outer_var_ref (tree var
, omp_context
*ctx
)
1010 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1012 else if (is_variable_sized (var
))
1014 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1015 x
= build_outer_var_ref (x
, ctx
);
1016 x
= build_simple_mem_ref (x
);
1018 else if (is_taskreg_ctx (ctx
))
1020 bool by_ref
= use_pointer_for_field (var
, NULL
);
1021 x
= build_receiver_ref (var
, by_ref
, ctx
);
1023 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1024 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
1026 /* #pragma omp simd isn't a worksharing construct, and can reference even
1027 private vars in its linear etc. clauses. */
1029 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1030 x
= lookup_decl (var
, ctx
->outer
);
1031 else if (ctx
->outer
)
1032 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1036 else if (ctx
->outer
)
1037 x
= lookup_decl (var
, ctx
->outer
);
1038 else if (is_reference (var
))
1039 /* This can happen with orphaned constructs. If var is reference, it is
1040 possible it is shared and as such valid. */
1045 if (is_reference (var
))
1046 x
= build_simple_mem_ref (x
);
1051 /* Build tree nodes to access the field for VAR on the sender side. */
1054 build_sender_ref (tree var
, omp_context
*ctx
)
1056 tree field
= lookup_sfield (var
, ctx
);
1057 return omp_build_component_ref (ctx
->sender_decl
, field
);
1060 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1063 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1065 tree field
, type
, sfield
= NULL_TREE
;
1067 gcc_assert ((mask
& 1) == 0
1068 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1069 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1070 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1072 type
= TREE_TYPE (var
);
1075 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1076 type
= build_pointer_type (build_pointer_type (type
));
1079 type
= build_pointer_type (type
);
1080 else if ((mask
& 3) == 1 && is_reference (var
))
1081 type
= TREE_TYPE (type
);
1083 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1084 FIELD_DECL
, DECL_NAME (var
), type
);
1086 /* Remember what variable this field was created for. This does have a
1087 side effect of making dwarf2out ignore this member, so for helpful
1088 debugging we clear it later in delete_omp_context. */
1089 DECL_ABSTRACT_ORIGIN (field
) = var
;
1090 if (type
== TREE_TYPE (var
))
1092 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1093 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1094 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1097 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1099 if ((mask
& 3) == 3)
1101 insert_field_into_struct (ctx
->record_type
, field
);
1102 if (ctx
->srecord_type
)
1104 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1105 FIELD_DECL
, DECL_NAME (var
), type
);
1106 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1107 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1108 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1109 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1110 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1115 if (ctx
->srecord_type
== NULL_TREE
)
1119 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1120 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1121 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1123 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1124 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1125 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1126 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1127 splay_tree_insert (ctx
->sfield_map
,
1128 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1129 (splay_tree_value
) sfield
);
1133 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1134 : ctx
->srecord_type
, field
);
1138 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1139 (splay_tree_value
) field
);
1140 if ((mask
& 2) && ctx
->sfield_map
)
1141 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1142 (splay_tree_value
) sfield
);
1146 install_var_local (tree var
, omp_context
*ctx
)
1148 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1149 insert_decl_map (&ctx
->cb
, var
, new_var
);
1153 /* Adjust the replacement for DECL in CTX for the new context. This means
1154 copying the DECL_VALUE_EXPR, and fixing up the type. */
1157 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1159 tree new_decl
, size
;
1161 new_decl
= lookup_decl (decl
, ctx
);
1163 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1165 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1166 && DECL_HAS_VALUE_EXPR_P (decl
))
1168 tree ve
= DECL_VALUE_EXPR (decl
);
1169 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1170 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1171 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1174 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1176 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1177 if (size
== error_mark_node
)
1178 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1179 DECL_SIZE (new_decl
) = size
;
1181 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1182 if (size
== error_mark_node
)
1183 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1184 DECL_SIZE_UNIT (new_decl
) = size
;
1188 /* The callback for remap_decl. Search all containing contexts for a
1189 mapping of the variable; this avoids having to duplicate the splay
1190 tree ahead of time. We know a mapping doesn't already exist in the
1191 given context. Create new mappings to implement default semantics. */
1194 omp_copy_decl (tree var
, copy_body_data
*cb
)
1196 omp_context
*ctx
= (omp_context
*) cb
;
1199 if (TREE_CODE (var
) == LABEL_DECL
)
1201 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1202 DECL_CONTEXT (new_var
) = current_function_decl
;
1203 insert_decl_map (&ctx
->cb
, var
, new_var
);
1207 while (!is_taskreg_ctx (ctx
))
1212 new_var
= maybe_lookup_decl (var
, ctx
);
1217 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1220 return error_mark_node
;
1224 /* Debugging dumps for parallel regions. */
1225 void dump_omp_region (FILE *, struct omp_region
*, int);
1226 void debug_omp_region (struct omp_region
*);
1227 void debug_all_omp_regions (void);
1229 /* Dump the parallel region tree rooted at REGION. */
1232 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1234 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1235 gimple_code_name
[region
->type
]);
1238 dump_omp_region (file
, region
->inner
, indent
+ 4);
1242 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1243 region
->cont
->index
);
1247 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1248 region
->exit
->index
);
1250 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1253 dump_omp_region (file
, region
->next
, indent
);
1257 debug_omp_region (struct omp_region
*region
)
1259 dump_omp_region (stderr
, region
, 0);
1263 debug_all_omp_regions (void)
1265 dump_omp_region (stderr
, root_omp_region
, 0);
1269 /* Create a new parallel region starting at STMT inside region PARENT. */
1271 static struct omp_region
*
1272 new_omp_region (basic_block bb
, enum gimple_code type
,
1273 struct omp_region
*parent
)
1275 struct omp_region
*region
= XCNEW (struct omp_region
);
1277 region
->outer
= parent
;
1279 region
->type
= type
;
1283 /* This is a nested region. Add it to the list of inner
1284 regions in PARENT. */
1285 region
->next
= parent
->inner
;
1286 parent
->inner
= region
;
1290 /* This is a toplevel region. Add it to the list of toplevel
1291 regions in ROOT_OMP_REGION. */
1292 region
->next
= root_omp_region
;
1293 root_omp_region
= region
;
1299 /* Release the memory associated with the region tree rooted at REGION. */
1302 free_omp_region_1 (struct omp_region
*region
)
1304 struct omp_region
*i
, *n
;
1306 for (i
= region
->inner
; i
; i
= n
)
1309 free_omp_region_1 (i
);
1315 /* Release the memory for the entire omp region tree. */
1318 free_omp_regions (void)
1320 struct omp_region
*r
, *n
;
1321 for (r
= root_omp_region
; r
; r
= n
)
1324 free_omp_region_1 (r
);
1326 root_omp_region
= NULL
;
1330 /* Create a new context, with OUTER_CTX being the surrounding context. */
1332 static omp_context
*
1333 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1335 omp_context
*ctx
= XCNEW (omp_context
);
1337 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1338 (splay_tree_value
) ctx
);
1343 ctx
->outer
= outer_ctx
;
1344 ctx
->cb
= outer_ctx
->cb
;
1345 ctx
->cb
.block
= NULL
;
1346 ctx
->depth
= outer_ctx
->depth
+ 1;
1350 ctx
->cb
.src_fn
= current_function_decl
;
1351 ctx
->cb
.dst_fn
= current_function_decl
;
1352 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1353 gcc_checking_assert (ctx
->cb
.src_node
);
1354 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1355 ctx
->cb
.src_cfun
= cfun
;
1356 ctx
->cb
.copy_decl
= omp_copy_decl
;
1357 ctx
->cb
.eh_lp_nr
= 0;
1358 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1362 ctx
->cb
.decl_map
= pointer_map_create ();
1367 static gimple_seq
maybe_catch_exception (gimple_seq
);
1369 /* Finalize task copyfn. */
1372 finalize_task_copyfn (gimple task_stmt
)
1374 struct function
*child_cfun
;
1376 gimple_seq seq
= NULL
, new_seq
;
1379 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1380 if (child_fn
== NULL_TREE
)
1383 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1384 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1386 push_cfun (child_cfun
);
1387 bind
= gimplify_body (child_fn
, false);
1388 gimple_seq_add_stmt (&seq
, bind
);
1389 new_seq
= maybe_catch_exception (seq
);
1392 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1394 gimple_seq_add_stmt (&seq
, bind
);
1396 gimple_set_body (child_fn
, seq
);
1399 /* Inform the callgraph about the new function. */
1400 cgraph_add_new_function (child_fn
, false);
1403 /* Destroy a omp_context data structures. Called through the splay tree
1404 value delete callback. */
1407 delete_omp_context (splay_tree_value value
)
1409 omp_context
*ctx
= (omp_context
*) value
;
1411 pointer_map_destroy (ctx
->cb
.decl_map
);
1414 splay_tree_delete (ctx
->field_map
);
1415 if (ctx
->sfield_map
)
1416 splay_tree_delete (ctx
->sfield_map
);
1418 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1419 it produces corrupt debug information. */
1420 if (ctx
->record_type
)
1423 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1424 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1426 if (ctx
->srecord_type
)
1429 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1430 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1433 if (is_task_ctx (ctx
))
1434 finalize_task_copyfn (ctx
->stmt
);
1439 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1443 fixup_child_record_type (omp_context
*ctx
)
1445 tree f
, type
= ctx
->record_type
;
1447 /* ??? It isn't sufficient to just call remap_type here, because
1448 variably_modified_type_p doesn't work the way we expect for
1449 record types. Testing each field for whether it needs remapping
1450 and creating a new record by hand works, however. */
1451 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1452 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1456 tree name
, new_fields
= NULL
;
1458 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1459 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1460 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1461 TYPE_DECL
, name
, type
);
1462 TYPE_NAME (type
) = name
;
1464 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1466 tree new_f
= copy_node (f
);
1467 DECL_CONTEXT (new_f
) = type
;
1468 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1469 DECL_CHAIN (new_f
) = new_fields
;
1470 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1471 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1473 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1477 /* Arrange to be able to look up the receiver field
1478 given the sender field. */
1479 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1480 (splay_tree_value
) new_f
);
1482 TYPE_FIELDS (type
) = nreverse (new_fields
);
1486 TREE_TYPE (ctx
->receiver_decl
)
1487 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1490 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1491 specified by CLAUSES. */
1494 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1497 bool scan_array_reductions
= false;
1499 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1503 switch (OMP_CLAUSE_CODE (c
))
1505 case OMP_CLAUSE_PRIVATE
:
1506 decl
= OMP_CLAUSE_DECL (c
);
1507 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1509 else if (!is_variable_sized (decl
))
1510 install_var_local (decl
, ctx
);
1513 case OMP_CLAUSE_SHARED
:
1514 decl
= OMP_CLAUSE_DECL (c
);
1515 /* Ignore shared directives in teams construct. */
1516 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1518 /* Global variables don't need to be copied,
1519 the receiver side will use them directly. */
1520 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1521 if (is_global_var (odecl
))
1523 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1526 gcc_assert (is_taskreg_ctx (ctx
));
1527 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1528 || !is_variable_sized (decl
));
1529 /* Global variables don't need to be copied,
1530 the receiver side will use them directly. */
1531 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1533 by_ref
= use_pointer_for_field (decl
, ctx
);
1534 if (! TREE_READONLY (decl
)
1535 || TREE_ADDRESSABLE (decl
)
1537 || is_reference (decl
))
1539 install_var_field (decl
, by_ref
, 3, ctx
);
1540 install_var_local (decl
, ctx
);
1543 /* We don't need to copy const scalar vars back. */
1544 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1547 case OMP_CLAUSE_LASTPRIVATE
:
1548 /* Let the corresponding firstprivate clause create
1550 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1554 case OMP_CLAUSE_FIRSTPRIVATE
:
1555 case OMP_CLAUSE_REDUCTION
:
1556 case OMP_CLAUSE_LINEAR
:
1557 decl
= OMP_CLAUSE_DECL (c
);
1559 if (is_variable_sized (decl
))
1561 if (is_task_ctx (ctx
))
1562 install_var_field (decl
, false, 1, ctx
);
1565 else if (is_taskreg_ctx (ctx
))
1568 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1569 by_ref
= use_pointer_for_field (decl
, NULL
);
1571 if (is_task_ctx (ctx
)
1572 && (global
|| by_ref
|| is_reference (decl
)))
1574 install_var_field (decl
, false, 1, ctx
);
1576 install_var_field (decl
, by_ref
, 2, ctx
);
1579 install_var_field (decl
, by_ref
, 3, ctx
);
1581 install_var_local (decl
, ctx
);
1584 case OMP_CLAUSE__LOOPTEMP_
:
1585 gcc_assert (is_parallel_ctx (ctx
));
1586 decl
= OMP_CLAUSE_DECL (c
);
1587 install_var_field (decl
, false, 3, ctx
);
1588 install_var_local (decl
, ctx
);
1591 case OMP_CLAUSE_COPYPRIVATE
:
1592 case OMP_CLAUSE_COPYIN
:
1593 decl
= OMP_CLAUSE_DECL (c
);
1594 by_ref
= use_pointer_for_field (decl
, NULL
);
1595 install_var_field (decl
, by_ref
, 3, ctx
);
1598 case OMP_CLAUSE_DEFAULT
:
1599 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1602 case OMP_CLAUSE_FINAL
:
1604 case OMP_CLAUSE_NUM_THREADS
:
1605 case OMP_CLAUSE_NUM_TEAMS
:
1606 case OMP_CLAUSE_THREAD_LIMIT
:
1607 case OMP_CLAUSE_DEVICE
:
1608 case OMP_CLAUSE_SCHEDULE
:
1609 case OMP_CLAUSE_DIST_SCHEDULE
:
1610 case OMP_CLAUSE_DEPEND
:
1612 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1616 case OMP_CLAUSE_FROM
:
1617 case OMP_CLAUSE_MAP
:
1619 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1620 decl
= OMP_CLAUSE_DECL (c
);
1621 /* Global variables with "omp declare target" attribute
1622 don't need to be copied, the receiver side will use them
1624 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1626 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1627 && lookup_attribute ("omp declare target",
1628 DECL_ATTRIBUTES (decl
)))
1630 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1631 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1633 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1634 #pragma omp target data, there is nothing to map for
1636 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1637 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1638 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1643 if (DECL_SIZE (decl
)
1644 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1646 tree decl2
= DECL_VALUE_EXPR (decl
);
1647 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1648 decl2
= TREE_OPERAND (decl2
, 0);
1649 gcc_assert (DECL_P (decl2
));
1650 install_var_field (decl2
, true, 3, ctx
);
1651 install_var_local (decl2
, ctx
);
1652 install_var_local (decl
, ctx
);
1656 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1657 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1658 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1659 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1660 install_var_field (decl
, true, 7, ctx
);
1662 install_var_field (decl
, true, 3, ctx
);
1663 if (gimple_omp_target_kind (ctx
->stmt
)
1664 == GF_OMP_TARGET_KIND_REGION
)
1665 install_var_local (decl
, ctx
);
1670 tree base
= get_base_address (decl
);
1671 tree nc
= OMP_CLAUSE_CHAIN (c
);
1674 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1675 && OMP_CLAUSE_DECL (nc
) == base
1676 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1677 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1679 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1680 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1686 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1687 decl
= OMP_CLAUSE_DECL (c
);
1689 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1690 (splay_tree_key
) decl
));
1692 = build_decl (OMP_CLAUSE_LOCATION (c
),
1693 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1694 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1695 insert_field_into_struct (ctx
->record_type
, field
);
1696 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1697 (splay_tree_value
) field
);
1702 case OMP_CLAUSE_NOWAIT
:
1703 case OMP_CLAUSE_ORDERED
:
1704 case OMP_CLAUSE_COLLAPSE
:
1705 case OMP_CLAUSE_UNTIED
:
1706 case OMP_CLAUSE_MERGEABLE
:
1707 case OMP_CLAUSE_PROC_BIND
:
1708 case OMP_CLAUSE_SAFELEN
:
1711 case OMP_CLAUSE_ALIGNED
:
1712 decl
= OMP_CLAUSE_DECL (c
);
1713 if (is_global_var (decl
)
1714 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1715 install_var_local (decl
, ctx
);
1723 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1725 switch (OMP_CLAUSE_CODE (c
))
1727 case OMP_CLAUSE_LASTPRIVATE
:
1728 /* Let the corresponding firstprivate clause create
1730 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1731 scan_array_reductions
= true;
1732 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1736 case OMP_CLAUSE_PRIVATE
:
1737 case OMP_CLAUSE_FIRSTPRIVATE
:
1738 case OMP_CLAUSE_REDUCTION
:
1739 case OMP_CLAUSE_LINEAR
:
1740 decl
= OMP_CLAUSE_DECL (c
);
1741 if (is_variable_sized (decl
))
1742 install_var_local (decl
, ctx
);
1743 fixup_remapped_decl (decl
, ctx
,
1744 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1745 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1746 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1747 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1748 scan_array_reductions
= true;
1749 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1750 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1751 scan_array_reductions
= true;
1754 case OMP_CLAUSE_SHARED
:
1755 /* Ignore shared directives in teams construct. */
1756 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1758 decl
= OMP_CLAUSE_DECL (c
);
1759 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1760 fixup_remapped_decl (decl
, ctx
, false);
1763 case OMP_CLAUSE_MAP
:
1764 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1766 decl
= OMP_CLAUSE_DECL (c
);
1768 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1769 && lookup_attribute ("omp declare target",
1770 DECL_ATTRIBUTES (decl
)))
1774 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1775 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1776 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1778 tree new_decl
= lookup_decl (decl
, ctx
);
1779 TREE_TYPE (new_decl
)
1780 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1782 else if (DECL_SIZE (decl
)
1783 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1785 tree decl2
= DECL_VALUE_EXPR (decl
);
1786 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1787 decl2
= TREE_OPERAND (decl2
, 0);
1788 gcc_assert (DECL_P (decl2
));
1789 fixup_remapped_decl (decl2
, ctx
, false);
1790 fixup_remapped_decl (decl
, ctx
, true);
1793 fixup_remapped_decl (decl
, ctx
, false);
1797 case OMP_CLAUSE_COPYPRIVATE
:
1798 case OMP_CLAUSE_COPYIN
:
1799 case OMP_CLAUSE_DEFAULT
:
1801 case OMP_CLAUSE_NUM_THREADS
:
1802 case OMP_CLAUSE_NUM_TEAMS
:
1803 case OMP_CLAUSE_THREAD_LIMIT
:
1804 case OMP_CLAUSE_DEVICE
:
1805 case OMP_CLAUSE_SCHEDULE
:
1806 case OMP_CLAUSE_DIST_SCHEDULE
:
1807 case OMP_CLAUSE_NOWAIT
:
1808 case OMP_CLAUSE_ORDERED
:
1809 case OMP_CLAUSE_COLLAPSE
:
1810 case OMP_CLAUSE_UNTIED
:
1811 case OMP_CLAUSE_FINAL
:
1812 case OMP_CLAUSE_MERGEABLE
:
1813 case OMP_CLAUSE_PROC_BIND
:
1814 case OMP_CLAUSE_SAFELEN
:
1815 case OMP_CLAUSE_ALIGNED
:
1816 case OMP_CLAUSE_DEPEND
:
1817 case OMP_CLAUSE__LOOPTEMP_
:
1819 case OMP_CLAUSE_FROM
:
1827 if (scan_array_reductions
)
1828 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1829 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1830 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1832 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1833 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1835 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1836 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1837 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1838 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1839 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1840 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1843 /* Create a new name for omp child function. Returns an identifier. */
1846 create_omp_child_function_name (bool task_copy
)
1848 return (clone_function_name (current_function_decl
,
1849 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1852 /* Build a decl for the omp child function. It'll not contain a body
1853 yet, just the bare decl. */
1856 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1858 tree decl
, type
, name
, t
;
1860 name
= create_omp_child_function_name (task_copy
);
1862 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1863 ptr_type_node
, NULL_TREE
);
1865 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1867 decl
= build_decl (gimple_location (ctx
->stmt
),
1868 FUNCTION_DECL
, name
, type
);
1871 ctx
->cb
.dst_fn
= decl
;
1873 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1875 TREE_STATIC (decl
) = 1;
1876 TREE_USED (decl
) = 1;
1877 DECL_ARTIFICIAL (decl
) = 1;
1878 DECL_IGNORED_P (decl
) = 0;
1879 TREE_PUBLIC (decl
) = 0;
1880 DECL_UNINLINABLE (decl
) = 1;
1881 DECL_EXTERNAL (decl
) = 0;
1882 DECL_CONTEXT (decl
) = NULL_TREE
;
1883 DECL_INITIAL (decl
) = make_node (BLOCK
);
1884 bool target_p
= false;
1885 if (lookup_attribute ("omp declare target",
1886 DECL_ATTRIBUTES (current_function_decl
)))
1891 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1892 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1893 && gimple_omp_target_kind (octx
->stmt
)
1894 == GF_OMP_TARGET_KIND_REGION
)
1901 DECL_ATTRIBUTES (decl
)
1902 = tree_cons (get_identifier ("omp declare target"),
1903 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1905 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1906 RESULT_DECL
, NULL_TREE
, void_type_node
);
1907 DECL_ARTIFICIAL (t
) = 1;
1908 DECL_IGNORED_P (t
) = 1;
1909 DECL_CONTEXT (t
) = decl
;
1910 DECL_RESULT (decl
) = t
;
1912 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1913 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1914 DECL_ARTIFICIAL (t
) = 1;
1915 DECL_NAMELESS (t
) = 1;
1916 DECL_ARG_TYPE (t
) = ptr_type_node
;
1917 DECL_CONTEXT (t
) = current_function_decl
;
1919 DECL_ARGUMENTS (decl
) = t
;
1921 ctx
->receiver_decl
= t
;
1924 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1925 PARM_DECL
, get_identifier (".omp_data_o"),
1927 DECL_ARTIFICIAL (t
) = 1;
1928 DECL_NAMELESS (t
) = 1;
1929 DECL_ARG_TYPE (t
) = ptr_type_node
;
1930 DECL_CONTEXT (t
) = current_function_decl
;
1932 TREE_ADDRESSABLE (t
) = 1;
1933 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1934 DECL_ARGUMENTS (decl
) = t
;
1937 /* Allocate memory for the function structure. The call to
1938 allocate_struct_function clobbers CFUN, so we need to restore
1940 push_struct_function (decl
);
1941 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1945 /* Callback for walk_gimple_seq. Check if combined parallel
1946 contains gimple_omp_for_combined_into_p OMP_FOR. */
1949 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1950 bool *handled_ops_p
,
1951 struct walk_stmt_info
*wi
)
1953 gimple stmt
= gsi_stmt (*gsi_p
);
1955 *handled_ops_p
= true;
1956 switch (gimple_code (stmt
))
1960 case GIMPLE_OMP_FOR
:
1961 if (gimple_omp_for_combined_into_p (stmt
)
1962 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1965 return integer_zero_node
;
1974 /* Scan an OpenMP parallel directive. */
1977 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1981 gimple stmt
= gsi_stmt (*gsi
);
1983 /* Ignore parallel directives with empty bodies, unless there
1984 are copyin clauses. */
1986 && empty_body_p (gimple_omp_body (stmt
))
1987 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1988 OMP_CLAUSE_COPYIN
) == NULL
)
1990 gsi_replace (gsi
, gimple_build_nop (), false);
1994 if (gimple_omp_parallel_combined_p (stmt
))
1997 struct walk_stmt_info wi
;
1999 memset (&wi
, 0, sizeof (wi
));
2001 walk_gimple_seq (gimple_omp_body (stmt
),
2002 find_combined_for
, NULL
, &wi
);
2003 for_stmt
= (gimple
) wi
.info
;
2006 struct omp_for_data fd
;
2007 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2008 /* We need two temporaries with fd.loop.v type (istart/iend)
2009 and then (fd.collapse - 1) temporaries with the same
2010 type for count2 ... countN-1 vars if not constant. */
2011 size_t count
= 2, i
;
2012 tree type
= fd
.iter_type
;
2014 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2015 count
+= fd
.collapse
- 1;
2016 for (i
= 0; i
< count
; i
++)
2018 tree temp
= create_tmp_var (type
, NULL
);
2019 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2020 OMP_CLAUSE__LOOPTEMP_
);
2021 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2022 OMP_CLAUSE_DECL (c
) = temp
;
2023 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2024 gimple_omp_parallel_set_clauses (stmt
, c
);
2029 ctx
= new_omp_context (stmt
, outer_ctx
);
2030 taskreg_contexts
.safe_push (ctx
);
2031 if (taskreg_nesting_level
> 1)
2032 ctx
->is_nested
= true;
2033 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2034 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2035 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2036 name
= create_tmp_var_name (".omp_data_s");
2037 name
= build_decl (gimple_location (stmt
),
2038 TYPE_DECL
, name
, ctx
->record_type
);
2039 DECL_ARTIFICIAL (name
) = 1;
2040 DECL_NAMELESS (name
) = 1;
2041 TYPE_NAME (ctx
->record_type
) = name
;
2042 create_omp_child_function (ctx
, false);
2043 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2045 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2046 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2048 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2049 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2052 /* Scan an OpenMP task directive. */
2055 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2059 gimple stmt
= gsi_stmt (*gsi
);
2061 /* Ignore task directives with empty bodies. */
2063 && empty_body_p (gimple_omp_body (stmt
)))
2065 gsi_replace (gsi
, gimple_build_nop (), false);
2069 ctx
= new_omp_context (stmt
, outer_ctx
);
2070 taskreg_contexts
.safe_push (ctx
);
2071 if (taskreg_nesting_level
> 1)
2072 ctx
->is_nested
= true;
2073 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2074 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2075 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2076 name
= create_tmp_var_name (".omp_data_s");
2077 name
= build_decl (gimple_location (stmt
),
2078 TYPE_DECL
, name
, ctx
->record_type
);
2079 DECL_ARTIFICIAL (name
) = 1;
2080 DECL_NAMELESS (name
) = 1;
2081 TYPE_NAME (ctx
->record_type
) = name
;
2082 create_omp_child_function (ctx
, false);
2083 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2085 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2087 if (ctx
->srecord_type
)
2089 name
= create_tmp_var_name (".omp_data_a");
2090 name
= build_decl (gimple_location (stmt
),
2091 TYPE_DECL
, name
, ctx
->srecord_type
);
2092 DECL_ARTIFICIAL (name
) = 1;
2093 DECL_NAMELESS (name
) = 1;
2094 TYPE_NAME (ctx
->srecord_type
) = name
;
2095 create_omp_child_function (ctx
, true);
2098 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2100 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2102 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2103 t
= build_int_cst (long_integer_type_node
, 0);
2104 gimple_omp_task_set_arg_size (stmt
, t
);
2105 t
= build_int_cst (long_integer_type_node
, 1);
2106 gimple_omp_task_set_arg_align (stmt
, t
);
2111 /* If any decls have been made addressable during scan_omp,
2112 adjust their fields if needed, and layout record types
2113 of parallel/task constructs. */
2116 finish_taskreg_scan (omp_context
*ctx
)
2118 if (ctx
->record_type
== NULL_TREE
)
2121 /* If any task_shared_vars were needed, verify all
2122 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2123 statements if use_pointer_for_field hasn't changed
2124 because of that. If it did, update field types now. */
2125 if (task_shared_vars
)
2129 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2130 c
; c
= OMP_CLAUSE_CHAIN (c
))
2131 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2133 tree decl
= OMP_CLAUSE_DECL (c
);
2135 /* Global variables don't need to be copied,
2136 the receiver side will use them directly. */
2137 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2139 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2140 || !use_pointer_for_field (decl
, ctx
))
2142 tree field
= lookup_field (decl
, ctx
);
2143 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2144 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2146 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2147 TREE_THIS_VOLATILE (field
) = 0;
2148 DECL_USER_ALIGN (field
) = 0;
2149 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2150 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2151 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2152 if (ctx
->srecord_type
)
2154 tree sfield
= lookup_sfield (decl
, ctx
);
2155 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2156 TREE_THIS_VOLATILE (sfield
) = 0;
2157 DECL_USER_ALIGN (sfield
) = 0;
2158 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2159 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2160 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2165 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2167 layout_type (ctx
->record_type
);
2168 fixup_child_record_type (ctx
);
2172 location_t loc
= gimple_location (ctx
->stmt
);
2173 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2174 /* Move VLA fields to the end. */
2175 p
= &TYPE_FIELDS (ctx
->record_type
);
2177 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2178 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2181 *p
= TREE_CHAIN (*p
);
2182 TREE_CHAIN (*q
) = NULL_TREE
;
2183 q
= &TREE_CHAIN (*q
);
2186 p
= &DECL_CHAIN (*p
);
2188 layout_type (ctx
->record_type
);
2189 fixup_child_record_type (ctx
);
2190 if (ctx
->srecord_type
)
2191 layout_type (ctx
->srecord_type
);
2192 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2193 TYPE_SIZE_UNIT (ctx
->record_type
));
2194 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2195 t
= build_int_cst (long_integer_type_node
,
2196 TYPE_ALIGN_UNIT (ctx
->record_type
));
2197 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2202 /* Scan an OpenMP loop directive. */
2205 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2210 ctx
= new_omp_context (stmt
, outer_ctx
);
2212 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2214 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2215 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2217 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2218 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2219 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2220 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2222 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2225 /* Scan an OpenMP sections directive. */
2228 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2232 ctx
= new_omp_context (stmt
, outer_ctx
);
2233 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2234 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2237 /* Scan an OpenMP single directive. */
2240 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2245 ctx
= new_omp_context (stmt
, outer_ctx
);
2246 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2247 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2248 name
= create_tmp_var_name (".omp_copy_s");
2249 name
= build_decl (gimple_location (stmt
),
2250 TYPE_DECL
, name
, ctx
->record_type
);
2251 TYPE_NAME (ctx
->record_type
) = name
;
2253 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2254 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2256 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2257 ctx
->record_type
= NULL
;
2259 layout_type (ctx
->record_type
);
2262 /* Scan an OpenMP target{, data, update} directive. */
2265 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2269 int kind
= gimple_omp_target_kind (stmt
);
2271 ctx
= new_omp_context (stmt
, outer_ctx
);
2272 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2273 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2274 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2275 name
= create_tmp_var_name (".omp_data_t");
2276 name
= build_decl (gimple_location (stmt
),
2277 TYPE_DECL
, name
, ctx
->record_type
);
2278 DECL_ARTIFICIAL (name
) = 1;
2279 DECL_NAMELESS (name
) = 1;
2280 TYPE_NAME (ctx
->record_type
) = name
;
2281 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2283 create_omp_child_function (ctx
, false);
2284 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2287 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2288 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2290 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2291 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2294 TYPE_FIELDS (ctx
->record_type
)
2295 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2296 #ifdef ENABLE_CHECKING
2298 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2299 for (field
= TYPE_FIELDS (ctx
->record_type
);
2301 field
= DECL_CHAIN (field
))
2302 gcc_assert (DECL_ALIGN (field
) == align
);
2304 layout_type (ctx
->record_type
);
2305 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2306 fixup_child_record_type (ctx
);
2310 /* Scan an OpenMP teams directive. */
2313 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2315 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2316 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2317 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2320 /* Check OpenMP nesting restrictions. */
2322 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2326 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2327 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2329 error_at (gimple_location (stmt
),
2330 "OpenMP constructs may not be nested inside simd region");
2333 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2335 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2336 || (gimple_omp_for_kind (stmt
)
2337 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2338 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2340 error_at (gimple_location (stmt
),
2341 "only distribute or parallel constructs are allowed to "
2342 "be closely nested inside teams construct");
2347 switch (gimple_code (stmt
))
2349 case GIMPLE_OMP_FOR
:
2350 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_KIND_SIMD
)
2352 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2354 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2356 error_at (gimple_location (stmt
),
2357 "distribute construct must be closely nested inside "
2365 if (is_gimple_call (stmt
)
2366 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2367 == BUILT_IN_GOMP_CANCEL
2368 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2369 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2371 const char *bad
= NULL
;
2372 const char *kind
= NULL
;
2375 error_at (gimple_location (stmt
), "orphaned %qs construct",
2376 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2377 == BUILT_IN_GOMP_CANCEL
2378 ? "#pragma omp cancel"
2379 : "#pragma omp cancellation point");
2382 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2383 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2387 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2388 bad
= "#pragma omp parallel";
2389 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2390 == BUILT_IN_GOMP_CANCEL
2391 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2392 ctx
->cancellable
= true;
2396 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2397 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2398 bad
= "#pragma omp for";
2399 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2400 == BUILT_IN_GOMP_CANCEL
2401 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2403 ctx
->cancellable
= true;
2404 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2406 warning_at (gimple_location (stmt
), 0,
2407 "%<#pragma omp cancel for%> inside "
2408 "%<nowait%> for construct");
2409 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2410 OMP_CLAUSE_ORDERED
))
2411 warning_at (gimple_location (stmt
), 0,
2412 "%<#pragma omp cancel for%> inside "
2413 "%<ordered%> for construct");
2418 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2419 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2420 bad
= "#pragma omp sections";
2421 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2422 == BUILT_IN_GOMP_CANCEL
2423 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2425 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2427 ctx
->cancellable
= true;
2428 if (find_omp_clause (gimple_omp_sections_clauses
2431 warning_at (gimple_location (stmt
), 0,
2432 "%<#pragma omp cancel sections%> inside "
2433 "%<nowait%> sections construct");
2437 gcc_assert (ctx
->outer
2438 && gimple_code (ctx
->outer
->stmt
)
2439 == GIMPLE_OMP_SECTIONS
);
2440 ctx
->outer
->cancellable
= true;
2441 if (find_omp_clause (gimple_omp_sections_clauses
2444 warning_at (gimple_location (stmt
), 0,
2445 "%<#pragma omp cancel sections%> inside "
2446 "%<nowait%> sections construct");
2452 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2453 bad
= "#pragma omp task";
2455 ctx
->cancellable
= true;
2459 error_at (gimple_location (stmt
), "invalid arguments");
2464 error_at (gimple_location (stmt
),
2465 "%<%s %s%> construct not closely nested inside of %qs",
2466 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2467 == BUILT_IN_GOMP_CANCEL
2468 ? "#pragma omp cancel"
2469 : "#pragma omp cancellation point", kind
, bad
);
2474 case GIMPLE_OMP_SECTIONS
:
2475 case GIMPLE_OMP_SINGLE
:
2476 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2477 switch (gimple_code (ctx
->stmt
))
2479 case GIMPLE_OMP_FOR
:
2480 case GIMPLE_OMP_SECTIONS
:
2481 case GIMPLE_OMP_SINGLE
:
2482 case GIMPLE_OMP_ORDERED
:
2483 case GIMPLE_OMP_MASTER
:
2484 case GIMPLE_OMP_TASK
:
2485 case GIMPLE_OMP_CRITICAL
:
2486 if (is_gimple_call (stmt
))
2488 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2489 != BUILT_IN_GOMP_BARRIER
)
2491 error_at (gimple_location (stmt
),
2492 "barrier region may not be closely nested inside "
2493 "of work-sharing, critical, ordered, master or "
2494 "explicit task region");
2497 error_at (gimple_location (stmt
),
2498 "work-sharing region may not be closely nested inside "
2499 "of work-sharing, critical, ordered, master or explicit "
2502 case GIMPLE_OMP_PARALLEL
:
2508 case GIMPLE_OMP_MASTER
:
2509 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2510 switch (gimple_code (ctx
->stmt
))
2512 case GIMPLE_OMP_FOR
:
2513 case GIMPLE_OMP_SECTIONS
:
2514 case GIMPLE_OMP_SINGLE
:
2515 case GIMPLE_OMP_TASK
:
2516 error_at (gimple_location (stmt
),
2517 "master region may not be closely nested inside "
2518 "of work-sharing or explicit task region");
2520 case GIMPLE_OMP_PARALLEL
:
2526 case GIMPLE_OMP_ORDERED
:
2527 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2528 switch (gimple_code (ctx
->stmt
))
2530 case GIMPLE_OMP_CRITICAL
:
2531 case GIMPLE_OMP_TASK
:
2532 error_at (gimple_location (stmt
),
2533 "ordered region may not be closely nested inside "
2534 "of critical or explicit task region");
2536 case GIMPLE_OMP_FOR
:
2537 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2538 OMP_CLAUSE_ORDERED
) == NULL
)
2540 error_at (gimple_location (stmt
),
2541 "ordered region must be closely nested inside "
2542 "a loop region with an ordered clause");
2546 case GIMPLE_OMP_PARALLEL
:
2547 error_at (gimple_location (stmt
),
2548 "ordered region must be closely nested inside "
2549 "a loop region with an ordered clause");
2555 case GIMPLE_OMP_CRITICAL
:
2556 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2557 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2558 && (gimple_omp_critical_name (stmt
)
2559 == gimple_omp_critical_name (ctx
->stmt
)))
2561 error_at (gimple_location (stmt
),
2562 "critical region may not be nested inside a critical "
2563 "region with the same name");
2567 case GIMPLE_OMP_TEAMS
:
2569 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2570 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2572 error_at (gimple_location (stmt
),
2573 "teams construct not closely nested inside of target "
2578 case GIMPLE_OMP_TARGET
:
2579 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2580 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2581 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2584 switch (gimple_omp_target_kind (stmt
))
2586 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2587 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2588 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2589 default: gcc_unreachable ();
2591 warning_at (gimple_location (stmt
), 0,
2592 "%s construct inside of target region", name
);
2602 /* Helper function scan_omp.
2604 Callback for walk_tree or operators in walk_gimple_stmt used to
2605 scan for OpenMP directives in TP. */
2608 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2610 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2611 omp_context
*ctx
= (omp_context
*) wi
->info
;
2614 switch (TREE_CODE (t
))
2621 *tp
= remap_decl (t
, &ctx
->cb
);
2625 if (ctx
&& TYPE_P (t
))
2626 *tp
= remap_type (t
, &ctx
->cb
);
2627 else if (!DECL_P (t
))
2632 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2633 if (tem
!= TREE_TYPE (t
))
2635 if (TREE_CODE (t
) == INTEGER_CST
)
2636 *tp
= build_int_cst_wide (tem
,
2637 TREE_INT_CST_LOW (t
),
2638 TREE_INT_CST_HIGH (t
));
2640 TREE_TYPE (t
) = tem
;
2650 /* Return true if FNDECL is a setjmp or a longjmp. */
2653 setjmp_or_longjmp_p (const_tree fndecl
)
2655 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2656 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2657 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2660 tree declname
= DECL_NAME (fndecl
);
2663 const char *name
= IDENTIFIER_POINTER (declname
);
2664 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2668 /* Helper function for scan_omp.
2670 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2671 the current statement in GSI. */
2674 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2675 struct walk_stmt_info
*wi
)
2677 gimple stmt
= gsi_stmt (*gsi
);
2678 omp_context
*ctx
= (omp_context
*) wi
->info
;
2680 if (gimple_has_location (stmt
))
2681 input_location
= gimple_location (stmt
);
2683 /* Check the OpenMP nesting restrictions. */
2684 bool remove
= false;
2685 if (is_gimple_omp (stmt
))
2686 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2687 else if (is_gimple_call (stmt
))
2689 tree fndecl
= gimple_call_fndecl (stmt
);
2692 if (setjmp_or_longjmp_p (fndecl
)
2694 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2695 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2698 error_at (gimple_location (stmt
),
2699 "setjmp/longjmp inside simd construct");
2701 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2702 switch (DECL_FUNCTION_CODE (fndecl
))
2704 case BUILT_IN_GOMP_BARRIER
:
2705 case BUILT_IN_GOMP_CANCEL
:
2706 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2707 case BUILT_IN_GOMP_TASKYIELD
:
2708 case BUILT_IN_GOMP_TASKWAIT
:
2709 case BUILT_IN_GOMP_TASKGROUP_START
:
2710 case BUILT_IN_GOMP_TASKGROUP_END
:
2711 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2720 stmt
= gimple_build_nop ();
2721 gsi_replace (gsi
, stmt
, false);
2724 *handled_ops_p
= true;
2726 switch (gimple_code (stmt
))
2728 case GIMPLE_OMP_PARALLEL
:
2729 taskreg_nesting_level
++;
2730 scan_omp_parallel (gsi
, ctx
);
2731 taskreg_nesting_level
--;
2734 case GIMPLE_OMP_TASK
:
2735 taskreg_nesting_level
++;
2736 scan_omp_task (gsi
, ctx
);
2737 taskreg_nesting_level
--;
2740 case GIMPLE_OMP_FOR
:
2741 scan_omp_for (stmt
, ctx
);
2744 case GIMPLE_OMP_SECTIONS
:
2745 scan_omp_sections (stmt
, ctx
);
2748 case GIMPLE_OMP_SINGLE
:
2749 scan_omp_single (stmt
, ctx
);
2752 case GIMPLE_OMP_SECTION
:
2753 case GIMPLE_OMP_MASTER
:
2754 case GIMPLE_OMP_TASKGROUP
:
2755 case GIMPLE_OMP_ORDERED
:
2756 case GIMPLE_OMP_CRITICAL
:
2757 ctx
= new_omp_context (stmt
, ctx
);
2758 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2761 case GIMPLE_OMP_TARGET
:
2762 scan_omp_target (stmt
, ctx
);
2765 case GIMPLE_OMP_TEAMS
:
2766 scan_omp_teams (stmt
, ctx
);
2773 *handled_ops_p
= false;
2775 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2776 insert_decl_map (&ctx
->cb
, var
, var
);
2780 *handled_ops_p
= false;
2788 /* Scan all the statements starting at the current statement. CTX
2789 contains context information about the OpenMP directives and
2790 clauses found during the scan. */
2793 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2795 location_t saved_location
;
2796 struct walk_stmt_info wi
;
2798 memset (&wi
, 0, sizeof (wi
));
2800 wi
.want_locations
= true;
2802 saved_location
= input_location
;
2803 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2804 input_location
= saved_location
;
2807 /* Re-gimplification and code generation routines. */
2809 /* Build a call to GOMP_barrier. */
2812 build_omp_barrier (tree lhs
)
2814 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2815 : BUILT_IN_GOMP_BARRIER
);
2816 gimple g
= gimple_build_call (fndecl
, 0);
2818 gimple_call_set_lhs (g
, lhs
);
2822 /* If a context was created for STMT when it was scanned, return it. */
2824 static omp_context
*
2825 maybe_lookup_ctx (gimple stmt
)
2828 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2829 return n
? (omp_context
*) n
->value
: NULL
;
2833 /* Find the mapping for DECL in CTX or the immediately enclosing
2834 context that has a mapping for DECL.
2836 If CTX is a nested parallel directive, we may have to use the decl
2837 mappings created in CTX's parent context. Suppose that we have the
2838 following parallel nesting (variable UIDs showed for clarity):
2841 #omp parallel shared(iD.1562) -> outer parallel
2842 iD.1562 = iD.1562 + 1;
2844 #omp parallel shared (iD.1562) -> inner parallel
2845 iD.1562 = iD.1562 - 1;
2847 Each parallel structure will create a distinct .omp_data_s structure
2848 for copying iD.1562 in/out of the directive:
2850 outer parallel .omp_data_s.1.i -> iD.1562
2851 inner parallel .omp_data_s.2.i -> iD.1562
2853 A shared variable mapping will produce a copy-out operation before
2854 the parallel directive and a copy-in operation after it. So, in
2855 this case we would have:
2858 .omp_data_o.1.i = iD.1562;
2859 #omp parallel shared(iD.1562) -> outer parallel
2860 .omp_data_i.1 = &.omp_data_o.1
2861 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2863 .omp_data_o.2.i = iD.1562; -> **
2864 #omp parallel shared(iD.1562) -> inner parallel
2865 .omp_data_i.2 = &.omp_data_o.2
2866 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2869 ** This is a problem. The symbol iD.1562 cannot be referenced
2870 inside the body of the outer parallel region. But since we are
2871 emitting this copy operation while expanding the inner parallel
2872 directive, we need to access the CTX structure of the outer
2873 parallel directive to get the correct mapping:
2875 .omp_data_o.2.i = .omp_data_i.1->i
2877 Since there may be other workshare or parallel directives enclosing
2878 the parallel directive, it may be necessary to walk up the context
2879 parent chain. This is not a problem in general because nested
2880 parallelism happens only rarely. */
2883 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2888 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2889 t
= maybe_lookup_decl (decl
, up
);
2891 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2893 return t
? t
: decl
;
2897 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2898 in outer contexts. */
2901 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2906 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2907 t
= maybe_lookup_decl (decl
, up
);
2909 return t
? t
: decl
;
2913 /* Construct the initialization value for reduction CLAUSE. */
2916 omp_reduction_init (tree clause
, tree type
)
2918 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2919 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2926 case TRUTH_ORIF_EXPR
:
2927 case TRUTH_XOR_EXPR
:
2929 return build_zero_cst (type
);
2932 case TRUTH_AND_EXPR
:
2933 case TRUTH_ANDIF_EXPR
:
2935 return fold_convert_loc (loc
, type
, integer_one_node
);
2938 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2941 if (SCALAR_FLOAT_TYPE_P (type
))
2943 REAL_VALUE_TYPE max
, min
;
2944 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2947 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2950 real_maxval (&min
, 1, TYPE_MODE (type
));
2951 return build_real (type
, min
);
2955 gcc_assert (INTEGRAL_TYPE_P (type
));
2956 return TYPE_MIN_VALUE (type
);
2960 if (SCALAR_FLOAT_TYPE_P (type
))
2962 REAL_VALUE_TYPE max
;
2963 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2966 real_maxval (&max
, 0, TYPE_MODE (type
));
2967 return build_real (type
, max
);
2971 gcc_assert (INTEGRAL_TYPE_P (type
));
2972 return TYPE_MAX_VALUE (type
);
2980 /* Return alignment to be assumed for var in CLAUSE, which should be
2981 OMP_CLAUSE_ALIGNED. */
2984 omp_clause_aligned_alignment (tree clause
)
2986 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2987 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2989 /* Otherwise return implementation defined alignment. */
2990 unsigned int al
= 1;
2991 enum machine_mode mode
, vmode
;
2992 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2994 vs
= 1 << floor_log2 (vs
);
2995 static enum mode_class classes
[]
2996 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2997 for (int i
= 0; i
< 4; i
+= 2)
2998 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3000 mode
= GET_MODE_WIDER_MODE (mode
))
3002 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3003 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3006 && GET_MODE_SIZE (vmode
) < vs
3007 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3008 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3010 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3011 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3013 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3014 / GET_MODE_SIZE (mode
));
3015 if (TYPE_MODE (type
) != vmode
)
3017 if (TYPE_ALIGN_UNIT (type
) > al
)
3018 al
= TYPE_ALIGN_UNIT (type
);
3020 return build_int_cst (integer_type_node
, al
);
3023 /* Return maximum possible vectorization factor for the target. */
3030 || !flag_tree_loop_optimize
3031 || (!flag_tree_loop_vectorize
3032 && (global_options_set
.x_flag_tree_loop_vectorize
3033 || global_options_set
.x_flag_tree_vectorize
)))
3036 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3039 vs
= 1 << floor_log2 (vs
);
3042 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3043 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3044 return GET_MODE_NUNITS (vqimode
);
3048 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3052 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3053 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3057 max_vf
= omp_max_vf ();
3060 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3061 OMP_CLAUSE_SAFELEN
);
3062 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3064 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3066 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3070 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3071 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3077 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3078 tree avar
= create_tmp_var_raw (atype
, NULL
);
3079 if (TREE_ADDRESSABLE (new_var
))
3080 TREE_ADDRESSABLE (avar
) = 1;
3081 DECL_ATTRIBUTES (avar
)
3082 = tree_cons (get_identifier ("omp simd array"), NULL
,
3083 DECL_ATTRIBUTES (avar
));
3084 gimple_add_tmp_var (avar
);
3085 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3086 NULL_TREE
, NULL_TREE
);
3087 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3088 NULL_TREE
, NULL_TREE
);
3089 if (DECL_P (new_var
))
3091 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3092 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3097 /* Helper function of lower_rec_input_clauses. For a reference
3098 in simd reduction, add an underlying variable it will reference. */
3101 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3103 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3104 if (TREE_CONSTANT (z
))
3106 const char *name
= NULL
;
3107 if (DECL_NAME (new_vard
))
3108 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3110 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3111 gimple_add_tmp_var (z
);
3112 TREE_ADDRESSABLE (z
) = 1;
3113 z
= build_fold_addr_expr_loc (loc
, z
);
3114 gimplify_assign (new_vard
, z
, ilist
);
3118 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3119 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3120 private variables. Initialization statements go in ILIST, while calls
3121 to destructors go in DLIST. */
3124 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3125 omp_context
*ctx
, struct omp_for_data
*fd
)
3127 tree c
, dtor
, copyin_seq
, x
, ptr
;
3128 bool copyin_by_ref
= false;
3129 bool lastprivate_firstprivate
= false;
3130 bool reduction_omp_orig_ref
= false;
3132 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3133 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
);
3135 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3136 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3137 gimple_seq llist
[2] = { NULL
, NULL
};
3141 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3142 with data sharing clauses referencing variable sized vars. That
3143 is unnecessarily hard to support and very unlikely to result in
3144 vectorized code anyway. */
3146 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3147 switch (OMP_CLAUSE_CODE (c
))
3149 case OMP_CLAUSE_LINEAR
:
3150 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3153 case OMP_CLAUSE_REDUCTION
:
3154 case OMP_CLAUSE_PRIVATE
:
3155 case OMP_CLAUSE_FIRSTPRIVATE
:
3156 case OMP_CLAUSE_LASTPRIVATE
:
3157 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3164 /* Do all the fixed sized types in the first pass, and the variable sized
3165 types in the second pass. This makes sure that the scalar arguments to
3166 the variable sized types are processed before we use them in the
3167 variable sized operations. */
3168 for (pass
= 0; pass
< 2; ++pass
)
3170 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3172 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3175 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3179 case OMP_CLAUSE_PRIVATE
:
3180 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3183 case OMP_CLAUSE_SHARED
:
3184 /* Ignore shared directives in teams construct. */
3185 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3187 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3189 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3192 case OMP_CLAUSE_FIRSTPRIVATE
:
3193 case OMP_CLAUSE_COPYIN
:
3194 case OMP_CLAUSE_LINEAR
:
3196 case OMP_CLAUSE_REDUCTION
:
3197 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3198 reduction_omp_orig_ref
= true;
3200 case OMP_CLAUSE__LOOPTEMP_
:
3201 /* Handle _looptemp_ clauses only on parallel. */
3205 case OMP_CLAUSE_LASTPRIVATE
:
3206 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3208 lastprivate_firstprivate
= true;
3212 /* Even without corresponding firstprivate, if
3213 decl is Fortran allocatable, it needs outer var
3216 && lang_hooks
.decls
.omp_private_outer_ref
3217 (OMP_CLAUSE_DECL (c
)))
3218 lastprivate_firstprivate
= true;
3220 case OMP_CLAUSE_ALIGNED
:
3223 var
= OMP_CLAUSE_DECL (c
);
3224 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3225 && !is_global_var (var
))
3227 new_var
= maybe_lookup_decl (var
, ctx
);
3228 if (new_var
== NULL_TREE
)
3229 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3230 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3231 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3232 omp_clause_aligned_alignment (c
));
3233 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3234 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3235 gimplify_and_add (x
, ilist
);
3237 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3238 && is_global_var (var
))
3240 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3241 new_var
= lookup_decl (var
, ctx
);
3242 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3243 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3244 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3245 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3246 omp_clause_aligned_alignment (c
));
3247 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3248 x
= create_tmp_var (ptype
, NULL
);
3249 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3250 gimplify_and_add (t
, ilist
);
3251 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3252 SET_DECL_VALUE_EXPR (new_var
, t
);
3253 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3260 new_var
= var
= OMP_CLAUSE_DECL (c
);
3261 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3262 new_var
= lookup_decl (var
, ctx
);
3264 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3269 else if (is_variable_sized (var
))
3271 /* For variable sized types, we need to allocate the
3272 actual storage here. Call alloca and store the
3273 result in the pointer decl that we created elsewhere. */
3277 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3282 ptr
= DECL_VALUE_EXPR (new_var
);
3283 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3284 ptr
= TREE_OPERAND (ptr
, 0);
3285 gcc_assert (DECL_P (ptr
));
3286 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3288 /* void *tmp = __builtin_alloca */
3289 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3290 stmt
= gimple_build_call (atmp
, 1, x
);
3291 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3292 gimple_add_tmp_var (tmp
);
3293 gimple_call_set_lhs (stmt
, tmp
);
3295 gimple_seq_add_stmt (ilist
, stmt
);
3297 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3298 gimplify_assign (ptr
, x
, ilist
);
3301 else if (is_reference (var
))
3303 /* For references that are being privatized for Fortran,
3304 allocate new backing storage for the new pointer
3305 variable. This allows us to avoid changing all the
3306 code that expects a pointer to something that expects
3307 a direct variable. */
3311 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3312 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3314 x
= build_receiver_ref (var
, false, ctx
);
3315 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3317 else if (TREE_CONSTANT (x
))
3319 /* For reduction in SIMD loop, defer adding the
3320 initialization of the reference, because if we decide
3321 to use SIMD array for it, the initilization could cause
3323 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3327 const char *name
= NULL
;
3328 if (DECL_NAME (var
))
3329 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3331 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3333 gimple_add_tmp_var (x
);
3334 TREE_ADDRESSABLE (x
) = 1;
3335 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3340 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3341 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3346 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3347 gimplify_assign (new_var
, x
, ilist
);
3350 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3352 else if (c_kind
== OMP_CLAUSE_REDUCTION
3353 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3361 switch (OMP_CLAUSE_CODE (c
))
3363 case OMP_CLAUSE_SHARED
:
3364 /* Ignore shared directives in teams construct. */
3365 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3367 /* Shared global vars are just accessed directly. */
3368 if (is_global_var (new_var
))
3370 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3371 needs to be delayed until after fixup_child_record_type so
3372 that we get the correct type during the dereference. */
3373 by_ref
= use_pointer_for_field (var
, ctx
);
3374 x
= build_receiver_ref (var
, by_ref
, ctx
);
3375 SET_DECL_VALUE_EXPR (new_var
, x
);
3376 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3378 /* ??? If VAR is not passed by reference, and the variable
3379 hasn't been initialized yet, then we'll get a warning for
3380 the store into the omp_data_s structure. Ideally, we'd be
3381 able to notice this and not store anything at all, but
3382 we're generating code too early. Suppress the warning. */
3384 TREE_NO_WARNING (var
) = 1;
3387 case OMP_CLAUSE_LASTPRIVATE
:
3388 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3392 case OMP_CLAUSE_PRIVATE
:
3393 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3394 x
= build_outer_var_ref (var
, ctx
);
3395 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3397 if (is_task_ctx (ctx
))
3398 x
= build_receiver_ref (var
, false, ctx
);
3400 x
= build_outer_var_ref (var
, ctx
);
3406 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3409 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3410 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3411 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3412 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3413 idx
, lane
, ivar
, lvar
))
3416 x
= lang_hooks
.decls
.omp_clause_default_ctor
3417 (c
, unshare_expr (ivar
), x
);
3419 gimplify_and_add (x
, &llist
[0]);
3422 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3425 gimple_seq tseq
= NULL
;
3428 gimplify_stmt (&dtor
, &tseq
);
3429 gimple_seq_add_seq (&llist
[1], tseq
);
3436 gimplify_and_add (nx
, ilist
);
3440 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3443 gimple_seq tseq
= NULL
;
3446 gimplify_stmt (&dtor
, &tseq
);
3447 gimple_seq_add_seq (dlist
, tseq
);
3451 case OMP_CLAUSE_LINEAR
:
3452 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3453 goto do_firstprivate
;
3454 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3457 x
= build_outer_var_ref (var
, ctx
);
3460 case OMP_CLAUSE_FIRSTPRIVATE
:
3461 if (is_task_ctx (ctx
))
3463 if (is_reference (var
) || is_variable_sized (var
))
3465 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3467 || use_pointer_for_field (var
, NULL
))
3469 x
= build_receiver_ref (var
, false, ctx
);
3470 SET_DECL_VALUE_EXPR (new_var
, x
);
3471 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3476 x
= build_outer_var_ref (var
, ctx
);
3479 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3480 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3482 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3483 tree stept
= TREE_TYPE (t
);
3484 tree ct
= find_omp_clause (clauses
,
3485 OMP_CLAUSE__LOOPTEMP_
);
3487 tree l
= OMP_CLAUSE_DECL (ct
);
3488 tree n1
= fd
->loop
.n1
;
3489 tree step
= fd
->loop
.step
;
3490 tree itype
= TREE_TYPE (l
);
3491 if (POINTER_TYPE_P (itype
))
3492 itype
= signed_type_for (itype
);
3493 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3494 if (TYPE_UNSIGNED (itype
)
3495 && fd
->loop
.cond_code
== GT_EXPR
)
3496 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3497 fold_build1 (NEGATE_EXPR
, itype
, l
),
3498 fold_build1 (NEGATE_EXPR
,
3501 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3502 t
= fold_build2 (MULT_EXPR
, stept
,
3503 fold_convert (stept
, l
), t
);
3505 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3507 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3509 gimplify_and_add (x
, ilist
);
3513 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3514 x
= fold_build2 (POINTER_PLUS_EXPR
,
3515 TREE_TYPE (x
), x
, t
);
3517 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3520 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3521 || TREE_ADDRESSABLE (new_var
))
3522 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3523 idx
, lane
, ivar
, lvar
))
3525 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3527 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3528 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3529 gimplify_and_add (x
, ilist
);
3530 gimple_stmt_iterator gsi
3531 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3533 = gimple_build_assign (unshare_expr (lvar
), iv
);
3534 gsi_insert_before_without_update (&gsi
, g
,
3536 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3537 enum tree_code code
= PLUS_EXPR
;
3538 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3539 code
= POINTER_PLUS_EXPR
;
3540 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3541 gsi_insert_before_without_update (&gsi
, g
,
3545 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3546 (c
, unshare_expr (ivar
), x
);
3547 gimplify_and_add (x
, &llist
[0]);
3548 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3551 gimple_seq tseq
= NULL
;
3554 gimplify_stmt (&dtor
, &tseq
);
3555 gimple_seq_add_seq (&llist
[1], tseq
);
3560 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3561 gimplify_and_add (x
, ilist
);
3564 case OMP_CLAUSE__LOOPTEMP_
:
3565 gcc_assert (is_parallel_ctx (ctx
));
3566 x
= build_outer_var_ref (var
, ctx
);
3567 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3568 gimplify_and_add (x
, ilist
);
3571 case OMP_CLAUSE_COPYIN
:
3572 by_ref
= use_pointer_for_field (var
, NULL
);
3573 x
= build_receiver_ref (var
, by_ref
, ctx
);
3574 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3575 append_to_statement_list (x
, ©in_seq
);
3576 copyin_by_ref
|= by_ref
;
3579 case OMP_CLAUSE_REDUCTION
:
3580 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3582 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3584 x
= build_outer_var_ref (var
, ctx
);
3586 if (is_reference (var
)
3587 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3589 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3590 SET_DECL_VALUE_EXPR (placeholder
, x
);
3591 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3592 tree new_vard
= new_var
;
3593 if (is_reference (var
))
3595 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3596 new_vard
= TREE_OPERAND (new_var
, 0);
3597 gcc_assert (DECL_P (new_vard
));
3600 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3601 idx
, lane
, ivar
, lvar
))
3603 if (new_vard
== new_var
)
3605 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3606 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3610 SET_DECL_VALUE_EXPR (new_vard
,
3611 build_fold_addr_expr (ivar
));
3612 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3614 x
= lang_hooks
.decls
.omp_clause_default_ctor
3615 (c
, unshare_expr (ivar
),
3616 build_outer_var_ref (var
, ctx
));
3618 gimplify_and_add (x
, &llist
[0]);
3619 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3621 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3622 lower_omp (&tseq
, ctx
);
3623 gimple_seq_add_seq (&llist
[0], tseq
);
3625 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3626 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3627 lower_omp (&tseq
, ctx
);
3628 gimple_seq_add_seq (&llist
[1], tseq
);
3629 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3630 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3631 if (new_vard
== new_var
)
3632 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3634 SET_DECL_VALUE_EXPR (new_vard
,
3635 build_fold_addr_expr (lvar
));
3636 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3641 gimplify_stmt (&dtor
, &tseq
);
3642 gimple_seq_add_seq (&llist
[1], tseq
);
3646 /* If this is a reference to constant size reduction var
3647 with placeholder, we haven't emitted the initializer
3648 for it because it is undesirable if SIMD arrays are used.
3649 But if they aren't used, we need to emit the deferred
3650 initialization now. */
3651 else if (is_reference (var
) && is_simd
)
3652 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3653 x
= lang_hooks
.decls
.omp_clause_default_ctor
3654 (c
, unshare_expr (new_var
),
3655 build_outer_var_ref (var
, ctx
));
3657 gimplify_and_add (x
, ilist
);
3658 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3660 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3661 lower_omp (&tseq
, ctx
);
3662 gimple_seq_add_seq (ilist
, tseq
);
3664 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3667 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3668 lower_omp (&tseq
, ctx
);
3669 gimple_seq_add_seq (dlist
, tseq
);
3670 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3672 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3677 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3678 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3679 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3681 /* reduction(-:var) sums up the partial results, so it
3682 acts identically to reduction(+:var). */
3683 if (code
== MINUS_EXPR
)
3686 tree new_vard
= new_var
;
3687 if (is_simd
&& is_reference (var
))
3689 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3690 new_vard
= TREE_OPERAND (new_var
, 0);
3691 gcc_assert (DECL_P (new_vard
));
3694 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3695 idx
, lane
, ivar
, lvar
))
3697 tree ref
= build_outer_var_ref (var
, ctx
);
3699 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3701 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3702 ref
= build_outer_var_ref (var
, ctx
);
3703 gimplify_assign (ref
, x
, &llist
[1]);
3705 if (new_vard
!= new_var
)
3707 SET_DECL_VALUE_EXPR (new_vard
,
3708 build_fold_addr_expr (lvar
));
3709 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3714 if (is_reference (var
) && is_simd
)
3715 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3716 gimplify_assign (new_var
, x
, ilist
);
3719 tree ref
= build_outer_var_ref (var
, ctx
);
3721 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3722 ref
= build_outer_var_ref (var
, ctx
);
3723 gimplify_assign (ref
, x
, dlist
);
3737 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3738 /* Don't want uninit warnings on simduid, it is always uninitialized,
3739 but we use it not for the value, but for the DECL_UID only. */
3740 TREE_NO_WARNING (uid
) = 1;
3742 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3743 gimple_call_set_lhs (g
, lane
);
3744 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3745 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3746 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3747 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3748 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3749 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3750 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3751 build_int_cst (unsigned_type_node
, 0),
3753 gimple_seq_add_stmt (ilist
, g
);
3754 for (int i
= 0; i
< 2; i
++)
3757 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3758 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3759 gimple_call_set_lhs (g
, vf
);
3760 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3761 gimple_seq_add_stmt (seq
, g
);
3762 tree t
= build_int_cst (unsigned_type_node
, 0);
3763 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3764 gimple_seq_add_stmt (seq
, g
);
3765 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3766 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3767 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3768 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3769 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3770 gimple_seq_add_seq (seq
, llist
[i
]);
3771 t
= build_int_cst (unsigned_type_node
, 1);
3772 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3773 gimple_seq_add_stmt (seq
, g
);
3774 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3775 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3776 gimple_seq_add_stmt (seq
, g
);
3777 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3781 /* The copyin sequence is not to be executed by the main thread, since
3782 that would result in self-copies. Perhaps not visible to scalars,
3783 but it certainly is to C++ operator=. */
3786 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3788 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3789 build_int_cst (TREE_TYPE (x
), 0));
3790 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3791 gimplify_and_add (x
, ilist
);
3794 /* If any copyin variable is passed by reference, we must ensure the
3795 master thread doesn't modify it before it is copied over in all
3796 threads. Similarly for variables in both firstprivate and
3797 lastprivate clauses we need to ensure the lastprivate copying
3798 happens after firstprivate copying in all threads. And similarly
3799 for UDRs if initializer expression refers to omp_orig. */
3800 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3802 /* Don't add any barrier for #pragma omp simd or
3803 #pragma omp distribute. */
3804 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3805 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3806 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3809 /* If max_vf is non-zero, then we can use only a vectorization factor
3810 up to the max_vf we chose. So stick it into the safelen clause. */
3813 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3814 OMP_CLAUSE_SAFELEN
);
3816 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3817 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3820 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3821 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3823 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3824 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3830 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3831 both parallel and workshare constructs. PREDICATE may be NULL if it's
3835 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3838 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3839 bool par_clauses
= false;
3840 tree simduid
= NULL
, lastlane
= NULL
;
3842 /* Early exit if there are no lastprivate or linear clauses. */
3843 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3844 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3845 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3846 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3848 if (clauses
== NULL
)
3850 /* If this was a workshare clause, see if it had been combined
3851 with its parallel. In that case, look for the clauses on the
3852 parallel statement itself. */
3853 if (is_parallel_ctx (ctx
))
3857 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3860 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3861 OMP_CLAUSE_LASTPRIVATE
);
3862 if (clauses
== NULL
)
3870 tree label_true
, arm1
, arm2
;
3872 label
= create_artificial_label (UNKNOWN_LOCATION
);
3873 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3874 arm1
= TREE_OPERAND (predicate
, 0);
3875 arm2
= TREE_OPERAND (predicate
, 1);
3876 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3877 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3878 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3880 gimple_seq_add_stmt (stmt_list
, stmt
);
3881 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3884 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3885 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3887 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3889 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3892 for (c
= clauses
; c
;)
3895 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3897 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3898 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3899 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3901 var
= OMP_CLAUSE_DECL (c
);
3902 new_var
= lookup_decl (var
, ctx
);
3904 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3906 tree val
= DECL_VALUE_EXPR (new_var
);
3907 if (TREE_CODE (val
) == ARRAY_REF
3908 && VAR_P (TREE_OPERAND (val
, 0))
3909 && lookup_attribute ("omp simd array",
3910 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3913 if (lastlane
== NULL
)
3915 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3917 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3919 TREE_OPERAND (val
, 1));
3920 gimple_call_set_lhs (g
, lastlane
);
3921 gimple_seq_add_stmt (stmt_list
, g
);
3923 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3924 TREE_OPERAND (val
, 0), lastlane
,
3925 NULL_TREE
, NULL_TREE
);
3929 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3930 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3932 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3933 gimple_seq_add_seq (stmt_list
,
3934 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3935 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3937 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3938 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
3940 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
3941 gimple_seq_add_seq (stmt_list
,
3942 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
3943 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
3946 x
= build_outer_var_ref (var
, ctx
);
3947 if (is_reference (var
))
3948 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3949 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3950 gimplify_and_add (x
, stmt_list
);
3952 c
= OMP_CLAUSE_CHAIN (c
);
3953 if (c
== NULL
&& !par_clauses
)
3955 /* If this was a workshare clause, see if it had been combined
3956 with its parallel. In that case, continue looking for the
3957 clauses also on the parallel statement itself. */
3958 if (is_parallel_ctx (ctx
))
3962 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3965 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3966 OMP_CLAUSE_LASTPRIVATE
);
3972 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3976 /* Generate code to implement the REDUCTION clauses. */
3979 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3981 gimple_seq sub_seq
= NULL
;
3986 /* SIMD reductions are handled in lower_rec_input_clauses. */
3987 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3988 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3991 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3992 update in that case, otherwise use a lock. */
3993 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3994 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3996 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3998 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4008 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4010 tree var
, ref
, new_var
;
4011 enum tree_code code
;
4012 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4014 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4017 var
= OMP_CLAUSE_DECL (c
);
4018 new_var
= lookup_decl (var
, ctx
);
4019 if (is_reference (var
))
4020 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4021 ref
= build_outer_var_ref (var
, ctx
);
4022 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4024 /* reduction(-:var) sums up the partial results, so it acts
4025 identically to reduction(+:var). */
4026 if (code
== MINUS_EXPR
)
4031 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4033 addr
= save_expr (addr
);
4034 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4035 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4036 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4037 gimplify_and_add (x
, stmt_seqp
);
4041 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4043 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4045 if (is_reference (var
)
4046 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4048 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4049 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4050 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4051 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4052 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4053 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4054 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4058 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4059 ref
= build_outer_var_ref (var
, ctx
);
4060 gimplify_assign (ref
, x
, &sub_seq
);
4064 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4066 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4068 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4070 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4072 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4076 /* Generate code to implement the COPYPRIVATE clauses. */
4079 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4084 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4086 tree var
, new_var
, ref
, x
;
4088 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4090 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4093 var
= OMP_CLAUSE_DECL (c
);
4094 by_ref
= use_pointer_for_field (var
, NULL
);
4096 ref
= build_sender_ref (var
, ctx
);
4097 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4100 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4101 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4103 gimplify_assign (ref
, x
, slist
);
4105 ref
= build_receiver_ref (var
, false, ctx
);
4108 ref
= fold_convert_loc (clause_loc
,
4109 build_pointer_type (TREE_TYPE (new_var
)),
4111 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4113 if (is_reference (var
))
4115 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4116 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4117 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4119 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4120 gimplify_and_add (x
, rlist
);
4125 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4126 and REDUCTION from the sender (aka parent) side. */
4129 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4134 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4136 tree val
, ref
, x
, var
;
4137 bool by_ref
, do_in
= false, do_out
= false;
4138 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4140 switch (OMP_CLAUSE_CODE (c
))
4142 case OMP_CLAUSE_PRIVATE
:
4143 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4146 case OMP_CLAUSE_FIRSTPRIVATE
:
4147 case OMP_CLAUSE_COPYIN
:
4148 case OMP_CLAUSE_LASTPRIVATE
:
4149 case OMP_CLAUSE_REDUCTION
:
4150 case OMP_CLAUSE__LOOPTEMP_
:
4156 val
= OMP_CLAUSE_DECL (c
);
4157 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4159 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4160 && is_global_var (var
))
4162 if (is_variable_sized (val
))
4164 by_ref
= use_pointer_for_field (val
, NULL
);
4166 switch (OMP_CLAUSE_CODE (c
))
4168 case OMP_CLAUSE_PRIVATE
:
4169 case OMP_CLAUSE_FIRSTPRIVATE
:
4170 case OMP_CLAUSE_COPYIN
:
4171 case OMP_CLAUSE__LOOPTEMP_
:
4175 case OMP_CLAUSE_LASTPRIVATE
:
4176 if (by_ref
|| is_reference (val
))
4178 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4185 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4190 case OMP_CLAUSE_REDUCTION
:
4192 do_out
= !(by_ref
|| is_reference (val
));
4201 ref
= build_sender_ref (val
, ctx
);
4202 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4203 gimplify_assign (ref
, x
, ilist
);
4204 if (is_task_ctx (ctx
))
4205 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4210 ref
= build_sender_ref (val
, ctx
);
4211 gimplify_assign (var
, ref
, olist
);
4216 /* Generate code to implement SHARED from the sender (aka parent)
4217 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4218 list things that got automatically shared. */
4221 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4223 tree var
, ovar
, nvar
, f
, x
, record_type
;
4225 if (ctx
->record_type
== NULL
)
4228 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4229 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4231 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4232 nvar
= maybe_lookup_decl (ovar
, ctx
);
4233 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4236 /* If CTX is a nested parallel directive. Find the immediately
4237 enclosing parallel or workshare construct that contains a
4238 mapping for OVAR. */
4239 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4241 if (use_pointer_for_field (ovar
, ctx
))
4243 x
= build_sender_ref (ovar
, ctx
);
4244 var
= build_fold_addr_expr (var
);
4245 gimplify_assign (x
, var
, ilist
);
4249 x
= build_sender_ref (ovar
, ctx
);
4250 gimplify_assign (x
, var
, ilist
);
4252 if (!TREE_READONLY (var
)
4253 /* We don't need to receive a new reference to a result
4254 or parm decl. In fact we may not store to it as we will
4255 invalidate any pending RSO and generate wrong gimple
4257 && !((TREE_CODE (var
) == RESULT_DECL
4258 || TREE_CODE (var
) == PARM_DECL
)
4259 && DECL_BY_REFERENCE (var
)))
4261 x
= build_sender_ref (ovar
, ctx
);
4262 gimplify_assign (var
, x
, olist
);
4269 /* A convenience function to build an empty GIMPLE_COND with just the
4273 gimple_build_cond_empty (tree cond
)
4275 enum tree_code pred_code
;
4278 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4279 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4283 /* Build the function calls to GOMP_parallel_start etc to actually
4284 generate the parallel operation. REGION is the parallel region
4285 being expanded. BB is the block where to insert the code. WS_ARGS
4286 will be set if this is a call to a combined parallel+workshare
4287 construct, it contains the list of additional arguments needed by
4288 the workshare construct. */
4291 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4292 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4294 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4295 gimple_stmt_iterator gsi
;
4297 enum built_in_function start_ix
;
4299 location_t clause_loc
;
4300 vec
<tree
, va_gc
> *args
;
4302 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4304 /* Determine what flavor of GOMP_parallel we will be
4306 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4307 if (is_combined_parallel (region
))
4309 switch (region
->inner
->type
)
4311 case GIMPLE_OMP_FOR
:
4312 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4313 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4314 + (region
->inner
->sched_kind
4315 == OMP_CLAUSE_SCHEDULE_RUNTIME
4316 ? 3 : region
->inner
->sched_kind
));
4317 start_ix
= (enum built_in_function
)start_ix2
;
4319 case GIMPLE_OMP_SECTIONS
:
4320 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4327 /* By default, the value of NUM_THREADS is zero (selected at run time)
4328 and there is no conditional. */
4330 val
= build_int_cst (unsigned_type_node
, 0);
4331 flags
= build_int_cst (unsigned_type_node
, 0);
4333 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4335 cond
= OMP_CLAUSE_IF_EXPR (c
);
4337 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4340 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4341 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4344 clause_loc
= gimple_location (entry_stmt
);
4346 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4348 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4350 /* Ensure 'val' is of the correct type. */
4351 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4353 /* If we found the clause 'if (cond)', build either
4354 (cond != 0) or (cond ? val : 1u). */
4357 gimple_stmt_iterator gsi
;
4359 cond
= gimple_boolify (cond
);
4361 if (integer_zerop (val
))
4362 val
= fold_build2_loc (clause_loc
,
4363 EQ_EXPR
, unsigned_type_node
, cond
,
4364 build_int_cst (TREE_TYPE (cond
), 0));
4367 basic_block cond_bb
, then_bb
, else_bb
;
4368 edge e
, e_then
, e_else
;
4369 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4371 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4372 if (gimple_in_ssa_p (cfun
))
4374 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4375 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4376 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4385 e
= split_block (bb
, NULL
);
4390 then_bb
= create_empty_bb (cond_bb
);
4391 else_bb
= create_empty_bb (then_bb
);
4392 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4393 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4395 stmt
= gimple_build_cond_empty (cond
);
4396 gsi
= gsi_start_bb (cond_bb
);
4397 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4399 gsi
= gsi_start_bb (then_bb
);
4400 stmt
= gimple_build_assign (tmp_then
, val
);
4401 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4403 gsi
= gsi_start_bb (else_bb
);
4404 stmt
= gimple_build_assign
4405 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4406 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4408 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4409 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4412 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4413 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4415 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4416 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4418 if (gimple_in_ssa_p (cfun
))
4420 gimple phi
= create_phi_node (tmp_join
, bb
);
4421 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4422 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4428 gsi
= gsi_start_bb (bb
);
4429 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4430 false, GSI_CONTINUE_LINKING
);
4433 gsi
= gsi_last_bb (bb
);
4434 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4436 t1
= null_pointer_node
;
4438 t1
= build_fold_addr_expr (t
);
4439 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4441 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4442 args
->quick_push (t2
);
4443 args
->quick_push (t1
);
4444 args
->quick_push (val
);
4446 args
->splice (*ws_args
);
4447 args
->quick_push (flags
);
4449 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4450 builtin_decl_explicit (start_ix
), args
);
4452 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4453 false, GSI_CONTINUE_LINKING
);
4457 /* Build the function call to GOMP_task to actually
4458 generate the task operation. BB is the block where to insert the code. */
4461 expand_task_call (basic_block bb
, gimple entry_stmt
)
4463 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4464 gimple_stmt_iterator gsi
;
4465 location_t loc
= gimple_location (entry_stmt
);
4467 clauses
= gimple_omp_task_clauses (entry_stmt
);
4469 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4471 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4473 cond
= boolean_true_node
;
4475 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4476 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4477 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4478 flags
= build_int_cst (unsigned_type_node
,
4479 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4481 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4484 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4485 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4486 build_int_cst (unsigned_type_node
, 2),
4487 build_int_cst (unsigned_type_node
, 0));
4488 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4491 depend
= OMP_CLAUSE_DECL (depend
);
4493 depend
= build_int_cst (ptr_type_node
, 0);
4495 gsi
= gsi_last_bb (bb
);
4496 t
= gimple_omp_task_data_arg (entry_stmt
);
4498 t2
= null_pointer_node
;
4500 t2
= build_fold_addr_expr_loc (loc
, t
);
4501 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4502 t
= gimple_omp_task_copy_fn (entry_stmt
);
4504 t3
= null_pointer_node
;
4506 t3
= build_fold_addr_expr_loc (loc
, t
);
4508 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4510 gimple_omp_task_arg_size (entry_stmt
),
4511 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4514 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4515 false, GSI_CONTINUE_LINKING
);
4519 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4520 catch handler and return it. This prevents programs from violating the
4521 structured block semantics with throws. */
4524 maybe_catch_exception (gimple_seq body
)
4529 if (!flag_exceptions
)
4532 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4533 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4535 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4537 g
= gimple_build_eh_must_not_throw (decl
);
4538 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4541 return gimple_seq_alloc_with_stmt (g
);
4544 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4547 vec2chain (vec
<tree
, va_gc
> *v
)
4549 tree chain
= NULL_TREE
, t
;
4552 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4554 DECL_CHAIN (t
) = chain
;
4562 /* Remove barriers in REGION->EXIT's block. Note that this is only
4563 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4564 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4565 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4569 remove_exit_barrier (struct omp_region
*region
)
4571 gimple_stmt_iterator gsi
;
4572 basic_block exit_bb
;
4576 int any_addressable_vars
= -1;
4578 exit_bb
= region
->exit
;
4580 /* If the parallel region doesn't return, we don't have REGION->EXIT
4585 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4586 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4587 statements that can appear in between are extremely limited -- no
4588 memory operations at all. Here, we allow nothing at all, so the
4589 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4590 gsi
= gsi_last_bb (exit_bb
);
4591 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4593 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4596 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4598 gsi
= gsi_last_bb (e
->src
);
4599 if (gsi_end_p (gsi
))
4601 stmt
= gsi_stmt (gsi
);
4602 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4603 && !gimple_omp_return_nowait_p (stmt
))
4605 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4606 in many cases. If there could be tasks queued, the barrier
4607 might be needed to let the tasks run before some local
4608 variable of the parallel that the task uses as shared
4609 runs out of scope. The task can be spawned either
4610 from within current function (this would be easy to check)
4611 or from some function it calls and gets passed an address
4612 of such a variable. */
4613 if (any_addressable_vars
< 0)
4615 gimple parallel_stmt
= last_stmt (region
->entry
);
4616 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4617 tree local_decls
, block
, decl
;
4620 any_addressable_vars
= 0;
4621 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4622 if (TREE_ADDRESSABLE (decl
))
4624 any_addressable_vars
= 1;
4627 for (block
= gimple_block (stmt
);
4628 !any_addressable_vars
4630 && TREE_CODE (block
) == BLOCK
;
4631 block
= BLOCK_SUPERCONTEXT (block
))
4633 for (local_decls
= BLOCK_VARS (block
);
4635 local_decls
= DECL_CHAIN (local_decls
))
4636 if (TREE_ADDRESSABLE (local_decls
))
4638 any_addressable_vars
= 1;
4641 if (block
== gimple_block (parallel_stmt
))
4645 if (!any_addressable_vars
)
4646 gimple_omp_return_set_nowait (stmt
);
4652 remove_exit_barriers (struct omp_region
*region
)
4654 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4655 remove_exit_barrier (region
);
4659 region
= region
->inner
;
4660 remove_exit_barriers (region
);
4661 while (region
->next
)
4663 region
= region
->next
;
4664 remove_exit_barriers (region
);
4669 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4670 calls. These can't be declared as const functions, but
4671 within one parallel body they are constant, so they can be
4672 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4673 which are declared const. Similarly for task body, except
4674 that in untied task omp_get_thread_num () can change at any task
4675 scheduling point. */
4678 optimize_omp_library_calls (gimple entry_stmt
)
4681 gimple_stmt_iterator gsi
;
4682 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4683 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4684 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4685 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4686 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4687 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4688 OMP_CLAUSE_UNTIED
) != NULL
);
4690 FOR_EACH_BB_FN (bb
, cfun
)
4691 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4693 gimple call
= gsi_stmt (gsi
);
4696 if (is_gimple_call (call
)
4697 && (decl
= gimple_call_fndecl (call
))
4698 && DECL_EXTERNAL (decl
)
4699 && TREE_PUBLIC (decl
)
4700 && DECL_INITIAL (decl
) == NULL
)
4704 if (DECL_NAME (decl
) == thr_num_id
)
4706 /* In #pragma omp task untied omp_get_thread_num () can change
4707 during the execution of the task region. */
4710 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4712 else if (DECL_NAME (decl
) == num_thr_id
)
4713 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4717 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4718 || gimple_call_num_args (call
) != 0)
4721 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4724 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4725 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4726 TREE_TYPE (TREE_TYPE (built_in
))))
4729 gimple_call_set_fndecl (call
, built_in
);
4734 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4738 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4742 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4743 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4746 if (TREE_CODE (t
) == ADDR_EXPR
)
4747 recompute_tree_invariant_for_addr_expr (t
);
4749 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4753 /* Prepend TO = FROM assignment before *GSI_P. */
4756 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4758 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4759 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4760 true, GSI_SAME_STMT
);
4761 gimple stmt
= gimple_build_assign (to
, from
);
4762 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4763 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4764 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4766 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4767 gimple_regimplify_operands (stmt
, &gsi
);
4771 /* Expand the OpenMP parallel or task directive starting at REGION. */
4774 expand_omp_taskreg (struct omp_region
*region
)
4776 basic_block entry_bb
, exit_bb
, new_bb
;
4777 struct function
*child_cfun
;
4778 tree child_fn
, block
, t
;
4779 gimple_stmt_iterator gsi
;
4780 gimple entry_stmt
, stmt
;
4782 vec
<tree
, va_gc
> *ws_args
;
4784 entry_stmt
= last_stmt (region
->entry
);
4785 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4786 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4788 entry_bb
= region
->entry
;
4789 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
4790 exit_bb
= region
->cont
;
4792 exit_bb
= region
->exit
;
4794 if (is_combined_parallel (region
))
4795 ws_args
= region
->ws_args
;
4799 if (child_cfun
->cfg
)
4801 /* Due to inlining, it may happen that we have already outlined
4802 the region, in which case all we need to do is make the
4803 sub-graph unreachable and emit the parallel call. */
4804 edge entry_succ_e
, exit_succ_e
;
4805 gimple_stmt_iterator gsi
;
4807 entry_succ_e
= single_succ_edge (entry_bb
);
4809 gsi
= gsi_last_bb (entry_bb
);
4810 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4811 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4812 gsi_remove (&gsi
, true);
4817 exit_succ_e
= single_succ_edge (exit_bb
);
4818 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4820 remove_edge_and_dominated_blocks (entry_succ_e
);
4824 unsigned srcidx
, dstidx
, num
;
4826 /* If the parallel region needs data sent from the parent
4827 function, then the very first statement (except possible
4828 tree profile counter updates) of the parallel body
4829 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4830 &.OMP_DATA_O is passed as an argument to the child function,
4831 we need to replace it with the argument as seen by the child
4834 In most cases, this will end up being the identity assignment
4835 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4836 a function call that has been inlined, the original PARM_DECL
4837 .OMP_DATA_I may have been converted into a different local
4838 variable. In which case, we need to keep the assignment. */
4839 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4841 basic_block entry_succ_bb
4842 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
4843 : FALLTHRU_EDGE (entry_bb
)->dest
;
4844 gimple_stmt_iterator gsi
;
4846 gimple parcopy_stmt
= NULL
;
4848 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4852 gcc_assert (!gsi_end_p (gsi
));
4853 stmt
= gsi_stmt (gsi
);
4854 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4857 if (gimple_num_ops (stmt
) == 2)
4859 tree arg
= gimple_assign_rhs1 (stmt
);
4861 /* We're ignore the subcode because we're
4862 effectively doing a STRIP_NOPS. */
4864 if (TREE_CODE (arg
) == ADDR_EXPR
4865 && TREE_OPERAND (arg
, 0)
4866 == gimple_omp_taskreg_data_arg (entry_stmt
))
4868 parcopy_stmt
= stmt
;
4874 gcc_assert (parcopy_stmt
!= NULL
);
4875 arg
= DECL_ARGUMENTS (child_fn
);
4877 if (!gimple_in_ssa_p (cfun
))
4879 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4880 gsi_remove (&gsi
, true);
4883 /* ?? Is setting the subcode really necessary ?? */
4884 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4885 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4890 /* If we are in ssa form, we must load the value from the default
4891 definition of the argument. That should not be defined now,
4892 since the argument is not used uninitialized. */
4893 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4894 narg
= make_ssa_name (arg
, gimple_build_nop ());
4895 set_ssa_default_def (cfun
, arg
, narg
);
4896 /* ?? Is setting the subcode really necessary ?? */
4897 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4898 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4899 update_stmt (parcopy_stmt
);
4903 /* Declare local variables needed in CHILD_CFUN. */
4904 block
= DECL_INITIAL (child_fn
);
4905 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4906 /* The gimplifier could record temporaries in parallel/task block
4907 rather than in containing function's local_decls chain,
4908 which would mean cgraph missed finalizing them. Do it now. */
4909 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4910 if (TREE_CODE (t
) == VAR_DECL
4912 && !DECL_EXTERNAL (t
))
4913 varpool_finalize_decl (t
);
4914 DECL_SAVED_TREE (child_fn
) = NULL
;
4915 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4916 gimple_set_body (child_fn
, NULL
);
4917 TREE_USED (block
) = 1;
4919 /* Reset DECL_CONTEXT on function arguments. */
4920 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4921 DECL_CONTEXT (t
) = child_fn
;
4923 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4924 so that it can be moved to the child function. */
4925 gsi
= gsi_last_bb (entry_bb
);
4926 stmt
= gsi_stmt (gsi
);
4927 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4928 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4929 gsi_remove (&gsi
, true);
4930 e
= split_block (entry_bb
, stmt
);
4933 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4934 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4937 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
4938 gcc_assert (e2
->dest
== region
->exit
);
4939 remove_edge (BRANCH_EDGE (entry_bb
));
4940 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
4941 gsi
= gsi_last_bb (region
->exit
);
4942 gcc_assert (!gsi_end_p (gsi
)
4943 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4944 gsi_remove (&gsi
, true);
4947 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
4950 gsi
= gsi_last_bb (exit_bb
);
4951 gcc_assert (!gsi_end_p (gsi
)
4952 && (gimple_code (gsi_stmt (gsi
))
4953 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
4954 stmt
= gimple_build_return (NULL
);
4955 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4956 gsi_remove (&gsi
, true);
4959 /* Move the parallel region into CHILD_CFUN. */
4961 if (gimple_in_ssa_p (cfun
))
4963 init_tree_ssa (child_cfun
);
4964 init_ssa_operands (child_cfun
);
4965 child_cfun
->gimple_df
->in_ssa_p
= true;
4969 block
= gimple_block (entry_stmt
);
4971 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4973 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4976 basic_block dest_bb
= e2
->dest
;
4978 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
4980 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
4982 /* When the OMP expansion process cannot guarantee an up-to-date
4983 loop tree arrange for the child function to fixup loops. */
4984 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4985 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4987 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4988 num
= vec_safe_length (child_cfun
->local_decls
);
4989 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4991 t
= (*child_cfun
->local_decls
)[srcidx
];
4992 if (DECL_CONTEXT (t
) == cfun
->decl
)
4994 if (srcidx
!= dstidx
)
4995 (*child_cfun
->local_decls
)[dstidx
] = t
;
4999 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5001 /* Inform the callgraph about the new function. */
5002 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5003 cgraph_add_new_function (child_fn
, true);
5005 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5006 fixed in a following pass. */
5007 push_cfun (child_cfun
);
5009 optimize_omp_library_calls (entry_stmt
);
5010 rebuild_cgraph_edges ();
5012 /* Some EH regions might become dead, see PR34608. If
5013 pass_cleanup_cfg isn't the first pass to happen with the
5014 new child, these dead EH edges might cause problems.
5015 Clean them up now. */
5016 if (flag_exceptions
)
5019 bool changed
= false;
5021 FOR_EACH_BB_FN (bb
, cfun
)
5022 changed
|= gimple_purge_dead_eh_edges (bb
);
5024 cleanup_tree_cfg ();
5026 if (gimple_in_ssa_p (cfun
))
5027 update_ssa (TODO_update_ssa
);
5031 /* Emit a library call to launch the children threads. */
5032 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5033 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5035 expand_task_call (new_bb
, entry_stmt
);
5036 if (gimple_in_ssa_p (cfun
))
5037 update_ssa (TODO_update_ssa_only_virtuals
);
5041 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5042 of the combined collapse > 1 loop constructs, generate code like:
5043 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5048 count3 = (adj + N32 - N31) / STEP3;
5049 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5054 count2 = (adj + N22 - N21) / STEP2;
5055 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5060 count1 = (adj + N12 - N11) / STEP1;
5061 count = count1 * count2 * count3;
5062 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5064 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5065 of the combined loop constructs, just initialize COUNTS array
5066 from the _looptemp_ clauses. */
5068 /* NOTE: It *could* be better to moosh all of the BBs together,
5069 creating one larger BB with all the computation and the unexpected
5070 jump at the end. I.e.
5072 bool zero3, zero2, zero1, zero;
5075 count3 = (N32 - N31) /[cl] STEP3;
5077 count2 = (N22 - N21) /[cl] STEP2;
5079 count1 = (N12 - N11) /[cl] STEP1;
5080 zero = zero3 || zero2 || zero1;
5081 count = count1 * count2 * count3;
5082 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5084 After all, we expect the zero=false, and thus we expect to have to
5085 evaluate all of the comparison expressions, so short-circuiting
5086 oughtn't be a win. Since the condition isn't protecting a
5087 denominator, we're not concerned about divide-by-zero, so we can
5088 fully evaluate count even if a numerator turned out to be wrong.
5090 It seems like putting this all together would create much better
5091 scheduling opportunities, and less pressure on the chip's branch
5095 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5096 basic_block
&entry_bb
, tree
*counts
,
5097 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5098 basic_block
&l2_dom_bb
)
5100 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5105 /* Collapsed loops need work for expansion into SSA form. */
5106 gcc_assert (!gimple_in_ssa_p (cfun
));
5108 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5109 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5111 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5112 isn't supposed to be handled, as the inner loop doesn't
5114 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5115 OMP_CLAUSE__LOOPTEMP_
);
5116 gcc_assert (innerc
);
5117 for (i
= 0; i
< fd
->collapse
; i
++)
5119 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5120 OMP_CLAUSE__LOOPTEMP_
);
5121 gcc_assert (innerc
);
5123 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5125 counts
[0] = NULL_TREE
;
5130 for (i
= 0; i
< fd
->collapse
; i
++)
5132 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5134 if (SSA_VAR_P (fd
->loop
.n2
)
5135 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5136 fold_convert (itype
, fd
->loops
[i
].n1
),
5137 fold_convert (itype
, fd
->loops
[i
].n2
)))
5138 == NULL_TREE
|| !integer_onep (t
)))
5141 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5142 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5143 true, GSI_SAME_STMT
);
5144 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5145 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5146 true, GSI_SAME_STMT
);
5147 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5148 NULL_TREE
, NULL_TREE
);
5149 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5150 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5151 expand_omp_regimplify_p
, NULL
, NULL
)
5152 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5153 expand_omp_regimplify_p
, NULL
, NULL
))
5155 *gsi
= gsi_for_stmt (stmt
);
5156 gimple_regimplify_operands (stmt
, gsi
);
5158 e
= split_block (entry_bb
, stmt
);
5159 if (zero_iter_bb
== NULL
)
5161 first_zero_iter
= i
;
5162 zero_iter_bb
= create_empty_bb (entry_bb
);
5164 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5165 *gsi
= gsi_after_labels (zero_iter_bb
);
5166 stmt
= gimple_build_assign (fd
->loop
.n2
,
5167 build_zero_cst (type
));
5168 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5169 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5172 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5173 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5174 e
->flags
= EDGE_TRUE_VALUE
;
5175 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5176 if (l2_dom_bb
== NULL
)
5177 l2_dom_bb
= entry_bb
;
5179 *gsi
= gsi_last_bb (entry_bb
);
5182 if (POINTER_TYPE_P (itype
))
5183 itype
= signed_type_for (itype
);
5184 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5186 t
= fold_build2 (PLUS_EXPR
, itype
,
5187 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5188 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5189 fold_convert (itype
, fd
->loops
[i
].n2
));
5190 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5191 fold_convert (itype
, fd
->loops
[i
].n1
));
5192 /* ?? We could probably use CEIL_DIV_EXPR instead of
5193 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5194 generate the same code in the end because generically we
5195 don't know that the values involved must be negative for
5197 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5198 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5199 fold_build1 (NEGATE_EXPR
, itype
, t
),
5200 fold_build1 (NEGATE_EXPR
, itype
,
5201 fold_convert (itype
,
5202 fd
->loops
[i
].step
)));
5204 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5205 fold_convert (itype
, fd
->loops
[i
].step
));
5206 t
= fold_convert (type
, t
);
5207 if (TREE_CODE (t
) == INTEGER_CST
)
5211 counts
[i
] = create_tmp_reg (type
, ".count");
5212 expand_omp_build_assign (gsi
, counts
[i
], t
);
5214 if (SSA_VAR_P (fd
->loop
.n2
))
5219 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5220 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5226 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5228 V3 = N31 + (T % count3) * STEP3;
5230 V2 = N21 + (T % count2) * STEP2;
5232 V1 = N11 + T * STEP1;
5233 if this loop doesn't have an inner loop construct combined with it.
5234 If it does have an inner loop construct combined with it and the
5235 iteration count isn't known constant, store values from counts array
5236 into its _looptemp_ temporaries instead. */
5239 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5240 tree
*counts
, gimple inner_stmt
, tree startvar
)
5243 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5245 /* If fd->loop.n2 is constant, then no propagation of the counts
5246 is needed, they are constant. */
5247 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5250 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5251 ? gimple_omp_parallel_clauses (inner_stmt
)
5252 : gimple_omp_for_clauses (inner_stmt
);
5253 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5254 isn't supposed to be handled, as the inner loop doesn't
5256 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5257 gcc_assert (innerc
);
5258 for (i
= 0; i
< fd
->collapse
; i
++)
5260 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5261 OMP_CLAUSE__LOOPTEMP_
);
5262 gcc_assert (innerc
);
5265 tree tem
= OMP_CLAUSE_DECL (innerc
);
5266 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5267 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5268 false, GSI_CONTINUE_LINKING
);
5269 gimple stmt
= gimple_build_assign (tem
, t
);
5270 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5276 tree type
= TREE_TYPE (fd
->loop
.v
);
5277 tree tem
= create_tmp_reg (type
, ".tem");
5278 gimple stmt
= gimple_build_assign (tem
, startvar
);
5279 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5281 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5283 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5285 if (POINTER_TYPE_P (vtype
))
5286 itype
= signed_type_for (vtype
);
5288 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5291 t
= fold_convert (itype
, t
);
5292 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5293 fold_convert (itype
, fd
->loops
[i
].step
));
5294 if (POINTER_TYPE_P (vtype
))
5295 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5297 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5298 t
= force_gimple_operand_gsi (gsi
, t
,
5299 DECL_P (fd
->loops
[i
].v
)
5300 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5302 GSI_CONTINUE_LINKING
);
5303 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5304 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5307 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5308 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5309 false, GSI_CONTINUE_LINKING
);
5310 stmt
= gimple_build_assign (tem
, t
);
5311 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5317 /* Helper function for expand_omp_for_*. Generate code like:
5320 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5324 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5331 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5332 basic_block body_bb
)
5334 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5336 gimple_stmt_iterator gsi
;
5342 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5344 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5346 bb
= create_empty_bb (last_bb
);
5348 add_bb_to_loop (bb
, last_bb
->loop_father
);
5349 gsi
= gsi_start_bb (bb
);
5351 if (i
< fd
->collapse
- 1)
5353 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5354 e
->probability
= REG_BR_PROB_BASE
/ 8;
5356 t
= fd
->loops
[i
+ 1].n1
;
5357 t
= force_gimple_operand_gsi (&gsi
, t
,
5358 DECL_P (fd
->loops
[i
+ 1].v
)
5359 && TREE_ADDRESSABLE (fd
->loops
[i
5362 GSI_CONTINUE_LINKING
);
5363 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5364 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5369 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5371 if (POINTER_TYPE_P (vtype
))
5372 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5374 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5375 t
= force_gimple_operand_gsi (&gsi
, t
,
5376 DECL_P (fd
->loops
[i
].v
)
5377 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5378 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5379 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5380 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5384 t
= fd
->loops
[i
].n2
;
5385 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5386 false, GSI_CONTINUE_LINKING
);
5387 tree v
= fd
->loops
[i
].v
;
5388 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5389 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5390 false, GSI_CONTINUE_LINKING
);
5391 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5392 stmt
= gimple_build_cond_empty (t
);
5393 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5394 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5395 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5398 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5406 /* A subroutine of expand_omp_for. Generate code for a parallel
5407 loop with any schedule. Given parameters:
5409 for (V = N1; V cond N2; V += STEP) BODY;
5411 where COND is "<" or ">", we generate pseudocode
5413 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5414 if (more) goto L0; else goto L3;
5421 if (V cond iend) goto L1; else goto L2;
5423 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5426 If this is a combined omp parallel loop, instead of the call to
5427 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5428 If this is gimple_omp_for_combined_p loop, then instead of assigning
5429 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5430 inner GIMPLE_OMP_FOR and V += STEP; and
5431 if (V cond iend) goto L1; else goto L2; are removed.
5433 For collapsed loops, given parameters:
5435 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5436 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5437 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5440 we generate pseudocode
5442 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5447 count3 = (adj + N32 - N31) / STEP3;
5448 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5453 count2 = (adj + N22 - N21) / STEP2;
5454 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5459 count1 = (adj + N12 - N11) / STEP1;
5460 count = count1 * count2 * count3;
5465 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5466 if (more) goto L0; else goto L3;
5470 V3 = N31 + (T % count3) * STEP3;
5472 V2 = N21 + (T % count2) * STEP2;
5474 V1 = N11 + T * STEP1;
5479 if (V < iend) goto L10; else goto L2;
5482 if (V3 cond3 N32) goto L1; else goto L11;
5486 if (V2 cond2 N22) goto L1; else goto L12;
5492 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5498 expand_omp_for_generic (struct omp_region
*region
,
5499 struct omp_for_data
*fd
,
5500 enum built_in_function start_fn
,
5501 enum built_in_function next_fn
,
5504 tree type
, istart0
, iend0
, iend
;
5505 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5506 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5507 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5508 gimple_stmt_iterator gsi
;
5510 bool in_combined_parallel
= is_combined_parallel (region
);
5511 bool broken_loop
= region
->cont
== NULL
;
5513 tree
*counts
= NULL
;
5516 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5517 gcc_assert (fd
->iter_type
== long_integer_type_node
5518 || !in_combined_parallel
);
5520 type
= TREE_TYPE (fd
->loop
.v
);
5521 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5522 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5523 TREE_ADDRESSABLE (istart0
) = 1;
5524 TREE_ADDRESSABLE (iend0
) = 1;
5526 /* See if we need to bias by LLONG_MIN. */
5527 if (fd
->iter_type
== long_long_unsigned_type_node
5528 && TREE_CODE (type
) == INTEGER_TYPE
5529 && !TYPE_UNSIGNED (type
))
5533 if (fd
->loop
.cond_code
== LT_EXPR
)
5536 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5540 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5543 if (TREE_CODE (n1
) != INTEGER_CST
5544 || TREE_CODE (n2
) != INTEGER_CST
5545 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5546 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5549 entry_bb
= region
->entry
;
5550 cont_bb
= region
->cont
;
5552 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5553 gcc_assert (broken_loop
5554 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5555 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5556 l1_bb
= single_succ (l0_bb
);
5559 l2_bb
= create_empty_bb (cont_bb
);
5560 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5561 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5565 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5566 exit_bb
= region
->exit
;
5568 gsi
= gsi_last_bb (entry_bb
);
5570 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5571 if (fd
->collapse
> 1)
5573 int first_zero_iter
= -1;
5574 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5576 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5577 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5578 zero_iter_bb
, first_zero_iter
,
5583 /* Some counts[i] vars might be uninitialized if
5584 some loop has zero iterations. But the body shouldn't
5585 be executed in that case, so just avoid uninit warnings. */
5586 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5587 if (SSA_VAR_P (counts
[i
]))
5588 TREE_NO_WARNING (counts
[i
]) = 1;
5590 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5592 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5593 gsi
= gsi_last_bb (entry_bb
);
5594 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5595 get_immediate_dominator (CDI_DOMINATORS
,
5599 if (in_combined_parallel
)
5601 /* In a combined parallel loop, emit a call to
5602 GOMP_loop_foo_next. */
5603 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5604 build_fold_addr_expr (istart0
),
5605 build_fold_addr_expr (iend0
));
5609 tree t0
, t1
, t2
, t3
, t4
;
5610 /* If this is not a combined parallel loop, emit a call to
5611 GOMP_loop_foo_start in ENTRY_BB. */
5612 t4
= build_fold_addr_expr (iend0
);
5613 t3
= build_fold_addr_expr (istart0
);
5614 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5617 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5619 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5620 OMP_CLAUSE__LOOPTEMP_
);
5621 gcc_assert (innerc
);
5622 t0
= OMP_CLAUSE_DECL (innerc
);
5623 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5624 OMP_CLAUSE__LOOPTEMP_
);
5625 gcc_assert (innerc
);
5626 t1
= OMP_CLAUSE_DECL (innerc
);
5628 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5629 && TYPE_PRECISION (TREE_TYPE (t0
))
5630 != TYPE_PRECISION (fd
->iter_type
))
5632 /* Avoid casting pointers to integer of a different size. */
5633 tree itype
= signed_type_for (type
);
5634 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5635 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5639 t1
= fold_convert (fd
->iter_type
, t1
);
5640 t0
= fold_convert (fd
->iter_type
, t0
);
5644 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5645 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5647 if (fd
->iter_type
== long_integer_type_node
)
5651 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5652 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5653 6, t0
, t1
, t2
, t
, t3
, t4
);
5656 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5657 5, t0
, t1
, t2
, t3
, t4
);
5665 /* The GOMP_loop_ull_*start functions have additional boolean
5666 argument, true for < loops and false for > loops.
5667 In Fortran, the C bool type can be different from
5668 boolean_type_node. */
5669 bfn_decl
= builtin_decl_explicit (start_fn
);
5670 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5671 t5
= build_int_cst (c_bool_type
,
5672 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5675 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5676 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5677 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5680 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5681 6, t5
, t0
, t1
, t2
, t3
, t4
);
5684 if (TREE_TYPE (t
) != boolean_type_node
)
5685 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5686 t
, build_int_cst (TREE_TYPE (t
), 0));
5687 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5688 true, GSI_SAME_STMT
);
5689 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5691 /* Remove the GIMPLE_OMP_FOR statement. */
5692 gsi_remove (&gsi
, true);
5694 /* Iteration setup for sequential loop goes in L0_BB. */
5695 tree startvar
= fd
->loop
.v
;
5696 tree endvar
= NULL_TREE
;
5698 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5700 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5701 && gimple_omp_for_kind (inner_stmt
)
5702 == GF_OMP_FOR_KIND_SIMD
);
5703 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5704 OMP_CLAUSE__LOOPTEMP_
);
5705 gcc_assert (innerc
);
5706 startvar
= OMP_CLAUSE_DECL (innerc
);
5707 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5708 OMP_CLAUSE__LOOPTEMP_
);
5709 gcc_assert (innerc
);
5710 endvar
= OMP_CLAUSE_DECL (innerc
);
5713 gsi
= gsi_start_bb (l0_bb
);
5716 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5717 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5718 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5719 t
= fold_convert (TREE_TYPE (startvar
), t
);
5720 t
= force_gimple_operand_gsi (&gsi
, t
,
5722 && TREE_ADDRESSABLE (startvar
),
5723 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5724 stmt
= gimple_build_assign (startvar
, t
);
5725 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5729 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5730 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5731 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5732 t
= fold_convert (TREE_TYPE (startvar
), t
);
5733 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5734 false, GSI_CONTINUE_LINKING
);
5737 stmt
= gimple_build_assign (endvar
, iend
);
5738 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5739 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5740 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5742 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5744 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5746 if (fd
->collapse
> 1)
5747 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5751 /* Code to control the increment and predicate for the sequential
5752 loop goes in the CONT_BB. */
5753 gsi
= gsi_last_bb (cont_bb
);
5754 stmt
= gsi_stmt (gsi
);
5755 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5756 vmain
= gimple_omp_continue_control_use (stmt
);
5757 vback
= gimple_omp_continue_control_def (stmt
);
5759 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5761 if (POINTER_TYPE_P (type
))
5762 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5764 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5765 t
= force_gimple_operand_gsi (&gsi
, t
,
5767 && TREE_ADDRESSABLE (vback
),
5768 NULL_TREE
, true, GSI_SAME_STMT
);
5769 stmt
= gimple_build_assign (vback
, t
);
5770 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5772 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5773 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5775 stmt
= gimple_build_cond_empty (t
);
5776 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5779 /* Remove GIMPLE_OMP_CONTINUE. */
5780 gsi_remove (&gsi
, true);
5782 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5783 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5785 /* Emit code to get the next parallel iteration in L2_BB. */
5786 gsi
= gsi_start_bb (l2_bb
);
5788 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5789 build_fold_addr_expr (istart0
),
5790 build_fold_addr_expr (iend0
));
5791 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5792 false, GSI_CONTINUE_LINKING
);
5793 if (TREE_TYPE (t
) != boolean_type_node
)
5794 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5795 t
, build_int_cst (TREE_TYPE (t
), 0));
5796 stmt
= gimple_build_cond_empty (t
);
5797 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5800 /* Add the loop cleanup function. */
5801 gsi
= gsi_last_bb (exit_bb
);
5802 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5803 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5804 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5805 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5807 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5808 stmt
= gimple_build_call (t
, 0);
5809 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5810 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5811 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5812 gsi_remove (&gsi
, true);
5814 /* Connect the new blocks. */
5815 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5816 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5822 e
= find_edge (cont_bb
, l3_bb
);
5823 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5825 phis
= phi_nodes (l3_bb
);
5826 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5828 gimple phi
= gsi_stmt (gsi
);
5829 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5830 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5834 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5836 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5837 e
= find_edge (cont_bb
, l1_bb
);
5838 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5843 else if (fd
->collapse
> 1)
5846 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5849 e
->flags
= EDGE_TRUE_VALUE
;
5852 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5853 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5857 e
= find_edge (cont_bb
, l2_bb
);
5858 e
->flags
= EDGE_FALLTHRU
;
5860 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5862 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5863 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5864 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5865 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5866 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5867 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5868 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5869 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5871 struct loop
*outer_loop
= alloc_loop ();
5872 outer_loop
->header
= l0_bb
;
5873 outer_loop
->latch
= l2_bb
;
5874 add_loop (outer_loop
, l0_bb
->loop_father
);
5876 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5878 struct loop
*loop
= alloc_loop ();
5879 loop
->header
= l1_bb
;
5880 /* The loop may have multiple latches. */
5881 add_loop (loop
, outer_loop
);
5887 /* A subroutine of expand_omp_for. Generate code for a parallel
5888 loop with static schedule and no specified chunk size. Given
5891 for (V = N1; V cond N2; V += STEP) BODY;
5893 where COND is "<" or ">", we generate pseudocode
5895 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5900 if ((__typeof (V)) -1 > 0 && cond is >)
5901 n = -(adj + N2 - N1) / -STEP;
5903 n = (adj + N2 - N1) / STEP;
5906 if (threadid < tt) goto L3; else goto L4;
5911 s0 = q * threadid + tt;
5914 if (s0 >= e0) goto L2; else goto L0;
5920 if (V cond e) goto L1;
5925 expand_omp_for_static_nochunk (struct omp_region
*region
,
5926 struct omp_for_data
*fd
,
5929 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5930 tree type
, itype
, vmain
, vback
;
5931 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5932 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5934 gimple_stmt_iterator gsi
;
5937 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5938 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5939 bool broken_loop
= region
->cont
== NULL
;
5940 tree
*counts
= NULL
;
5943 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5944 if (POINTER_TYPE_P (type
))
5945 itype
= signed_type_for (type
);
5947 entry_bb
= region
->entry
;
5948 cont_bb
= region
->cont
;
5949 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5950 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5951 gcc_assert (broken_loop
5952 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5953 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5954 body_bb
= single_succ (seq_start_bb
);
5957 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5958 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5960 exit_bb
= region
->exit
;
5962 /* Iteration space partitioning goes in ENTRY_BB. */
5963 gsi
= gsi_last_bb (entry_bb
);
5964 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5966 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5968 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5969 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5972 if (fd
->collapse
> 1)
5974 int first_zero_iter
= -1;
5975 basic_block l2_dom_bb
= NULL
;
5977 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5978 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5979 fin_bb
, first_zero_iter
,
5983 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5984 t
= integer_one_node
;
5986 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5987 fold_convert (type
, fd
->loop
.n1
),
5988 fold_convert (type
, fd
->loop
.n2
));
5989 if (fd
->collapse
== 1
5990 && TYPE_UNSIGNED (type
)
5991 && (t
== NULL_TREE
|| !integer_onep (t
)))
5993 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5994 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5995 true, GSI_SAME_STMT
);
5996 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5997 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5998 true, GSI_SAME_STMT
);
5999 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6000 NULL_TREE
, NULL_TREE
);
6001 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6002 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6003 expand_omp_regimplify_p
, NULL
, NULL
)
6004 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6005 expand_omp_regimplify_p
, NULL
, NULL
))
6007 gsi
= gsi_for_stmt (stmt
);
6008 gimple_regimplify_operands (stmt
, &gsi
);
6010 ep
= split_block (entry_bb
, stmt
);
6011 ep
->flags
= EDGE_TRUE_VALUE
;
6012 entry_bb
= ep
->dest
;
6013 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6014 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6015 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6016 if (gimple_in_ssa_p (cfun
))
6018 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6019 for (gsi
= gsi_start_phis (fin_bb
);
6020 !gsi_end_p (gsi
); gsi_next (&gsi
))
6022 gimple phi
= gsi_stmt (gsi
);
6023 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6024 ep
, UNKNOWN_LOCATION
);
6027 gsi
= gsi_last_bb (entry_bb
);
6030 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6031 t
= fold_convert (itype
, t
);
6032 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6033 true, GSI_SAME_STMT
);
6035 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6036 t
= fold_convert (itype
, t
);
6037 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6038 true, GSI_SAME_STMT
);
6042 step
= fd
->loop
.step
;
6043 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6045 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6046 OMP_CLAUSE__LOOPTEMP_
);
6047 gcc_assert (innerc
);
6048 n1
= OMP_CLAUSE_DECL (innerc
);
6049 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6050 OMP_CLAUSE__LOOPTEMP_
);
6051 gcc_assert (innerc
);
6052 n2
= OMP_CLAUSE_DECL (innerc
);
6054 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6055 true, NULL_TREE
, true, GSI_SAME_STMT
);
6056 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6057 true, NULL_TREE
, true, GSI_SAME_STMT
);
6058 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6059 true, NULL_TREE
, true, GSI_SAME_STMT
);
6061 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6062 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6063 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6064 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6065 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6066 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6067 fold_build1 (NEGATE_EXPR
, itype
, t
),
6068 fold_build1 (NEGATE_EXPR
, itype
, step
));
6070 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6071 t
= fold_convert (itype
, t
);
6072 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6074 q
= create_tmp_reg (itype
, "q");
6075 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6076 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6077 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6079 tt
= create_tmp_reg (itype
, "tt");
6080 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6081 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6082 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6084 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6085 stmt
= gimple_build_cond_empty (t
);
6086 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6088 second_bb
= split_block (entry_bb
, stmt
)->dest
;
6089 gsi
= gsi_last_bb (second_bb
);
6090 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6092 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6094 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6095 build_int_cst (itype
, 1));
6096 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6098 third_bb
= split_block (second_bb
, stmt
)->dest
;
6099 gsi
= gsi_last_bb (third_bb
);
6100 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6102 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6103 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6104 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6106 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6107 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6109 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6110 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6112 /* Remove the GIMPLE_OMP_FOR statement. */
6113 gsi_remove (&gsi
, true);
6115 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6116 gsi
= gsi_start_bb (seq_start_bb
);
6118 tree startvar
= fd
->loop
.v
;
6119 tree endvar
= NULL_TREE
;
6121 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6123 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6124 ? gimple_omp_parallel_clauses (inner_stmt
)
6125 : gimple_omp_for_clauses (inner_stmt
);
6126 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6127 gcc_assert (innerc
);
6128 startvar
= OMP_CLAUSE_DECL (innerc
);
6129 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6130 OMP_CLAUSE__LOOPTEMP_
);
6131 gcc_assert (innerc
);
6132 endvar
= OMP_CLAUSE_DECL (innerc
);
6134 t
= fold_convert (itype
, s0
);
6135 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6136 if (POINTER_TYPE_P (type
))
6137 t
= fold_build_pointer_plus (n1
, t
);
6139 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6140 t
= fold_convert (TREE_TYPE (startvar
), t
);
6141 t
= force_gimple_operand_gsi (&gsi
, t
,
6143 && TREE_ADDRESSABLE (startvar
),
6144 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6145 stmt
= gimple_build_assign (startvar
, t
);
6146 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6148 t
= fold_convert (itype
, e0
);
6149 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6150 if (POINTER_TYPE_P (type
))
6151 t
= fold_build_pointer_plus (n1
, t
);
6153 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6154 t
= fold_convert (TREE_TYPE (startvar
), t
);
6155 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6156 false, GSI_CONTINUE_LINKING
);
6159 stmt
= gimple_build_assign (endvar
, e
);
6160 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6161 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6162 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6164 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6166 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6168 if (fd
->collapse
> 1)
6169 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6173 /* The code controlling the sequential loop replaces the
6174 GIMPLE_OMP_CONTINUE. */
6175 gsi
= gsi_last_bb (cont_bb
);
6176 stmt
= gsi_stmt (gsi
);
6177 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6178 vmain
= gimple_omp_continue_control_use (stmt
);
6179 vback
= gimple_omp_continue_control_def (stmt
);
6181 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6183 if (POINTER_TYPE_P (type
))
6184 t
= fold_build_pointer_plus (vmain
, step
);
6186 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6187 t
= force_gimple_operand_gsi (&gsi
, t
,
6189 && TREE_ADDRESSABLE (vback
),
6190 NULL_TREE
, true, GSI_SAME_STMT
);
6191 stmt
= gimple_build_assign (vback
, t
);
6192 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6194 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6195 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6197 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6200 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6201 gsi_remove (&gsi
, true);
6203 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6204 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6207 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6208 gsi
= gsi_last_bb (exit_bb
);
6209 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6211 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6212 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6214 gsi_remove (&gsi
, true);
6216 /* Connect all the blocks. */
6217 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6218 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6219 ep
= find_edge (entry_bb
, second_bb
);
6220 ep
->flags
= EDGE_TRUE_VALUE
;
6221 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6222 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6223 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6227 ep
= find_edge (cont_bb
, body_bb
);
6228 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6233 else if (fd
->collapse
> 1)
6236 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6239 ep
->flags
= EDGE_TRUE_VALUE
;
6240 find_edge (cont_bb
, fin_bb
)->flags
6241 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6244 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6245 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6246 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6248 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6249 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6250 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6251 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6253 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6255 struct loop
*loop
= alloc_loop ();
6256 loop
->header
= body_bb
;
6257 if (collapse_bb
== NULL
)
6258 loop
->latch
= cont_bb
;
6259 add_loop (loop
, body_bb
->loop_father
);
6264 /* A subroutine of expand_omp_for. Generate code for a parallel
6265 loop with static schedule and a specified chunk size. Given
6268 for (V = N1; V cond N2; V += STEP) BODY;
6270 where COND is "<" or ">", we generate pseudocode
6272 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6277 if ((__typeof (V)) -1 > 0 && cond is >)
6278 n = -(adj + N2 - N1) / -STEP;
6280 n = (adj + N2 - N1) / STEP;
6282 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6283 here so that V is defined
6284 if the loop is not entered
6286 s0 = (trip * nthreads + threadid) * CHUNK;
6287 e0 = min(s0 + CHUNK, n);
6288 if (s0 < n) goto L1; else goto L4;
6295 if (V cond e) goto L2; else goto L3;
6303 expand_omp_for_static_chunk (struct omp_region
*region
,
6304 struct omp_for_data
*fd
, gimple inner_stmt
)
6306 tree n
, s0
, e0
, e
, t
;
6307 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6308 tree type
, itype
, v_main
, v_back
, v_extra
;
6309 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6310 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6311 gimple_stmt_iterator si
;
6314 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6315 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6316 bool broken_loop
= region
->cont
== NULL
;
6317 tree
*counts
= NULL
;
6320 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6321 if (POINTER_TYPE_P (type
))
6322 itype
= signed_type_for (type
);
6324 entry_bb
= region
->entry
;
6325 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6327 iter_part_bb
= se
->dest
;
6328 cont_bb
= region
->cont
;
6329 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6330 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6331 gcc_assert (broken_loop
6332 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6333 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6334 body_bb
= single_succ (seq_start_bb
);
6337 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6338 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6339 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6341 exit_bb
= region
->exit
;
6343 /* Trip and adjustment setup goes in ENTRY_BB. */
6344 si
= gsi_last_bb (entry_bb
);
6345 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6347 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6349 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6350 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6353 if (fd
->collapse
> 1)
6355 int first_zero_iter
= -1;
6356 basic_block l2_dom_bb
= NULL
;
6358 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6359 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6360 fin_bb
, first_zero_iter
,
6364 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6365 t
= integer_one_node
;
6367 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6368 fold_convert (type
, fd
->loop
.n1
),
6369 fold_convert (type
, fd
->loop
.n2
));
6370 if (fd
->collapse
== 1
6371 && TYPE_UNSIGNED (type
)
6372 && (t
== NULL_TREE
|| !integer_onep (t
)))
6374 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6375 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6376 true, GSI_SAME_STMT
);
6377 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6378 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6379 true, GSI_SAME_STMT
);
6380 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6381 NULL_TREE
, NULL_TREE
);
6382 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6383 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6384 expand_omp_regimplify_p
, NULL
, NULL
)
6385 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6386 expand_omp_regimplify_p
, NULL
, NULL
))
6388 si
= gsi_for_stmt (stmt
);
6389 gimple_regimplify_operands (stmt
, &si
);
6391 se
= split_block (entry_bb
, stmt
);
6392 se
->flags
= EDGE_TRUE_VALUE
;
6393 entry_bb
= se
->dest
;
6394 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6395 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6396 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6397 if (gimple_in_ssa_p (cfun
))
6399 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6400 for (si
= gsi_start_phis (fin_bb
);
6401 !gsi_end_p (si
); gsi_next (&si
))
6403 gimple phi
= gsi_stmt (si
);
6404 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6405 se
, UNKNOWN_LOCATION
);
6408 si
= gsi_last_bb (entry_bb
);
6411 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6412 t
= fold_convert (itype
, t
);
6413 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6414 true, GSI_SAME_STMT
);
6416 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6417 t
= fold_convert (itype
, t
);
6418 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6419 true, GSI_SAME_STMT
);
6423 step
= fd
->loop
.step
;
6424 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6426 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6427 OMP_CLAUSE__LOOPTEMP_
);
6428 gcc_assert (innerc
);
6429 n1
= OMP_CLAUSE_DECL (innerc
);
6430 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6431 OMP_CLAUSE__LOOPTEMP_
);
6432 gcc_assert (innerc
);
6433 n2
= OMP_CLAUSE_DECL (innerc
);
6435 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6436 true, NULL_TREE
, true, GSI_SAME_STMT
);
6437 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6438 true, NULL_TREE
, true, GSI_SAME_STMT
);
6439 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6440 true, NULL_TREE
, true, GSI_SAME_STMT
);
6442 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6443 true, NULL_TREE
, true, GSI_SAME_STMT
);
6445 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6446 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6447 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6448 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6449 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6450 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6451 fold_build1 (NEGATE_EXPR
, itype
, t
),
6452 fold_build1 (NEGATE_EXPR
, itype
, step
));
6454 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6455 t
= fold_convert (itype
, t
);
6456 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6457 true, GSI_SAME_STMT
);
6459 trip_var
= create_tmp_reg (itype
, ".trip");
6460 if (gimple_in_ssa_p (cfun
))
6462 trip_init
= make_ssa_name (trip_var
, NULL
);
6463 trip_main
= make_ssa_name (trip_var
, NULL
);
6464 trip_back
= make_ssa_name (trip_var
, NULL
);
6468 trip_init
= trip_var
;
6469 trip_main
= trip_var
;
6470 trip_back
= trip_var
;
6473 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6474 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6476 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6477 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6478 if (POINTER_TYPE_P (type
))
6479 t
= fold_build_pointer_plus (n1
, t
);
6481 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6482 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6483 true, GSI_SAME_STMT
);
6485 /* Remove the GIMPLE_OMP_FOR. */
6486 gsi_remove (&si
, true);
6488 /* Iteration space partitioning goes in ITER_PART_BB. */
6489 si
= gsi_last_bb (iter_part_bb
);
6491 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6492 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6493 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6494 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6495 false, GSI_CONTINUE_LINKING
);
6497 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6498 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6499 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6500 false, GSI_CONTINUE_LINKING
);
6502 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6503 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6505 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6506 si
= gsi_start_bb (seq_start_bb
);
6508 tree startvar
= fd
->loop
.v
;
6509 tree endvar
= NULL_TREE
;
6511 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6513 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6514 ? gimple_omp_parallel_clauses (inner_stmt
)
6515 : gimple_omp_for_clauses (inner_stmt
);
6516 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6517 gcc_assert (innerc
);
6518 startvar
= OMP_CLAUSE_DECL (innerc
);
6519 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6520 OMP_CLAUSE__LOOPTEMP_
);
6521 gcc_assert (innerc
);
6522 endvar
= OMP_CLAUSE_DECL (innerc
);
6525 t
= fold_convert (itype
, s0
);
6526 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6527 if (POINTER_TYPE_P (type
))
6528 t
= fold_build_pointer_plus (n1
, t
);
6530 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6531 t
= fold_convert (TREE_TYPE (startvar
), t
);
6532 t
= force_gimple_operand_gsi (&si
, t
,
6534 && TREE_ADDRESSABLE (startvar
),
6535 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6536 stmt
= gimple_build_assign (startvar
, t
);
6537 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6539 t
= fold_convert (itype
, e0
);
6540 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6541 if (POINTER_TYPE_P (type
))
6542 t
= fold_build_pointer_plus (n1
, t
);
6544 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6545 t
= fold_convert (TREE_TYPE (startvar
), t
);
6546 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6547 false, GSI_CONTINUE_LINKING
);
6550 stmt
= gimple_build_assign (endvar
, e
);
6551 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6552 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6553 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6555 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6557 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6559 if (fd
->collapse
> 1)
6560 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6564 /* The code controlling the sequential loop goes in CONT_BB,
6565 replacing the GIMPLE_OMP_CONTINUE. */
6566 si
= gsi_last_bb (cont_bb
);
6567 stmt
= gsi_stmt (si
);
6568 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6569 v_main
= gimple_omp_continue_control_use (stmt
);
6570 v_back
= gimple_omp_continue_control_def (stmt
);
6572 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6574 if (POINTER_TYPE_P (type
))
6575 t
= fold_build_pointer_plus (v_main
, step
);
6577 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6578 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6579 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6580 true, GSI_SAME_STMT
);
6581 stmt
= gimple_build_assign (v_back
, t
);
6582 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6584 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6585 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6587 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6590 /* Remove GIMPLE_OMP_CONTINUE. */
6591 gsi_remove (&si
, true);
6593 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6594 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6596 /* Trip update code goes into TRIP_UPDATE_BB. */
6597 si
= gsi_start_bb (trip_update_bb
);
6599 t
= build_int_cst (itype
, 1);
6600 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6601 stmt
= gimple_build_assign (trip_back
, t
);
6602 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6605 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6606 si
= gsi_last_bb (exit_bb
);
6607 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6609 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6610 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6612 gsi_remove (&si
, true);
6614 /* Connect the new blocks. */
6615 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6616 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6620 se
= find_edge (cont_bb
, body_bb
);
6621 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6626 else if (fd
->collapse
> 1)
6629 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6632 se
->flags
= EDGE_TRUE_VALUE
;
6633 find_edge (cont_bb
, trip_update_bb
)->flags
6634 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6636 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6639 if (gimple_in_ssa_p (cfun
))
6641 gimple_stmt_iterator psi
;
6644 edge_var_map_vector
*head
;
6648 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6650 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6651 remove arguments of the phi nodes in fin_bb. We need to create
6652 appropriate phi nodes in iter_part_bb instead. */
6653 se
= single_pred_edge (fin_bb
);
6654 re
= single_succ_edge (trip_update_bb
);
6655 head
= redirect_edge_var_map_vector (re
);
6656 ene
= single_succ_edge (entry_bb
);
6658 psi
= gsi_start_phis (fin_bb
);
6659 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6660 gsi_next (&psi
), ++i
)
6663 source_location locus
;
6665 phi
= gsi_stmt (psi
);
6666 t
= gimple_phi_result (phi
);
6667 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6668 nphi
= create_phi_node (t
, iter_part_bb
);
6670 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6671 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6673 /* A special case -- fd->loop.v is not yet computed in
6674 iter_part_bb, we need to use v_extra instead. */
6675 if (t
== fd
->loop
.v
)
6677 add_phi_arg (nphi
, t
, ene
, locus
);
6678 locus
= redirect_edge_var_map_location (vm
);
6679 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6681 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6682 redirect_edge_var_map_clear (re
);
6685 psi
= gsi_start_phis (fin_bb
);
6686 if (gsi_end_p (psi
))
6688 remove_phi_node (&psi
, false);
6691 /* Make phi node for trip. */
6692 phi
= create_phi_node (trip_main
, iter_part_bb
);
6693 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6695 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6700 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6701 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6702 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6703 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6704 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6705 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6706 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6707 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6708 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6712 struct loop
*trip_loop
= alloc_loop ();
6713 trip_loop
->header
= iter_part_bb
;
6714 trip_loop
->latch
= trip_update_bb
;
6715 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6717 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6719 struct loop
*loop
= alloc_loop ();
6720 loop
->header
= body_bb
;
6721 if (collapse_bb
== NULL
)
6722 loop
->latch
= cont_bb
;
6723 add_loop (loop
, trip_loop
);
6729 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6730 loop. Given parameters:
6732 for (V = N1; V cond N2; V += STEP) BODY;
6734 where COND is "<" or ">", we generate pseudocode
6742 if (V cond N2) goto L0; else goto L2;
6745 For collapsed loops, given parameters:
6747 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6748 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6749 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6752 we generate pseudocode
6758 count3 = (adj + N32 - N31) / STEP3;
6763 count2 = (adj + N22 - N21) / STEP2;
6768 count1 = (adj + N12 - N11) / STEP1;
6769 count = count1 * count2 * count3;
6779 V2 += (V3 cond3 N32) ? 0 : STEP2;
6780 V3 = (V3 cond3 N32) ? V3 : N31;
6781 V1 += (V2 cond2 N22) ? 0 : STEP1;
6782 V2 = (V2 cond2 N22) ? V2 : N21;
6784 if (V < count) goto L0; else goto L2;
6790 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6793 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6794 gimple_stmt_iterator gsi
;
6796 bool broken_loop
= region
->cont
== NULL
;
6798 tree
*counts
= NULL
;
6800 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6801 OMP_CLAUSE_SAFELEN
);
6802 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6803 OMP_CLAUSE__SIMDUID_
);
6806 type
= TREE_TYPE (fd
->loop
.v
);
6807 entry_bb
= region
->entry
;
6808 cont_bb
= region
->cont
;
6809 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6810 gcc_assert (broken_loop
6811 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6812 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6815 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6816 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6817 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6818 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6822 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6823 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6824 l2_bb
= single_succ (l1_bb
);
6826 exit_bb
= region
->exit
;
6829 gsi
= gsi_last_bb (entry_bb
);
6831 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6832 /* Not needed in SSA form right now. */
6833 gcc_assert (!gimple_in_ssa_p (cfun
));
6834 if (fd
->collapse
> 1)
6836 int first_zero_iter
= -1;
6837 basic_block zero_iter_bb
= l2_bb
;
6839 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6840 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6841 zero_iter_bb
, first_zero_iter
,
6844 if (l2_dom_bb
== NULL
)
6849 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6851 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6852 OMP_CLAUSE__LOOPTEMP_
);
6853 gcc_assert (innerc
);
6854 n1
= OMP_CLAUSE_DECL (innerc
);
6855 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6856 OMP_CLAUSE__LOOPTEMP_
);
6857 gcc_assert (innerc
);
6858 n2
= OMP_CLAUSE_DECL (innerc
);
6859 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6860 fold_convert (type
, n1
));
6861 if (fd
->collapse
> 1)
6864 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6870 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6871 fold_convert (type
, fd
->loop
.n1
));
6872 if (fd
->collapse
> 1)
6873 for (i
= 0; i
< fd
->collapse
; i
++)
6875 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6876 if (POINTER_TYPE_P (itype
))
6877 itype
= signed_type_for (itype
);
6878 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6879 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6883 /* Remove the GIMPLE_OMP_FOR statement. */
6884 gsi_remove (&gsi
, true);
6888 /* Code to control the increment goes in the CONT_BB. */
6889 gsi
= gsi_last_bb (cont_bb
);
6890 stmt
= gsi_stmt (gsi
);
6891 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6893 if (POINTER_TYPE_P (type
))
6894 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6896 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6897 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6899 if (fd
->collapse
> 1)
6901 i
= fd
->collapse
- 1;
6902 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6904 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6905 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6909 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6911 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6914 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6916 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6918 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6919 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6920 if (POINTER_TYPE_P (itype2
))
6921 itype2
= signed_type_for (itype2
);
6922 t
= build3 (COND_EXPR
, itype2
,
6923 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6925 fold_convert (itype
, fd
->loops
[i
].n2
)),
6926 build_int_cst (itype2
, 0),
6927 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6928 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6929 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6931 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6932 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6934 t
= build3 (COND_EXPR
, itype
,
6935 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6937 fold_convert (itype
, fd
->loops
[i
].n2
)),
6939 fold_convert (itype
, fd
->loops
[i
].n1
));
6940 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6944 /* Remove GIMPLE_OMP_CONTINUE. */
6945 gsi_remove (&gsi
, true);
6948 /* Emit the condition in L1_BB. */
6949 gsi
= gsi_start_bb (l1_bb
);
6951 t
= fold_convert (type
, n2
);
6952 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6953 false, GSI_CONTINUE_LINKING
);
6954 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6955 stmt
= gimple_build_cond_empty (t
);
6956 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6957 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6959 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6962 gsi
= gsi_for_stmt (stmt
);
6963 gimple_regimplify_operands (stmt
, &gsi
);
6966 /* Remove GIMPLE_OMP_RETURN. */
6967 gsi
= gsi_last_bb (exit_bb
);
6968 gsi_remove (&gsi
, true);
6970 /* Connect the new blocks. */
6971 remove_edge (FALLTHRU_EDGE (entry_bb
));
6975 remove_edge (BRANCH_EDGE (entry_bb
));
6976 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6978 e
= BRANCH_EDGE (l1_bb
);
6979 ne
= FALLTHRU_EDGE (l1_bb
);
6980 e
->flags
= EDGE_TRUE_VALUE
;
6984 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6986 ne
= single_succ_edge (l1_bb
);
6987 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6990 ne
->flags
= EDGE_FALSE_VALUE
;
6991 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6992 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6994 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6995 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6996 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7000 struct loop
*loop
= alloc_loop ();
7001 loop
->header
= l1_bb
;
7002 loop
->latch
= cont_bb
;
7003 add_loop (loop
, l1_bb
->loop_father
);
7004 if (safelen
== NULL_TREE
)
7005 loop
->safelen
= INT_MAX
;
7008 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7009 if (TREE_CODE (safelen
) != INTEGER_CST
)
7011 else if (!tree_fits_uhwi_p (safelen
)
7012 || tree_to_uhwi (safelen
) > INT_MAX
)
7013 loop
->safelen
= INT_MAX
;
7015 loop
->safelen
= tree_to_uhwi (safelen
);
7016 if (loop
->safelen
== 1)
7021 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7022 cfun
->has_simduid_loops
= true;
7024 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7026 if ((flag_tree_loop_vectorize
7027 || (!global_options_set
.x_flag_tree_loop_vectorize
7028 && !global_options_set
.x_flag_tree_vectorize
))
7029 && flag_tree_loop_optimize
7030 && loop
->safelen
> 1)
7032 loop
->force_vect
= true;
7033 cfun
->has_force_vect_loops
= true;
7039 /* Expand the OpenMP loop defined by REGION. */
7042 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7044 struct omp_for_data fd
;
7045 struct omp_for_data_loop
*loops
;
7048 = (struct omp_for_data_loop
*)
7049 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7050 * sizeof (struct omp_for_data_loop
));
7051 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7052 region
->sched_kind
= fd
.sched_kind
;
7054 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7055 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7056 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7059 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7060 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7061 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7064 /* If there isn't a continue then this is a degerate case where
7065 the introduction of abnormal edges during lowering will prevent
7066 original loops from being detected. Fix that up. */
7067 loops_state_set (LOOPS_NEED_FIXUP
);
7069 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_KIND_SIMD
)
7070 expand_omp_simd (region
, &fd
);
7071 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7072 && !fd
.have_ordered
)
7074 if (fd
.chunk_size
== NULL
)
7075 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7077 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7081 int fn_index
, start_ix
, next_ix
;
7083 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7084 == GF_OMP_FOR_KIND_FOR
);
7085 if (fd
.chunk_size
== NULL
7086 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7087 fd
.chunk_size
= integer_zero_node
;
7088 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7089 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7090 ? 3 : fd
.sched_kind
;
7091 fn_index
+= fd
.have_ordered
* 4;
7092 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7093 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7094 if (fd
.iter_type
== long_long_unsigned_type_node
)
7096 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7097 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7098 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7099 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7101 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7102 (enum built_in_function
) next_ix
, inner_stmt
);
7105 if (gimple_in_ssa_p (cfun
))
7106 update_ssa (TODO_update_ssa_only_virtuals
);
7110 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7112 v = GOMP_sections_start (n);
7129 v = GOMP_sections_next ();
7134 If this is a combined parallel sections, replace the call to
7135 GOMP_sections_start with call to GOMP_sections_next. */
7138 expand_omp_sections (struct omp_region
*region
)
7140 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7142 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7143 gimple_stmt_iterator si
, switch_si
;
7144 gimple sections_stmt
, stmt
, cont
;
7147 struct omp_region
*inner
;
7149 bool exit_reachable
= region
->cont
!= NULL
;
7151 gcc_assert (region
->exit
!= NULL
);
7152 entry_bb
= region
->entry
;
7153 l0_bb
= single_succ (entry_bb
);
7154 l1_bb
= region
->cont
;
7155 l2_bb
= region
->exit
;
7156 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7157 l2
= gimple_block_label (l2_bb
);
7160 /* This can happen if there are reductions. */
7161 len
= EDGE_COUNT (l0_bb
->succs
);
7162 gcc_assert (len
> 0);
7163 e
= EDGE_SUCC (l0_bb
, len
- 1);
7164 si
= gsi_last_bb (e
->dest
);
7167 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7168 l2
= gimple_block_label (e
->dest
);
7170 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7172 si
= gsi_last_bb (e
->dest
);
7174 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7176 l2
= gimple_block_label (e
->dest
);
7182 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7184 default_bb
= create_empty_bb (l0_bb
);
7186 /* We will build a switch() with enough cases for all the
7187 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7188 and a default case to abort if something goes wrong. */
7189 len
= EDGE_COUNT (l0_bb
->succs
);
7191 /* Use vec::quick_push on label_vec throughout, since we know the size
7193 auto_vec
<tree
> label_vec (len
);
7195 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7196 GIMPLE_OMP_SECTIONS statement. */
7197 si
= gsi_last_bb (entry_bb
);
7198 sections_stmt
= gsi_stmt (si
);
7199 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7200 vin
= gimple_omp_sections_control (sections_stmt
);
7201 if (!is_combined_parallel (region
))
7203 /* If we are not inside a combined parallel+sections region,
7204 call GOMP_sections_start. */
7205 t
= build_int_cst (unsigned_type_node
, len
- 1);
7206 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7207 stmt
= gimple_build_call (u
, 1, t
);
7211 /* Otherwise, call GOMP_sections_next. */
7212 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7213 stmt
= gimple_build_call (u
, 0);
7215 gimple_call_set_lhs (stmt
, vin
);
7216 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7217 gsi_remove (&si
, true);
7219 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7221 switch_si
= gsi_last_bb (l0_bb
);
7222 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7225 cont
= last_stmt (l1_bb
);
7226 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7227 vmain
= gimple_omp_continue_control_use (cont
);
7228 vnext
= gimple_omp_continue_control_def (cont
);
7236 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7237 label_vec
.quick_push (t
);
7240 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7241 for (inner
= region
->inner
, casei
= 1;
7243 inner
= inner
->next
, i
++, casei
++)
7245 basic_block s_entry_bb
, s_exit_bb
;
7247 /* Skip optional reduction region. */
7248 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7255 s_entry_bb
= inner
->entry
;
7256 s_exit_bb
= inner
->exit
;
7258 t
= gimple_block_label (s_entry_bb
);
7259 u
= build_int_cst (unsigned_type_node
, casei
);
7260 u
= build_case_label (u
, NULL
, t
);
7261 label_vec
.quick_push (u
);
7263 si
= gsi_last_bb (s_entry_bb
);
7264 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7265 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7266 gsi_remove (&si
, true);
7267 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7269 if (s_exit_bb
== NULL
)
7272 si
= gsi_last_bb (s_exit_bb
);
7273 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7274 gsi_remove (&si
, true);
7276 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7279 /* Error handling code goes in DEFAULT_BB. */
7280 t
= gimple_block_label (default_bb
);
7281 u
= build_case_label (NULL
, NULL
, t
);
7282 make_edge (l0_bb
, default_bb
, 0);
7284 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7286 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7287 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7288 gsi_remove (&switch_si
, true);
7290 si
= gsi_start_bb (default_bb
);
7291 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7292 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7298 /* Code to get the next section goes in L1_BB. */
7299 si
= gsi_last_bb (l1_bb
);
7300 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7302 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7303 stmt
= gimple_build_call (bfn_decl
, 0);
7304 gimple_call_set_lhs (stmt
, vnext
);
7305 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7306 gsi_remove (&si
, true);
7308 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7311 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7312 si
= gsi_last_bb (l2_bb
);
7313 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7314 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7315 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7316 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7318 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7319 stmt
= gimple_build_call (t
, 0);
7320 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7321 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7322 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7323 gsi_remove (&si
, true);
7325 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7329 /* Expand code for an OpenMP single directive. We've already expanded
7330 much of the code, here we simply place the GOMP_barrier call. */
7333 expand_omp_single (struct omp_region
*region
)
7335 basic_block entry_bb
, exit_bb
;
7336 gimple_stmt_iterator si
;
7338 entry_bb
= region
->entry
;
7339 exit_bb
= region
->exit
;
7341 si
= gsi_last_bb (entry_bb
);
7342 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7343 gsi_remove (&si
, true);
7344 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7346 si
= gsi_last_bb (exit_bb
);
7347 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7349 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7350 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7352 gsi_remove (&si
, true);
7353 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7357 /* Generic expansion for OpenMP synchronization directives: master,
7358 ordered and critical. All we need to do here is remove the entry
7359 and exit markers for REGION. */
7362 expand_omp_synch (struct omp_region
*region
)
7364 basic_block entry_bb
, exit_bb
;
7365 gimple_stmt_iterator si
;
7367 entry_bb
= region
->entry
;
7368 exit_bb
= region
->exit
;
7370 si
= gsi_last_bb (entry_bb
);
7371 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7372 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7373 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7374 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7375 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7376 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7377 gsi_remove (&si
, true);
7378 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7382 si
= gsi_last_bb (exit_bb
);
7383 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7384 gsi_remove (&si
, true);
7385 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7389 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7390 operation as a normal volatile load. */
7393 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7394 tree loaded_val
, int index
)
7396 enum built_in_function tmpbase
;
7397 gimple_stmt_iterator gsi
;
7398 basic_block store_bb
;
7401 tree decl
, call
, type
, itype
;
7403 gsi
= gsi_last_bb (load_bb
);
7404 stmt
= gsi_stmt (gsi
);
7405 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7406 loc
= gimple_location (stmt
);
7408 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7409 is smaller than word size, then expand_atomic_load assumes that the load
7410 is atomic. We could avoid the builtin entirely in this case. */
7412 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7413 decl
= builtin_decl_explicit (tmpbase
);
7414 if (decl
== NULL_TREE
)
7417 type
= TREE_TYPE (loaded_val
);
7418 itype
= TREE_TYPE (TREE_TYPE (decl
));
7420 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7421 build_int_cst (NULL
,
7422 gimple_omp_atomic_seq_cst_p (stmt
)
7424 : MEMMODEL_RELAXED
));
7425 if (!useless_type_conversion_p (type
, itype
))
7426 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7427 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7429 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7430 gsi_remove (&gsi
, true);
7432 store_bb
= single_succ (load_bb
);
7433 gsi
= gsi_last_bb (store_bb
);
7434 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7435 gsi_remove (&gsi
, true);
7437 if (gimple_in_ssa_p (cfun
))
7438 update_ssa (TODO_update_ssa_no_phi
);
7443 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7444 operation as a normal volatile store. */
7447 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7448 tree loaded_val
, tree stored_val
, int index
)
7450 enum built_in_function tmpbase
;
7451 gimple_stmt_iterator gsi
;
7452 basic_block store_bb
= single_succ (load_bb
);
7455 tree decl
, call
, type
, itype
;
7456 enum machine_mode imode
;
7459 gsi
= gsi_last_bb (load_bb
);
7460 stmt
= gsi_stmt (gsi
);
7461 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7463 /* If the load value is needed, then this isn't a store but an exchange. */
7464 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7466 gsi
= gsi_last_bb (store_bb
);
7467 stmt
= gsi_stmt (gsi
);
7468 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7469 loc
= gimple_location (stmt
);
7471 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7472 is smaller than word size, then expand_atomic_store assumes that the store
7473 is atomic. We could avoid the builtin entirely in this case. */
7475 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7476 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7477 decl
= builtin_decl_explicit (tmpbase
);
7478 if (decl
== NULL_TREE
)
7481 type
= TREE_TYPE (stored_val
);
7483 /* Dig out the type of the function's second argument. */
7484 itype
= TREE_TYPE (decl
);
7485 itype
= TYPE_ARG_TYPES (itype
);
7486 itype
= TREE_CHAIN (itype
);
7487 itype
= TREE_VALUE (itype
);
7488 imode
= TYPE_MODE (itype
);
7490 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7493 if (!useless_type_conversion_p (itype
, type
))
7494 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7495 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7496 build_int_cst (NULL
,
7497 gimple_omp_atomic_seq_cst_p (stmt
)
7499 : MEMMODEL_RELAXED
));
7502 if (!useless_type_conversion_p (type
, itype
))
7503 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7504 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7507 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7508 gsi_remove (&gsi
, true);
7510 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7511 gsi
= gsi_last_bb (load_bb
);
7512 gsi_remove (&gsi
, true);
7514 if (gimple_in_ssa_p (cfun
))
7515 update_ssa (TODO_update_ssa_no_phi
);
7520 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7521 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7522 size of the data type, and thus usable to find the index of the builtin
7523 decl. Returns false if the expression is not of the proper form. */
7526 expand_omp_atomic_fetch_op (basic_block load_bb
,
7527 tree addr
, tree loaded_val
,
7528 tree stored_val
, int index
)
7530 enum built_in_function oldbase
, newbase
, tmpbase
;
7531 tree decl
, itype
, call
;
7533 basic_block store_bb
= single_succ (load_bb
);
7534 gimple_stmt_iterator gsi
;
7537 enum tree_code code
;
7538 bool need_old
, need_new
;
7539 enum machine_mode imode
;
7542 /* We expect to find the following sequences:
7545 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7548 val = tmp OP something; (or: something OP tmp)
7549 GIMPLE_OMP_STORE (val)
7551 ???FIXME: Allow a more flexible sequence.
7552 Perhaps use data flow to pick the statements.
7556 gsi
= gsi_after_labels (store_bb
);
7557 stmt
= gsi_stmt (gsi
);
7558 loc
= gimple_location (stmt
);
7559 if (!is_gimple_assign (stmt
))
7562 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7564 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7565 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7566 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7567 gcc_checking_assert (!need_old
|| !need_new
);
7569 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7572 /* Check for one of the supported fetch-op operations. */
7573 code
= gimple_assign_rhs_code (stmt
);
7577 case POINTER_PLUS_EXPR
:
7578 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7579 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7582 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7583 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7586 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7587 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7590 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7591 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7594 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7595 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7601 /* Make sure the expression is of the proper form. */
7602 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7603 rhs
= gimple_assign_rhs2 (stmt
);
7604 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7605 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7606 rhs
= gimple_assign_rhs1 (stmt
);
7610 tmpbase
= ((enum built_in_function
)
7611 ((need_new
? newbase
: oldbase
) + index
+ 1));
7612 decl
= builtin_decl_explicit (tmpbase
);
7613 if (decl
== NULL_TREE
)
7615 itype
= TREE_TYPE (TREE_TYPE (decl
));
7616 imode
= TYPE_MODE (itype
);
7618 /* We could test all of the various optabs involved, but the fact of the
7619 matter is that (with the exception of i486 vs i586 and xadd) all targets
7620 that support any atomic operaton optab also implements compare-and-swap.
7621 Let optabs.c take care of expanding any compare-and-swap loop. */
7622 if (!can_compare_and_swap_p (imode
, true))
7625 gsi
= gsi_last_bb (load_bb
);
7626 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7628 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7629 It only requires that the operation happen atomically. Thus we can
7630 use the RELAXED memory model. */
7631 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7632 fold_convert_loc (loc
, itype
, rhs
),
7633 build_int_cst (NULL
,
7634 seq_cst
? MEMMODEL_SEQ_CST
7635 : MEMMODEL_RELAXED
));
7637 if (need_old
|| need_new
)
7639 lhs
= need_old
? loaded_val
: stored_val
;
7640 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7641 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7644 call
= fold_convert_loc (loc
, void_type_node
, call
);
7645 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7646 gsi_remove (&gsi
, true);
7648 gsi
= gsi_last_bb (store_bb
);
7649 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7650 gsi_remove (&gsi
, true);
7651 gsi
= gsi_last_bb (store_bb
);
7652 gsi_remove (&gsi
, true);
7654 if (gimple_in_ssa_p (cfun
))
7655 update_ssa (TODO_update_ssa_no_phi
);
7660 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7664 newval = rhs; // with oldval replacing *addr in rhs
7665 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7666 if (oldval != newval)
7669 INDEX is log2 of the size of the data type, and thus usable to find the
7670 index of the builtin decl. */
7673 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7674 tree addr
, tree loaded_val
, tree stored_val
,
7677 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7678 tree type
, itype
, cmpxchg
, iaddr
;
7679 gimple_stmt_iterator si
;
7680 basic_block loop_header
= single_succ (load_bb
);
7683 enum built_in_function fncode
;
7685 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7686 order to use the RELAXED memory model effectively. */
7687 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7689 cmpxchg
= builtin_decl_explicit (fncode
);
7690 if (cmpxchg
== NULL_TREE
)
7692 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7693 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7695 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7698 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7699 si
= gsi_last_bb (load_bb
);
7700 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7702 /* For floating-point values, we'll need to view-convert them to integers
7703 so that we can perform the atomic compare and swap. Simplify the
7704 following code by always setting up the "i"ntegral variables. */
7705 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7709 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7712 = force_gimple_operand_gsi (&si
,
7713 fold_convert (TREE_TYPE (iaddr
), addr
),
7714 false, NULL_TREE
, true, GSI_SAME_STMT
);
7715 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7716 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7717 loadedi
= create_tmp_var (itype
, NULL
);
7718 if (gimple_in_ssa_p (cfun
))
7719 loadedi
= make_ssa_name (loadedi
, NULL
);
7724 loadedi
= loaded_val
;
7727 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7728 tree loaddecl
= builtin_decl_explicit (fncode
);
7731 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
7732 build_call_expr (loaddecl
, 2, iaddr
,
7733 build_int_cst (NULL_TREE
,
7734 MEMMODEL_RELAXED
)));
7736 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
7737 build_int_cst (TREE_TYPE (iaddr
), 0));
7740 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
7743 /* Move the value to the LOADEDI temporary. */
7744 if (gimple_in_ssa_p (cfun
))
7746 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7747 phi
= create_phi_node (loadedi
, loop_header
);
7748 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7752 gsi_insert_before (&si
,
7753 gimple_build_assign (loadedi
, initial
),
7755 if (loadedi
!= loaded_val
)
7757 gimple_stmt_iterator gsi2
;
7760 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7761 gsi2
= gsi_start_bb (loop_header
);
7762 if (gimple_in_ssa_p (cfun
))
7765 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7766 true, GSI_SAME_STMT
);
7767 stmt
= gimple_build_assign (loaded_val
, x
);
7768 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7772 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7773 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7774 true, GSI_SAME_STMT
);
7777 gsi_remove (&si
, true);
7779 si
= gsi_last_bb (store_bb
);
7780 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7783 storedi
= stored_val
;
7786 force_gimple_operand_gsi (&si
,
7787 build1 (VIEW_CONVERT_EXPR
, itype
,
7788 stored_val
), true, NULL_TREE
, true,
7791 /* Build the compare&swap statement. */
7792 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7793 new_storedi
= force_gimple_operand_gsi (&si
,
7794 fold_convert (TREE_TYPE (loadedi
),
7797 true, GSI_SAME_STMT
);
7799 if (gimple_in_ssa_p (cfun
))
7803 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7804 stmt
= gimple_build_assign (old_vali
, loadedi
);
7805 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7807 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7808 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7811 /* Note that we always perform the comparison as an integer, even for
7812 floating point. This allows the atomic operation to properly
7813 succeed even with NaNs and -0.0. */
7814 stmt
= gimple_build_cond_empty
7815 (build2 (NE_EXPR
, boolean_type_node
,
7816 new_storedi
, old_vali
));
7817 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7820 e
= single_succ_edge (store_bb
);
7821 e
->flags
&= ~EDGE_FALLTHRU
;
7822 e
->flags
|= EDGE_FALSE_VALUE
;
7824 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7826 /* Copy the new value to loadedi (we already did that before the condition
7827 if we are not in SSA). */
7828 if (gimple_in_ssa_p (cfun
))
7830 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7831 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7834 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7835 gsi_remove (&si
, true);
7837 struct loop
*loop
= alloc_loop ();
7838 loop
->header
= loop_header
;
7839 loop
->latch
= store_bb
;
7840 add_loop (loop
, loop_header
->loop_father
);
7842 if (gimple_in_ssa_p (cfun
))
7843 update_ssa (TODO_update_ssa_no_phi
);
7848 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7850 GOMP_atomic_start ();
7854 The result is not globally atomic, but works so long as all parallel
7855 references are within #pragma omp atomic directives. According to
7856 responses received from omp@openmp.org, appears to be within spec.
7857 Which makes sense, since that's how several other compilers handle
7858 this situation as well.
7859 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7860 expanding. STORED_VAL is the operand of the matching
7861 GIMPLE_OMP_ATOMIC_STORE.
7864 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7868 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7873 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7874 tree addr
, tree loaded_val
, tree stored_val
)
7876 gimple_stmt_iterator si
;
7880 si
= gsi_last_bb (load_bb
);
7881 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7883 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7884 t
= build_call_expr (t
, 0);
7885 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7887 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7888 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7889 gsi_remove (&si
, true);
7891 si
= gsi_last_bb (store_bb
);
7892 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7894 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7896 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7898 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7899 t
= build_call_expr (t
, 0);
7900 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7901 gsi_remove (&si
, true);
7903 if (gimple_in_ssa_p (cfun
))
7904 update_ssa (TODO_update_ssa_no_phi
);
7908 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7909 using expand_omp_atomic_fetch_op. If it failed, we try to
7910 call expand_omp_atomic_pipeline, and if it fails too, the
7911 ultimate fallback is wrapping the operation in a mutex
7912 (expand_omp_atomic_mutex). REGION is the atomic region built
7913 by build_omp_regions_1(). */
7916 expand_omp_atomic (struct omp_region
*region
)
7918 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7919 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7920 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7921 tree addr
= gimple_omp_atomic_load_rhs (load
);
7922 tree stored_val
= gimple_omp_atomic_store_val (store
);
7923 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7924 HOST_WIDE_INT index
;
7926 /* Make sure the type is one of the supported sizes. */
7927 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7928 index
= exact_log2 (index
);
7929 if (index
>= 0 && index
<= 4)
7931 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7933 /* __sync builtins require strict data alignment. */
7934 if (exact_log2 (align
) >= index
)
7937 if (loaded_val
== stored_val
7938 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7939 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7940 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7941 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7945 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7946 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7947 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7948 && store_bb
== single_succ (load_bb
)
7949 && first_stmt (store_bb
) == store
7950 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7954 /* When possible, use specialized atomic update functions. */
7955 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7956 && store_bb
== single_succ (load_bb
)
7957 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7958 loaded_val
, stored_val
, index
))
7961 /* If we don't have specialized __sync builtins, try and implement
7962 as a compare and swap loop. */
7963 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7964 loaded_val
, stored_val
, index
))
7969 /* The ultimate fallback is wrapping the operation in a mutex. */
7970 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7974 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7977 expand_omp_target (struct omp_region
*region
)
7979 basic_block entry_bb
, exit_bb
, new_bb
;
7980 struct function
*child_cfun
= NULL
;
7981 tree child_fn
= NULL_TREE
, block
, t
;
7982 gimple_stmt_iterator gsi
;
7983 gimple entry_stmt
, stmt
;
7986 entry_stmt
= last_stmt (region
->entry
);
7987 new_bb
= region
->entry
;
7988 int kind
= gimple_omp_target_kind (entry_stmt
);
7989 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7991 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7992 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7995 entry_bb
= region
->entry
;
7996 exit_bb
= region
->exit
;
7998 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8000 unsigned srcidx
, dstidx
, num
;
8002 /* If the target region needs data sent from the parent
8003 function, then the very first statement (except possible
8004 tree profile counter updates) of the parallel body
8005 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8006 &.OMP_DATA_O is passed as an argument to the child function,
8007 we need to replace it with the argument as seen by the child
8010 In most cases, this will end up being the identity assignment
8011 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
8012 a function call that has been inlined, the original PARM_DECL
8013 .OMP_DATA_I may have been converted into a different local
8014 variable. In which case, we need to keep the assignment. */
8015 if (gimple_omp_target_data_arg (entry_stmt
))
8017 basic_block entry_succ_bb
= single_succ (entry_bb
);
8018 gimple_stmt_iterator gsi
;
8020 gimple tgtcopy_stmt
= NULL
;
8022 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
8024 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8026 gcc_assert (!gsi_end_p (gsi
));
8027 stmt
= gsi_stmt (gsi
);
8028 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8031 if (gimple_num_ops (stmt
) == 2)
8033 tree arg
= gimple_assign_rhs1 (stmt
);
8035 /* We're ignoring the subcode because we're
8036 effectively doing a STRIP_NOPS. */
8038 if (TREE_CODE (arg
) == ADDR_EXPR
8039 && TREE_OPERAND (arg
, 0) == sender
)
8041 tgtcopy_stmt
= stmt
;
8047 gcc_assert (tgtcopy_stmt
!= NULL
);
8048 arg
= DECL_ARGUMENTS (child_fn
);
8050 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8051 gsi_remove (&gsi
, true);
8054 /* Declare local variables needed in CHILD_CFUN. */
8055 block
= DECL_INITIAL (child_fn
);
8056 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8057 /* The gimplifier could record temporaries in target block
8058 rather than in containing function's local_decls chain,
8059 which would mean cgraph missed finalizing them. Do it now. */
8060 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8061 if (TREE_CODE (t
) == VAR_DECL
8063 && !DECL_EXTERNAL (t
))
8064 varpool_finalize_decl (t
);
8065 DECL_SAVED_TREE (child_fn
) = NULL
;
8066 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8067 gimple_set_body (child_fn
, NULL
);
8068 TREE_USED (block
) = 1;
8070 /* Reset DECL_CONTEXT on function arguments. */
8071 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8072 DECL_CONTEXT (t
) = child_fn
;
8074 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
8075 so that it can be moved to the child function. */
8076 gsi
= gsi_last_bb (entry_bb
);
8077 stmt
= gsi_stmt (gsi
);
8078 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
8079 && gimple_omp_target_kind (stmt
)
8080 == GF_OMP_TARGET_KIND_REGION
);
8081 gsi_remove (&gsi
, true);
8082 e
= split_block (entry_bb
, stmt
);
8084 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8086 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8089 gsi
= gsi_last_bb (exit_bb
);
8090 gcc_assert (!gsi_end_p (gsi
)
8091 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8092 stmt
= gimple_build_return (NULL
);
8093 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8094 gsi_remove (&gsi
, true);
8097 /* Move the target region into CHILD_CFUN. */
8099 block
= gimple_block (entry_stmt
);
8101 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8103 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8104 /* When the OMP expansion process cannot guarantee an up-to-date
8105 loop tree arrange for the child function to fixup loops. */
8106 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8107 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8109 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8110 num
= vec_safe_length (child_cfun
->local_decls
);
8111 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8113 t
= (*child_cfun
->local_decls
)[srcidx
];
8114 if (DECL_CONTEXT (t
) == cfun
->decl
)
8116 if (srcidx
!= dstidx
)
8117 (*child_cfun
->local_decls
)[dstidx
] = t
;
8121 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8123 /* Inform the callgraph about the new function. */
8124 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8125 cgraph_add_new_function (child_fn
, true);
8127 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8128 fixed in a following pass. */
8129 push_cfun (child_cfun
);
8130 rebuild_cgraph_edges ();
8132 /* Some EH regions might become dead, see PR34608. If
8133 pass_cleanup_cfg isn't the first pass to happen with the
8134 new child, these dead EH edges might cause problems.
8135 Clean them up now. */
8136 if (flag_exceptions
)
8139 bool changed
= false;
8141 FOR_EACH_BB_FN (bb
, cfun
)
8142 changed
|= gimple_purge_dead_eh_edges (bb
);
8144 cleanup_tree_cfg ();
8149 /* Emit a library call to launch the target region, or do data
8151 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8152 enum built_in_function start_ix
;
8153 location_t clause_loc
;
8155 clauses
= gimple_omp_target_clauses (entry_stmt
);
8157 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8158 start_ix
= BUILT_IN_GOMP_TARGET
;
8159 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8160 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8162 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8164 /* By default, the value of DEVICE is -1 (let runtime library choose)
8165 and there is no conditional. */
8167 device
= build_int_cst (integer_type_node
, -1);
8169 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8171 cond
= OMP_CLAUSE_IF_EXPR (c
);
8173 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8176 device
= OMP_CLAUSE_DEVICE_ID (c
);
8177 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8180 clause_loc
= gimple_location (entry_stmt
);
8182 /* Ensure 'device' is of the correct type. */
8183 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8185 /* If we found the clause 'if (cond)', build
8186 (cond ? device : -2). */
8189 cond
= gimple_boolify (cond
);
8191 basic_block cond_bb
, then_bb
, else_bb
;
8195 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8196 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8198 gsi
= gsi_last_bb (new_bb
);
8200 e
= split_block (new_bb
, gsi_stmt (gsi
));
8203 e
= split_block (new_bb
, NULL
);
8208 then_bb
= create_empty_bb (cond_bb
);
8209 else_bb
= create_empty_bb (then_bb
);
8210 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8211 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8213 stmt
= gimple_build_cond_empty (cond
);
8214 gsi
= gsi_last_bb (cond_bb
);
8215 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8217 gsi
= gsi_start_bb (then_bb
);
8218 stmt
= gimple_build_assign (tmp_var
, device
);
8219 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8221 gsi
= gsi_start_bb (else_bb
);
8222 stmt
= gimple_build_assign (tmp_var
,
8223 build_int_cst (integer_type_node
, -2));
8224 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8226 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8227 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8230 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8231 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8233 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8234 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8239 gsi
= gsi_last_bb (new_bb
);
8240 t
= gimple_omp_target_data_arg (entry_stmt
);
8243 t1
= size_zero_node
;
8244 t2
= build_zero_cst (ptr_type_node
);
8250 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8251 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8252 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8253 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8254 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8258 /* FIXME: This will be address of
8259 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8260 symbol, as soon as the linker plugin is able to create it for us. */
8261 tree openmp_target
= build_zero_cst (ptr_type_node
);
8262 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8264 tree fnaddr
= build_fold_addr_expr (child_fn
);
8265 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8266 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8269 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8270 device
, openmp_target
, t1
, t2
, t3
, t4
);
8271 gimple_set_location (g
, gimple_location (entry_stmt
));
8272 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8273 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8276 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8277 gsi_remove (&gsi
, true);
8279 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8281 gsi
= gsi_last_bb (region
->exit
);
8283 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8284 gsi_remove (&gsi
, true);
8289 /* Expand the parallel region tree rooted at REGION. Expansion
8290 proceeds in depth-first order. Innermost regions are expanded
8291 first. This way, parallel regions that require a new function to
8292 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8293 internal dependencies in their body. */
8296 expand_omp (struct omp_region
*region
)
8300 location_t saved_location
;
8301 gimple inner_stmt
= NULL
;
8303 /* First, determine whether this is a combined parallel+workshare
8305 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8306 determine_parallel_type (region
);
8308 if (region
->type
== GIMPLE_OMP_FOR
8309 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8310 inner_stmt
= last_stmt (region
->inner
->entry
);
8313 expand_omp (region
->inner
);
8315 saved_location
= input_location
;
8316 if (gimple_has_location (last_stmt (region
->entry
)))
8317 input_location
= gimple_location (last_stmt (region
->entry
));
8319 switch (region
->type
)
8321 case GIMPLE_OMP_PARALLEL
:
8322 case GIMPLE_OMP_TASK
:
8323 expand_omp_taskreg (region
);
8326 case GIMPLE_OMP_FOR
:
8327 expand_omp_for (region
, inner_stmt
);
8330 case GIMPLE_OMP_SECTIONS
:
8331 expand_omp_sections (region
);
8334 case GIMPLE_OMP_SECTION
:
8335 /* Individual omp sections are handled together with their
8336 parent GIMPLE_OMP_SECTIONS region. */
8339 case GIMPLE_OMP_SINGLE
:
8340 expand_omp_single (region
);
8343 case GIMPLE_OMP_MASTER
:
8344 case GIMPLE_OMP_TASKGROUP
:
8345 case GIMPLE_OMP_ORDERED
:
8346 case GIMPLE_OMP_CRITICAL
:
8347 case GIMPLE_OMP_TEAMS
:
8348 expand_omp_synch (region
);
8351 case GIMPLE_OMP_ATOMIC_LOAD
:
8352 expand_omp_atomic (region
);
8355 case GIMPLE_OMP_TARGET
:
8356 expand_omp_target (region
);
8363 input_location
= saved_location
;
8364 region
= region
->next
;
8369 /* Helper for build_omp_regions. Scan the dominator tree starting at
8370 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8371 true, the function ends once a single tree is built (otherwise, whole
8372 forest of OMP constructs may be built). */
8375 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8378 gimple_stmt_iterator gsi
;
8382 gsi
= gsi_last_bb (bb
);
8383 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8385 struct omp_region
*region
;
8386 enum gimple_code code
;
8388 stmt
= gsi_stmt (gsi
);
8389 code
= gimple_code (stmt
);
8390 if (code
== GIMPLE_OMP_RETURN
)
8392 /* STMT is the return point out of region PARENT. Mark it
8393 as the exit point and make PARENT the immediately
8394 enclosing region. */
8395 gcc_assert (parent
);
8398 parent
= parent
->outer
;
8400 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8402 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8403 GIMPLE_OMP_RETURN, but matches with
8404 GIMPLE_OMP_ATOMIC_LOAD. */
8405 gcc_assert (parent
);
8406 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8409 parent
= parent
->outer
;
8412 else if (code
== GIMPLE_OMP_CONTINUE
)
8414 gcc_assert (parent
);
8417 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8419 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8420 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8423 else if (code
== GIMPLE_OMP_TARGET
8424 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8425 new_omp_region (bb
, code
, parent
);
8428 /* Otherwise, this directive becomes the parent for a new
8430 region
= new_omp_region (bb
, code
, parent
);
8435 if (single_tree
&& !parent
)
8438 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8440 son
= next_dom_son (CDI_DOMINATORS
, son
))
8441 build_omp_regions_1 (son
, parent
, single_tree
);
8444 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8448 build_omp_regions_root (basic_block root
)
8450 gcc_assert (root_omp_region
== NULL
);
8451 build_omp_regions_1 (root
, NULL
, true);
8452 gcc_assert (root_omp_region
!= NULL
);
8455 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8458 omp_expand_local (basic_block head
)
8460 build_omp_regions_root (head
);
8461 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8463 fprintf (dump_file
, "\nOMP region tree\n\n");
8464 dump_omp_region (dump_file
, root_omp_region
, 0);
8465 fprintf (dump_file
, "\n");
8468 remove_exit_barriers (root_omp_region
);
8469 expand_omp (root_omp_region
);
8471 free_omp_regions ();
8474 /* Scan the CFG and build a tree of OMP regions. Return the root of
8475 the OMP region tree. */
8478 build_omp_regions (void)
8480 gcc_assert (root_omp_region
== NULL
);
8481 calculate_dominance_info (CDI_DOMINATORS
);
8482 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8485 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8488 execute_expand_omp (void)
8490 build_omp_regions ();
8492 if (!root_omp_region
)
8497 fprintf (dump_file
, "\nOMP region tree\n\n");
8498 dump_omp_region (dump_file
, root_omp_region
, 0);
8499 fprintf (dump_file
, "\n");
8502 remove_exit_barriers (root_omp_region
);
8504 expand_omp (root_omp_region
);
8506 cleanup_tree_cfg ();
8508 free_omp_regions ();
8513 /* OMP expansion -- the default pass, run before creation of SSA form. */
8516 gate_expand_omp (void)
8518 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8519 || flag_cilkplus
!= 0) && !seen_error ());
8524 const pass_data pass_data_expand_omp
=
8526 GIMPLE_PASS
, /* type */
8527 "ompexp", /* name */
8528 OPTGROUP_NONE
, /* optinfo_flags */
8529 true, /* has_gate */
8530 true, /* has_execute */
8531 TV_NONE
, /* tv_id */
8532 PROP_gimple_any
, /* properties_required */
8533 0, /* properties_provided */
8534 0, /* properties_destroyed */
8535 0, /* todo_flags_start */
8536 0, /* todo_flags_finish */
8539 class pass_expand_omp
: public gimple_opt_pass
8542 pass_expand_omp (gcc::context
*ctxt
)
8543 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8546 /* opt_pass methods: */
8547 bool gate () { return gate_expand_omp (); }
8548 unsigned int execute () { return execute_expand_omp (); }
8550 }; // class pass_expand_omp
8555 make_pass_expand_omp (gcc::context
*ctxt
)
8557 return new pass_expand_omp (ctxt
);
8560 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8562 /* If ctx is a worksharing context inside of a cancellable parallel
8563 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8564 and conditional branch to parallel's cancel_label to handle
8565 cancellation in the implicit barrier. */
8568 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8570 gimple omp_return
= gimple_seq_last_stmt (*body
);
8571 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8572 if (gimple_omp_return_nowait_p (omp_return
))
8575 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8576 && ctx
->outer
->cancellable
)
8578 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8579 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8580 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8581 gimple_omp_return_set_lhs (omp_return
, lhs
);
8582 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8583 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8584 fold_convert (c_bool_type
,
8585 boolean_false_node
),
8586 ctx
->outer
->cancel_label
, fallthru_label
);
8587 gimple_seq_add_stmt (body
, g
);
8588 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8592 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8593 CTX is the enclosing OMP context for the current statement. */
8596 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8598 tree block
, control
;
8599 gimple_stmt_iterator tgsi
;
8600 gimple stmt
, new_stmt
, bind
, t
;
8601 gimple_seq ilist
, dlist
, olist
, new_body
;
8603 stmt
= gsi_stmt (*gsi_p
);
8605 push_gimplify_context ();
8609 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8610 &ilist
, &dlist
, ctx
, NULL
);
8612 new_body
= gimple_omp_body (stmt
);
8613 gimple_omp_set_body (stmt
, NULL
);
8614 tgsi
= gsi_start (new_body
);
8615 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8620 sec_start
= gsi_stmt (tgsi
);
8621 sctx
= maybe_lookup_ctx (sec_start
);
8624 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8625 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8626 GSI_CONTINUE_LINKING
);
8627 gimple_omp_set_body (sec_start
, NULL
);
8629 if (gsi_one_before_end_p (tgsi
))
8631 gimple_seq l
= NULL
;
8632 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8634 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8635 gimple_omp_section_set_last (sec_start
);
8638 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8639 GSI_CONTINUE_LINKING
);
8642 block
= make_node (BLOCK
);
8643 bind
= gimple_build_bind (NULL
, new_body
, block
);
8646 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8648 block
= make_node (BLOCK
);
8649 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8650 gsi_replace (gsi_p
, new_stmt
, true);
8652 pop_gimplify_context (new_stmt
);
8653 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8654 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8655 if (BLOCK_VARS (block
))
8656 TREE_USED (block
) = 1;
8659 gimple_seq_add_seq (&new_body
, ilist
);
8660 gimple_seq_add_stmt (&new_body
, stmt
);
8661 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8662 gimple_seq_add_stmt (&new_body
, bind
);
8664 control
= create_tmp_var (unsigned_type_node
, ".section");
8665 t
= gimple_build_omp_continue (control
, control
);
8666 gimple_omp_sections_set_control (stmt
, control
);
8667 gimple_seq_add_stmt (&new_body
, t
);
8669 gimple_seq_add_seq (&new_body
, olist
);
8670 if (ctx
->cancellable
)
8671 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8672 gimple_seq_add_seq (&new_body
, dlist
);
8674 new_body
= maybe_catch_exception (new_body
);
8676 t
= gimple_build_omp_return
8677 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8678 OMP_CLAUSE_NOWAIT
));
8679 gimple_seq_add_stmt (&new_body
, t
);
8680 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8682 gimple_bind_set_body (new_stmt
, new_body
);
8686 /* A subroutine of lower_omp_single. Expand the simple form of
8687 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8689 if (GOMP_single_start ())
8691 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8693 FIXME. It may be better to delay expanding the logic of this until
8694 pass_expand_omp. The expanded logic may make the job more difficult
8695 to a synchronization analysis pass. */
8698 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8700 location_t loc
= gimple_location (single_stmt
);
8701 tree tlabel
= create_artificial_label (loc
);
8702 tree flabel
= create_artificial_label (loc
);
8706 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8707 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8708 call
= gimple_build_call (decl
, 0);
8709 gimple_call_set_lhs (call
, lhs
);
8710 gimple_seq_add_stmt (pre_p
, call
);
8712 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8713 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8716 gimple_seq_add_stmt (pre_p
, cond
);
8717 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8718 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8719 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8723 /* A subroutine of lower_omp_single. Expand the simple form of
8724 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8726 #pragma omp single copyprivate (a, b, c)
8728 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8731 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8737 GOMP_single_copy_end (©out);
8748 FIXME. It may be better to delay expanding the logic of this until
8749 pass_expand_omp. The expanded logic may make the job more difficult
8750 to a synchronization analysis pass. */
8753 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8755 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8756 gimple_seq copyin_seq
;
8757 location_t loc
= gimple_location (single_stmt
);
8759 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8761 ptr_type
= build_pointer_type (ctx
->record_type
);
8762 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8764 l0
= create_artificial_label (loc
);
8765 l1
= create_artificial_label (loc
);
8766 l2
= create_artificial_label (loc
);
8768 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8769 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8770 t
= fold_convert_loc (loc
, ptr_type
, t
);
8771 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8773 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8774 build_int_cst (ptr_type
, 0));
8775 t
= build3 (COND_EXPR
, void_type_node
, t
,
8776 build_and_jump (&l0
), build_and_jump (&l1
));
8777 gimplify_and_add (t
, pre_p
);
8779 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8781 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8784 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8787 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8788 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8789 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8790 gimplify_and_add (t
, pre_p
);
8792 t
= build_and_jump (&l2
);
8793 gimplify_and_add (t
, pre_p
);
8795 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8797 gimple_seq_add_seq (pre_p
, copyin_seq
);
8799 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8803 /* Expand code for an OpenMP single directive. */
8806 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8809 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8810 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8812 push_gimplify_context ();
8814 block
= make_node (BLOCK
);
8815 bind
= gimple_build_bind (NULL
, NULL
, block
);
8816 gsi_replace (gsi_p
, bind
, true);
8819 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8820 &bind_body
, &dlist
, ctx
, NULL
);
8821 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8823 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8825 if (ctx
->record_type
)
8826 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8828 lower_omp_single_simple (single_stmt
, &bind_body
);
8830 gimple_omp_set_body (single_stmt
, NULL
);
8832 gimple_seq_add_seq (&bind_body
, dlist
);
8834 bind_body
= maybe_catch_exception (bind_body
);
8836 t
= gimple_build_omp_return
8837 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8838 OMP_CLAUSE_NOWAIT
));
8839 gimple_seq_add_stmt (&bind_body_tail
, t
);
8840 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8841 if (ctx
->record_type
)
8843 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8844 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8845 TREE_THIS_VOLATILE (clobber
) = 1;
8846 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8847 clobber
), GSI_SAME_STMT
);
8849 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8850 gimple_bind_set_body (bind
, bind_body
);
8852 pop_gimplify_context (bind
);
8854 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8855 BLOCK_VARS (block
) = ctx
->block_vars
;
8856 if (BLOCK_VARS (block
))
8857 TREE_USED (block
) = 1;
8861 /* Expand code for an OpenMP master directive. */
8864 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8866 tree block
, lab
= NULL
, x
, bfn_decl
;
8867 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8868 location_t loc
= gimple_location (stmt
);
8871 push_gimplify_context ();
8873 block
= make_node (BLOCK
);
8874 bind
= gimple_build_bind (NULL
, NULL
, block
);
8875 gsi_replace (gsi_p
, bind
, true);
8876 gimple_bind_add_stmt (bind
, stmt
);
8878 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8879 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8880 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8881 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8883 gimplify_and_add (x
, &tseq
);
8884 gimple_bind_add_seq (bind
, tseq
);
8886 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8887 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8888 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8889 gimple_omp_set_body (stmt
, NULL
);
8891 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8893 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8895 pop_gimplify_context (bind
);
8897 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8898 BLOCK_VARS (block
) = ctx
->block_vars
;
8902 /* Expand code for an OpenMP taskgroup directive. */
8905 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8907 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8908 tree block
= make_node (BLOCK
);
8910 bind
= gimple_build_bind (NULL
, NULL
, block
);
8911 gsi_replace (gsi_p
, bind
, true);
8912 gimple_bind_add_stmt (bind
, stmt
);
8914 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8916 gimple_bind_add_stmt (bind
, x
);
8918 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8919 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8920 gimple_omp_set_body (stmt
, NULL
);
8922 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8924 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8925 BLOCK_VARS (block
) = ctx
->block_vars
;
8929 /* Expand code for an OpenMP ordered directive. */
8932 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8935 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8937 push_gimplify_context ();
8939 block
= make_node (BLOCK
);
8940 bind
= gimple_build_bind (NULL
, NULL
, block
);
8941 gsi_replace (gsi_p
, bind
, true);
8942 gimple_bind_add_stmt (bind
, stmt
);
8944 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8946 gimple_bind_add_stmt (bind
, x
);
8948 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8949 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8950 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8951 gimple_omp_set_body (stmt
, NULL
);
8953 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8954 gimple_bind_add_stmt (bind
, x
);
8956 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8958 pop_gimplify_context (bind
);
8960 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8961 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8965 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8966 substitution of a couple of function calls. But in the NAMED case,
8967 requires that languages coordinate a symbol name. It is therefore
8968 best put here in common code. */
8970 static GTY((param1_is (tree
), param2_is (tree
)))
8971 splay_tree critical_name_mutexes
;
8974 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8977 tree name
, lock
, unlock
;
8978 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8979 location_t loc
= gimple_location (stmt
);
8982 name
= gimple_omp_critical_name (stmt
);
8988 if (!critical_name_mutexes
)
8989 critical_name_mutexes
8990 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8991 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8992 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8994 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8999 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
9001 new_str
= ACONCAT ((".gomp_critical_user_",
9002 IDENTIFIER_POINTER (name
), NULL
));
9003 DECL_NAME (decl
) = get_identifier (new_str
);
9004 TREE_PUBLIC (decl
) = 1;
9005 TREE_STATIC (decl
) = 1;
9006 DECL_COMMON (decl
) = 1;
9007 DECL_ARTIFICIAL (decl
) = 1;
9008 DECL_IGNORED_P (decl
) = 1;
9009 varpool_finalize_decl (decl
);
9011 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
9012 (splay_tree_value
) decl
);
9015 decl
= (tree
) n
->value
;
9017 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
9018 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
9020 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
9021 unlock
= build_call_expr_loc (loc
, unlock
, 1,
9022 build_fold_addr_expr_loc (loc
, decl
));
9026 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9027 lock
= build_call_expr_loc (loc
, lock
, 0);
9029 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9030 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9033 push_gimplify_context ();
9035 block
= make_node (BLOCK
);
9036 bind
= gimple_build_bind (NULL
, NULL
, block
);
9037 gsi_replace (gsi_p
, bind
, true);
9038 gimple_bind_add_stmt (bind
, stmt
);
9040 tbody
= gimple_bind_body (bind
);
9041 gimplify_and_add (lock
, &tbody
);
9042 gimple_bind_set_body (bind
, tbody
);
9044 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9045 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9046 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9047 gimple_omp_set_body (stmt
, NULL
);
9049 tbody
= gimple_bind_body (bind
);
9050 gimplify_and_add (unlock
, &tbody
);
9051 gimple_bind_set_body (bind
, tbody
);
9053 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9055 pop_gimplify_context (bind
);
9056 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9057 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9061 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9062 for a lastprivate clause. Given a loop control predicate of (V
9063 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9064 is appended to *DLIST, iterator initialization is appended to
9068 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9069 gimple_seq
*dlist
, struct omp_context
*ctx
)
9071 tree clauses
, cond
, vinit
;
9072 enum tree_code cond_code
;
9075 cond_code
= fd
->loop
.cond_code
;
9076 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9078 /* When possible, use a strict equality expression. This can let VRP
9079 type optimizations deduce the value and remove a copy. */
9080 if (tree_fits_shwi_p (fd
->loop
.step
))
9082 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9083 if (step
== 1 || step
== -1)
9084 cond_code
= EQ_EXPR
;
9087 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
9089 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9091 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
9092 if (!gimple_seq_empty_p (stmts
))
9094 gimple_seq_add_seq (&stmts
, *dlist
);
9097 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9098 vinit
= fd
->loop
.n1
;
9099 if (cond_code
== EQ_EXPR
9100 && tree_fits_shwi_p (fd
->loop
.n2
)
9101 && ! integer_zerop (fd
->loop
.n2
))
9102 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9104 vinit
= unshare_expr (vinit
);
9106 /* Initialize the iterator variable, so that threads that don't execute
9107 any iterations don't execute the lastprivate clauses by accident. */
9108 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9113 /* Lower code for an OpenMP loop directive. */
9116 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9119 struct omp_for_data fd
, *fdp
= NULL
;
9120 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9121 gimple_seq omp_for_body
, body
, dlist
;
9124 push_gimplify_context ();
9126 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9128 block
= make_node (BLOCK
);
9129 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9130 /* Replace at gsi right away, so that 'stmt' is no member
9131 of a sequence anymore as we're going to add to to a different
9133 gsi_replace (gsi_p
, new_stmt
, true);
9135 /* Move declaration of temporaries in the loop body before we make
9137 omp_for_body
= gimple_omp_body (stmt
);
9138 if (!gimple_seq_empty_p (omp_for_body
)
9139 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9141 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9142 tree vars
= gimple_bind_vars (inner_bind
);
9143 gimple_bind_append_vars (new_stmt
, vars
);
9144 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9145 keep them on the inner_bind and it's block. */
9146 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9147 if (gimple_bind_block (inner_bind
))
9148 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9151 if (gimple_omp_for_combined_into_p (stmt
))
9153 extract_omp_for_data (stmt
, &fd
, NULL
);
9156 /* We need two temporaries with fd.loop.v type (istart/iend)
9157 and then (fd.collapse - 1) temporaries with the same
9158 type for count2 ... countN-1 vars if not constant. */
9160 tree type
= fd
.iter_type
;
9162 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9163 count
+= fd
.collapse
- 1;
9164 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9165 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9169 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9170 OMP_CLAUSE__LOOPTEMP_
);
9171 for (i
= 0; i
< count
; i
++)
9176 gcc_assert (outerc
);
9177 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9178 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9179 OMP_CLAUSE__LOOPTEMP_
);
9183 temp
= create_tmp_var (type
, NULL
);
9184 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9186 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9187 OMP_CLAUSE_DECL (*pc
) = temp
;
9188 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9193 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9196 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9198 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9200 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9202 /* Lower the header expressions. At this point, we can assume that
9203 the header is of the form:
9205 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9207 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9208 using the .omp_data_s mapping, if needed. */
9209 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9211 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9212 if (!is_gimple_min_invariant (*rhs_p
))
9213 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9215 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9216 if (!is_gimple_min_invariant (*rhs_p
))
9217 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9219 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9220 if (!is_gimple_min_invariant (*rhs_p
))
9221 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9224 /* Once lowered, extract the bounds and clauses. */
9225 extract_omp_for_data (stmt
, &fd
, NULL
);
9227 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9229 gimple_seq_add_stmt (&body
, stmt
);
9230 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9232 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9235 /* After the loop, add exit clauses. */
9236 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9238 if (ctx
->cancellable
)
9239 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9241 gimple_seq_add_seq (&body
, dlist
);
9243 body
= maybe_catch_exception (body
);
9245 /* Region exit marker goes at the end of the loop body. */
9246 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9247 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9248 pop_gimplify_context (new_stmt
);
9250 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9251 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9252 if (BLOCK_VARS (block
))
9253 TREE_USED (block
) = 1;
9255 gimple_bind_set_body (new_stmt
, body
);
9256 gimple_omp_set_body (stmt
, NULL
);
9257 gimple_omp_for_set_pre_body (stmt
, NULL
);
9260 /* Callback for walk_stmts. Check if the current statement only contains
9261 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9264 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9265 bool *handled_ops_p
,
9266 struct walk_stmt_info
*wi
)
9268 int *info
= (int *) wi
->info
;
9269 gimple stmt
= gsi_stmt (*gsi_p
);
9271 *handled_ops_p
= true;
9272 switch (gimple_code (stmt
))
9276 case GIMPLE_OMP_FOR
:
9277 case GIMPLE_OMP_SECTIONS
:
9278 *info
= *info
== 0 ? 1 : -1;
9287 struct omp_taskcopy_context
9289 /* This field must be at the beginning, as we do "inheritance": Some
9290 callback functions for tree-inline.c (e.g., omp_copy_decl)
9291 receive a copy_body_data pointer that is up-casted to an
9292 omp_context pointer. */
9298 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9300 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9302 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9303 return create_tmp_var (TREE_TYPE (var
), NULL
);
9309 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9311 tree name
, new_fields
= NULL
, type
, f
;
9313 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9314 name
= DECL_NAME (TYPE_NAME (orig_type
));
9315 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9316 TYPE_DECL
, name
, type
);
9317 TYPE_NAME (type
) = name
;
9319 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9321 tree new_f
= copy_node (f
);
9322 DECL_CONTEXT (new_f
) = type
;
9323 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9324 TREE_CHAIN (new_f
) = new_fields
;
9325 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9326 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9327 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9330 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9332 TYPE_FIELDS (type
) = nreverse (new_fields
);
9337 /* Create task copyfn. */
9340 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9342 struct function
*child_cfun
;
9343 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9344 tree record_type
, srecord_type
, bind
, list
;
9345 bool record_needs_remap
= false, srecord_needs_remap
= false;
9347 struct omp_taskcopy_context tcctx
;
9348 location_t loc
= gimple_location (task_stmt
);
9350 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9351 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9352 gcc_assert (child_cfun
->cfg
== NULL
);
9353 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9355 /* Reset DECL_CONTEXT on function arguments. */
9356 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9357 DECL_CONTEXT (t
) = child_fn
;
9359 /* Populate the function. */
9360 push_gimplify_context ();
9361 push_cfun (child_cfun
);
9363 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9364 TREE_SIDE_EFFECTS (bind
) = 1;
9366 DECL_SAVED_TREE (child_fn
) = bind
;
9367 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9369 /* Remap src and dst argument types if needed. */
9370 record_type
= ctx
->record_type
;
9371 srecord_type
= ctx
->srecord_type
;
9372 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9373 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9375 record_needs_remap
= true;
9378 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9379 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9381 srecord_needs_remap
= true;
9385 if (record_needs_remap
|| srecord_needs_remap
)
9387 memset (&tcctx
, '\0', sizeof (tcctx
));
9388 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9389 tcctx
.cb
.dst_fn
= child_fn
;
9390 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9391 gcc_checking_assert (tcctx
.cb
.src_node
);
9392 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9393 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9394 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9395 tcctx
.cb
.eh_lp_nr
= 0;
9396 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9397 tcctx
.cb
.decl_map
= pointer_map_create ();
9400 if (record_needs_remap
)
9401 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9402 if (srecord_needs_remap
)
9403 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9406 tcctx
.cb
.decl_map
= NULL
;
9408 arg
= DECL_ARGUMENTS (child_fn
);
9409 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9410 sarg
= DECL_CHAIN (arg
);
9411 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9413 /* First pass: initialize temporaries used in record_type and srecord_type
9414 sizes and field offsets. */
9415 if (tcctx
.cb
.decl_map
)
9416 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9417 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9421 decl
= OMP_CLAUSE_DECL (c
);
9422 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9425 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9426 sf
= (tree
) n
->value
;
9427 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9428 src
= build_simple_mem_ref_loc (loc
, sarg
);
9429 src
= omp_build_component_ref (src
, sf
);
9430 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9431 append_to_statement_list (t
, &list
);
9434 /* Second pass: copy shared var pointers and copy construct non-VLA
9435 firstprivate vars. */
9436 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9437 switch (OMP_CLAUSE_CODE (c
))
9439 case OMP_CLAUSE_SHARED
:
9440 decl
= OMP_CLAUSE_DECL (c
);
9441 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9444 f
= (tree
) n
->value
;
9445 if (tcctx
.cb
.decl_map
)
9446 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9447 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9448 sf
= (tree
) n
->value
;
9449 if (tcctx
.cb
.decl_map
)
9450 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9451 src
= build_simple_mem_ref_loc (loc
, sarg
);
9452 src
= omp_build_component_ref (src
, sf
);
9453 dst
= build_simple_mem_ref_loc (loc
, arg
);
9454 dst
= omp_build_component_ref (dst
, f
);
9455 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9456 append_to_statement_list (t
, &list
);
9458 case OMP_CLAUSE_FIRSTPRIVATE
:
9459 decl
= OMP_CLAUSE_DECL (c
);
9460 if (is_variable_sized (decl
))
9462 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9465 f
= (tree
) n
->value
;
9466 if (tcctx
.cb
.decl_map
)
9467 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9468 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9471 sf
= (tree
) n
->value
;
9472 if (tcctx
.cb
.decl_map
)
9473 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9474 src
= build_simple_mem_ref_loc (loc
, sarg
);
9475 src
= omp_build_component_ref (src
, sf
);
9476 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9477 src
= build_simple_mem_ref_loc (loc
, src
);
9481 dst
= build_simple_mem_ref_loc (loc
, arg
);
9482 dst
= omp_build_component_ref (dst
, f
);
9483 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9484 append_to_statement_list (t
, &list
);
9486 case OMP_CLAUSE_PRIVATE
:
9487 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9489 decl
= OMP_CLAUSE_DECL (c
);
9490 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9491 f
= (tree
) n
->value
;
9492 if (tcctx
.cb
.decl_map
)
9493 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9494 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9497 sf
= (tree
) n
->value
;
9498 if (tcctx
.cb
.decl_map
)
9499 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9500 src
= build_simple_mem_ref_loc (loc
, sarg
);
9501 src
= omp_build_component_ref (src
, sf
);
9502 if (use_pointer_for_field (decl
, NULL
))
9503 src
= build_simple_mem_ref_loc (loc
, src
);
9507 dst
= build_simple_mem_ref_loc (loc
, arg
);
9508 dst
= omp_build_component_ref (dst
, f
);
9509 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9510 append_to_statement_list (t
, &list
);
9516 /* Last pass: handle VLA firstprivates. */
9517 if (tcctx
.cb
.decl_map
)
9518 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9519 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9523 decl
= OMP_CLAUSE_DECL (c
);
9524 if (!is_variable_sized (decl
))
9526 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9529 f
= (tree
) n
->value
;
9530 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9531 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9532 ind
= DECL_VALUE_EXPR (decl
);
9533 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9534 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9535 n
= splay_tree_lookup (ctx
->sfield_map
,
9536 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9537 sf
= (tree
) n
->value
;
9538 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9539 src
= build_simple_mem_ref_loc (loc
, sarg
);
9540 src
= omp_build_component_ref (src
, sf
);
9541 src
= build_simple_mem_ref_loc (loc
, src
);
9542 dst
= build_simple_mem_ref_loc (loc
, arg
);
9543 dst
= omp_build_component_ref (dst
, f
);
9544 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9545 append_to_statement_list (t
, &list
);
9546 n
= splay_tree_lookup (ctx
->field_map
,
9547 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9548 df
= (tree
) n
->value
;
9549 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9550 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9551 ptr
= omp_build_component_ref (ptr
, df
);
9552 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9553 build_fold_addr_expr_loc (loc
, dst
));
9554 append_to_statement_list (t
, &list
);
9557 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9558 append_to_statement_list (t
, &list
);
9560 if (tcctx
.cb
.decl_map
)
9561 pointer_map_destroy (tcctx
.cb
.decl_map
);
9562 pop_gimplify_context (NULL
);
9563 BIND_EXPR_BODY (bind
) = list
;
9568 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9572 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9574 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9576 gcc_assert (clauses
);
9577 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9578 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9579 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9581 case OMP_CLAUSE_DEPEND_IN
:
9584 case OMP_CLAUSE_DEPEND_OUT
:
9585 case OMP_CLAUSE_DEPEND_INOUT
:
9591 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9592 tree array
= create_tmp_var (type
, NULL
);
9593 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9595 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9596 gimple_seq_add_stmt (iseq
, g
);
9597 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9599 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9600 gimple_seq_add_stmt (iseq
, g
);
9601 for (i
= 0; i
< 2; i
++)
9603 if ((i
? n_in
: n_out
) == 0)
9605 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9606 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9607 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9609 tree t
= OMP_CLAUSE_DECL (c
);
9610 t
= fold_convert (ptr_type_node
, t
);
9611 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9612 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9613 NULL_TREE
, NULL_TREE
);
9614 g
= gimple_build_assign (r
, t
);
9615 gimple_seq_add_stmt (iseq
, g
);
9618 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9619 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9620 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9621 OMP_CLAUSE_CHAIN (c
) = *p
;
9623 tree clobber
= build_constructor (type
, NULL
);
9624 TREE_THIS_VOLATILE (clobber
) = 1;
9625 g
= gimple_build_assign (array
, clobber
);
9626 gimple_seq_add_stmt (oseq
, g
);
9629 /* Lower the OpenMP parallel or task directive in the current statement
9630 in GSI_P. CTX holds context information for the directive. */
9633 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9637 gimple stmt
= gsi_stmt (*gsi_p
);
9638 gimple par_bind
, bind
, dep_bind
= NULL
;
9639 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9640 location_t loc
= gimple_location (stmt
);
9642 clauses
= gimple_omp_taskreg_clauses (stmt
);
9643 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9644 par_body
= gimple_bind_body (par_bind
);
9645 child_fn
= ctx
->cb
.dst_fn
;
9646 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9647 && !gimple_omp_parallel_combined_p (stmt
))
9649 struct walk_stmt_info wi
;
9652 memset (&wi
, 0, sizeof (wi
));
9655 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9657 gimple_omp_parallel_set_combined_p (stmt
, true);
9659 gimple_seq dep_ilist
= NULL
;
9660 gimple_seq dep_olist
= NULL
;
9661 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9662 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9664 push_gimplify_context ();
9665 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9666 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9669 if (ctx
->srecord_type
)
9670 create_task_copyfn (stmt
, ctx
);
9672 push_gimplify_context ();
9677 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9678 lower_omp (&par_body
, ctx
);
9679 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9680 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9682 /* Declare all the variables created by mapping and the variables
9683 declared in the scope of the parallel body. */
9684 record_vars_into (ctx
->block_vars
, child_fn
);
9685 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9687 if (ctx
->record_type
)
9690 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9691 : ctx
->record_type
, ".omp_data_o");
9692 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9693 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9694 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9699 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9700 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9702 if (ctx
->record_type
)
9704 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9705 TREE_THIS_VOLATILE (clobber
) = 1;
9706 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9710 /* Once all the expansions are done, sequence all the different
9711 fragments inside gimple_omp_body. */
9715 if (ctx
->record_type
)
9717 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9718 /* fixup_child_record_type might have changed receiver_decl's type. */
9719 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9720 gimple_seq_add_stmt (&new_body
,
9721 gimple_build_assign (ctx
->receiver_decl
, t
));
9724 gimple_seq_add_seq (&new_body
, par_ilist
);
9725 gimple_seq_add_seq (&new_body
, par_body
);
9726 gimple_seq_add_seq (&new_body
, par_rlist
);
9727 if (ctx
->cancellable
)
9728 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9729 gimple_seq_add_seq (&new_body
, par_olist
);
9730 new_body
= maybe_catch_exception (new_body
);
9731 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
9732 gimple_seq_add_stmt (&new_body
,
9733 gimple_build_omp_continue (integer_zero_node
,
9734 integer_zero_node
));
9735 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9736 gimple_omp_set_body (stmt
, new_body
);
9738 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9739 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9740 gimple_bind_add_seq (bind
, ilist
);
9741 gimple_bind_add_stmt (bind
, stmt
);
9742 gimple_bind_add_seq (bind
, olist
);
9744 pop_gimplify_context (NULL
);
9748 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9749 gimple_bind_add_stmt (dep_bind
, bind
);
9750 gimple_bind_add_seq (dep_bind
, dep_olist
);
9751 pop_gimplify_context (dep_bind
);
9755 /* Lower the OpenMP target directive in the current statement
9756 in GSI_P. CTX holds context information for the directive. */
9759 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9762 tree child_fn
, t
, c
;
9763 gimple stmt
= gsi_stmt (*gsi_p
);
9764 gimple tgt_bind
= NULL
, bind
;
9765 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9766 location_t loc
= gimple_location (stmt
);
9767 int kind
= gimple_omp_target_kind (stmt
);
9768 unsigned int map_cnt
= 0;
9770 clauses
= gimple_omp_target_clauses (stmt
);
9771 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9773 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9774 tgt_body
= gimple_bind_body (tgt_bind
);
9776 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9777 tgt_body
= gimple_omp_body (stmt
);
9778 child_fn
= ctx
->cb
.dst_fn
;
9780 push_gimplify_context ();
9782 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9783 switch (OMP_CLAUSE_CODE (c
))
9789 case OMP_CLAUSE_MAP
:
9791 case OMP_CLAUSE_FROM
:
9792 var
= OMP_CLAUSE_DECL (c
);
9795 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9796 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9802 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9804 tree var2
= DECL_VALUE_EXPR (var
);
9805 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9806 var2
= TREE_OPERAND (var2
, 0);
9807 gcc_assert (DECL_P (var2
));
9811 if (!maybe_lookup_field (var
, ctx
))
9814 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9816 x
= build_receiver_ref (var
, true, ctx
);
9817 tree new_var
= lookup_decl (var
, ctx
);
9818 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9819 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9820 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9821 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9822 x
= build_simple_mem_ref (x
);
9823 SET_DECL_VALUE_EXPR (new_var
, x
);
9824 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9829 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9831 target_nesting_level
++;
9832 lower_omp (&tgt_body
, ctx
);
9833 target_nesting_level
--;
9835 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9836 lower_omp (&tgt_body
, ctx
);
9838 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9840 /* Declare all the variables created by mapping and the variables
9841 declared in the scope of the target body. */
9842 record_vars_into (ctx
->block_vars
, child_fn
);
9843 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9848 if (ctx
->record_type
)
9851 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9852 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9853 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9854 t
= make_tree_vec (3);
9855 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9857 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9859 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9860 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9861 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9863 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9866 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9867 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9868 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9869 gimple_omp_target_set_data_arg (stmt
, t
);
9871 vec
<constructor_elt
, va_gc
> *vsize
;
9872 vec
<constructor_elt
, va_gc
> *vkind
;
9873 vec_alloc (vsize
, map_cnt
);
9874 vec_alloc (vkind
, map_cnt
);
9875 unsigned int map_idx
= 0;
9877 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9878 switch (OMP_CLAUSE_CODE (c
))
9884 case OMP_CLAUSE_MAP
:
9886 case OMP_CLAUSE_FROM
:
9888 ovar
= OMP_CLAUSE_DECL (c
);
9891 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9892 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9894 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9895 == get_base_address (ovar
));
9896 nc
= OMP_CLAUSE_CHAIN (c
);
9897 ovar
= OMP_CLAUSE_DECL (nc
);
9901 tree x
= build_sender_ref (ovar
, ctx
);
9903 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9904 gimplify_assign (x
, v
, &ilist
);
9910 if (DECL_SIZE (ovar
)
9911 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9913 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9914 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9915 ovar2
= TREE_OPERAND (ovar2
, 0);
9916 gcc_assert (DECL_P (ovar2
));
9919 if (!maybe_lookup_field (ovar
, ctx
))
9923 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9924 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9925 talign
= DECL_ALIGN_UNIT (ovar
);
9928 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9929 tree x
= build_sender_ref (ovar
, ctx
);
9930 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9931 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9932 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9933 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9935 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9937 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9938 mark_addressable (avar
);
9939 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9940 talign
= DECL_ALIGN_UNIT (avar
);
9941 avar
= build_fold_addr_expr (avar
);
9942 gimplify_assign (x
, avar
, &ilist
);
9944 else if (is_gimple_reg (var
))
9946 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9947 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9948 mark_addressable (avar
);
9949 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9950 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9951 gimplify_assign (avar
, var
, &ilist
);
9952 avar
= build_fold_addr_expr (avar
);
9953 gimplify_assign (x
, avar
, &ilist
);
9954 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9955 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9956 && !TYPE_READONLY (TREE_TYPE (var
)))
9958 x
= build_sender_ref (ovar
, ctx
);
9959 x
= build_simple_mem_ref (x
);
9960 gimplify_assign (var
, x
, &olist
);
9965 var
= build_fold_addr_expr (var
);
9966 gimplify_assign (x
, var
, &ilist
);
9969 tree s
= OMP_CLAUSE_SIZE (c
);
9971 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9972 s
= fold_convert (size_type_node
, s
);
9973 tree purpose
= size_int (map_idx
++);
9974 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9975 if (TREE_CODE (s
) != INTEGER_CST
)
9976 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9978 unsigned char tkind
= 0;
9979 switch (OMP_CLAUSE_CODE (c
))
9981 case OMP_CLAUSE_MAP
:
9982 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9985 tkind
= OMP_CLAUSE_MAP_TO
;
9987 case OMP_CLAUSE_FROM
:
9988 tkind
= OMP_CLAUSE_MAP_FROM
;
9993 talign
= ceil_log2 (talign
);
9994 tkind
|= talign
<< 3;
9995 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9996 build_int_cst (unsigned_char_type_node
,
10002 gcc_assert (map_idx
== map_cnt
);
10004 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
10005 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
10006 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
10007 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
10008 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
10010 gimple_seq initlist
= NULL
;
10011 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
10012 TREE_VEC_ELT (t
, 1)),
10013 &initlist
, true, NULL_TREE
);
10014 gimple_seq_add_seq (&ilist
, initlist
);
10016 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
10018 TREE_THIS_VOLATILE (clobber
) = 1;
10019 gimple_seq_add_stmt (&olist
,
10020 gimple_build_assign (TREE_VEC_ELT (t
, 1),
10024 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10025 TREE_THIS_VOLATILE (clobber
) = 1;
10026 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
10030 /* Once all the expansions are done, sequence all the different
10031 fragments inside gimple_omp_body. */
10035 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
10037 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10038 /* fixup_child_record_type might have changed receiver_decl's type. */
10039 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10040 gimple_seq_add_stmt (&new_body
,
10041 gimple_build_assign (ctx
->receiver_decl
, t
));
10044 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10046 gimple_seq_add_seq (&new_body
, tgt_body
);
10047 new_body
= maybe_catch_exception (new_body
);
10049 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10050 new_body
= tgt_body
;
10051 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
10053 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10054 gimple_omp_set_body (stmt
, new_body
);
10057 bind
= gimple_build_bind (NULL
, NULL
,
10058 tgt_bind
? gimple_bind_block (tgt_bind
)
10060 gsi_replace (gsi_p
, bind
, true);
10061 gimple_bind_add_seq (bind
, ilist
);
10062 gimple_bind_add_stmt (bind
, stmt
);
10063 gimple_bind_add_seq (bind
, olist
);
10065 pop_gimplify_context (NULL
);
10068 /* Expand code for an OpenMP teams directive. */
10071 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10073 gimple teams_stmt
= gsi_stmt (*gsi_p
);
10074 push_gimplify_context ();
10076 tree block
= make_node (BLOCK
);
10077 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
10078 gsi_replace (gsi_p
, bind
, true);
10079 gimple_seq bind_body
= NULL
;
10080 gimple_seq dlist
= NULL
;
10081 gimple_seq olist
= NULL
;
10083 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10084 OMP_CLAUSE_NUM_TEAMS
);
10085 if (num_teams
== NULL_TREE
)
10086 num_teams
= build_int_cst (unsigned_type_node
, 0);
10089 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
10090 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
10091 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
10093 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10094 OMP_CLAUSE_THREAD_LIMIT
);
10095 if (thread_limit
== NULL_TREE
)
10096 thread_limit
= build_int_cst (unsigned_type_node
, 0);
10099 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
10100 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
10101 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
10105 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
10106 &bind_body
, &dlist
, ctx
, NULL
);
10107 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
10108 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
10109 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
10111 location_t loc
= gimple_location (teams_stmt
);
10112 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
10113 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
10114 gimple_set_location (call
, loc
);
10115 gimple_seq_add_stmt (&bind_body
, call
);
10117 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10118 gimple_omp_set_body (teams_stmt
, NULL
);
10119 gimple_seq_add_seq (&bind_body
, olist
);
10120 gimple_seq_add_seq (&bind_body
, dlist
);
10121 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10122 gimple_bind_set_body (bind
, bind_body
);
10124 pop_gimplify_context (bind
);
10126 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10127 BLOCK_VARS (block
) = ctx
->block_vars
;
10128 if (BLOCK_VARS (block
))
10129 TREE_USED (block
) = 1;
10133 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10134 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10135 of OpenMP context, but with task_shared_vars set. */
10138 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10143 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10144 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10147 if (task_shared_vars
10149 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10152 /* If a global variable has been privatized, TREE_CONSTANT on
10153 ADDR_EXPR might be wrong. */
10154 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10155 recompute_tree_invariant_for_addr_expr (t
);
10157 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10162 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10164 gimple stmt
= gsi_stmt (*gsi_p
);
10165 struct walk_stmt_info wi
;
10167 if (gimple_has_location (stmt
))
10168 input_location
= gimple_location (stmt
);
10170 if (task_shared_vars
)
10171 memset (&wi
, '\0', sizeof (wi
));
10173 /* If we have issued syntax errors, avoid doing any heavy lifting.
10174 Just replace the OpenMP directives with a NOP to avoid
10175 confusing RTL expansion. */
10176 if (seen_error () && is_gimple_omp (stmt
))
10178 gsi_replace (gsi_p
, gimple_build_nop (), true);
10182 switch (gimple_code (stmt
))
10185 if ((ctx
|| task_shared_vars
)
10186 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10187 ctx
? NULL
: &wi
, NULL
)
10188 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10189 ctx
? NULL
: &wi
, NULL
)))
10190 gimple_regimplify_operands (stmt
, gsi_p
);
10193 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10195 case GIMPLE_EH_FILTER
:
10196 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10199 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10200 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10202 case GIMPLE_TRANSACTION
:
10203 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10206 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10208 case GIMPLE_OMP_PARALLEL
:
10209 case GIMPLE_OMP_TASK
:
10210 ctx
= maybe_lookup_ctx (stmt
);
10212 if (ctx
->cancellable
)
10213 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10214 lower_omp_taskreg (gsi_p
, ctx
);
10216 case GIMPLE_OMP_FOR
:
10217 ctx
= maybe_lookup_ctx (stmt
);
10219 if (ctx
->cancellable
)
10220 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10221 lower_omp_for (gsi_p
, ctx
);
10223 case GIMPLE_OMP_SECTIONS
:
10224 ctx
= maybe_lookup_ctx (stmt
);
10226 if (ctx
->cancellable
)
10227 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10228 lower_omp_sections (gsi_p
, ctx
);
10230 case GIMPLE_OMP_SINGLE
:
10231 ctx
= maybe_lookup_ctx (stmt
);
10233 lower_omp_single (gsi_p
, ctx
);
10235 case GIMPLE_OMP_MASTER
:
10236 ctx
= maybe_lookup_ctx (stmt
);
10238 lower_omp_master (gsi_p
, ctx
);
10240 case GIMPLE_OMP_TASKGROUP
:
10241 ctx
= maybe_lookup_ctx (stmt
);
10243 lower_omp_taskgroup (gsi_p
, ctx
);
10245 case GIMPLE_OMP_ORDERED
:
10246 ctx
= maybe_lookup_ctx (stmt
);
10248 lower_omp_ordered (gsi_p
, ctx
);
10250 case GIMPLE_OMP_CRITICAL
:
10251 ctx
= maybe_lookup_ctx (stmt
);
10253 lower_omp_critical (gsi_p
, ctx
);
10255 case GIMPLE_OMP_ATOMIC_LOAD
:
10256 if ((ctx
|| task_shared_vars
)
10257 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10258 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10259 gimple_regimplify_operands (stmt
, gsi_p
);
10261 case GIMPLE_OMP_TARGET
:
10262 ctx
= maybe_lookup_ctx (stmt
);
10264 lower_omp_target (gsi_p
, ctx
);
10266 case GIMPLE_OMP_TEAMS
:
10267 ctx
= maybe_lookup_ctx (stmt
);
10269 lower_omp_teams (gsi_p
, ctx
);
10273 fndecl
= gimple_call_fndecl (stmt
);
10275 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10276 switch (DECL_FUNCTION_CODE (fndecl
))
10278 case BUILT_IN_GOMP_BARRIER
:
10282 case BUILT_IN_GOMP_CANCEL
:
10283 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10286 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10287 cctx
= cctx
->outer
;
10288 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10289 if (!cctx
->cancellable
)
10291 if (DECL_FUNCTION_CODE (fndecl
)
10292 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10294 stmt
= gimple_build_nop ();
10295 gsi_replace (gsi_p
, stmt
, false);
10299 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10301 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10302 gimple_call_set_fndecl (stmt
, fndecl
);
10303 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10306 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10307 gimple_call_set_lhs (stmt
, lhs
);
10308 tree fallthru_label
;
10309 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10311 g
= gimple_build_label (fallthru_label
);
10312 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10313 g
= gimple_build_cond (NE_EXPR
, lhs
,
10314 fold_convert (TREE_TYPE (lhs
),
10315 boolean_false_node
),
10316 cctx
->cancel_label
, fallthru_label
);
10317 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10324 if ((ctx
|| task_shared_vars
)
10325 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10328 /* Just remove clobbers, this should happen only if we have
10329 "privatized" local addressable variables in SIMD regions,
10330 the clobber isn't needed in that case and gimplifying address
10331 of the ARRAY_REF into a pointer and creating MEM_REF based
10332 clobber would create worse code than we get with the clobber
10334 if (gimple_clobber_p (stmt
))
10336 gsi_replace (gsi_p
, gimple_build_nop (), true);
10339 gimple_regimplify_operands (stmt
, gsi_p
);
10346 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10348 location_t saved_location
= input_location
;
10349 gimple_stmt_iterator gsi
;
10350 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10351 lower_omp_1 (&gsi
, ctx
);
10352 /* During gimplification, we have not always invoked fold_stmt
10353 (gimplify.c:maybe_fold_stmt); call it now. */
10354 if (target_nesting_level
)
10355 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10357 input_location
= saved_location
;
10360 /* Main entry point. */
10362 static unsigned int
10363 execute_lower_omp (void)
10369 /* This pass always runs, to provide PROP_gimple_lomp.
10370 But there is nothing to do unless -fopenmp is given. */
10371 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10374 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10375 delete_omp_context
);
10377 body
= gimple_body (current_function_decl
);
10378 scan_omp (&body
, NULL
);
10379 gcc_assert (taskreg_nesting_level
== 0);
10380 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
10381 finish_taskreg_scan (ctx
);
10382 taskreg_contexts
.release ();
10384 if (all_contexts
->root
)
10386 if (task_shared_vars
)
10387 push_gimplify_context ();
10388 lower_omp (&body
, NULL
);
10389 if (task_shared_vars
)
10390 pop_gimplify_context (NULL
);
10395 splay_tree_delete (all_contexts
);
10396 all_contexts
= NULL
;
10398 BITMAP_FREE (task_shared_vars
);
10404 const pass_data pass_data_lower_omp
=
10406 GIMPLE_PASS
, /* type */
10407 "omplower", /* name */
10408 OPTGROUP_NONE
, /* optinfo_flags */
10409 false, /* has_gate */
10410 true, /* has_execute */
10411 TV_NONE
, /* tv_id */
10412 PROP_gimple_any
, /* properties_required */
10413 PROP_gimple_lomp
, /* properties_provided */
10414 0, /* properties_destroyed */
10415 0, /* todo_flags_start */
10416 0, /* todo_flags_finish */
10419 class pass_lower_omp
: public gimple_opt_pass
10422 pass_lower_omp (gcc::context
*ctxt
)
10423 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10426 /* opt_pass methods: */
10427 unsigned int execute () { return execute_lower_omp (); }
10429 }; // class pass_lower_omp
10431 } // anon namespace
10434 make_pass_lower_omp (gcc::context
*ctxt
)
10436 return new pass_lower_omp (ctxt
);
10439 /* The following is a utility to diagnose OpenMP structured block violations.
10440 It is not part of the "omplower" pass, as that's invoked too late. It
10441 should be invoked by the respective front ends after gimplification. */
10443 static splay_tree all_labels
;
10445 /* Check for mismatched contexts and generate an error if needed. Return
10446 true if an error is detected. */
10449 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10450 gimple branch_ctx
, gimple label_ctx
)
10452 if (label_ctx
== branch_ctx
)
10457 Previously we kept track of the label's entire context in diagnose_sb_[12]
10458 so we could traverse it and issue a correct "exit" or "enter" error
10459 message upon a structured block violation.
10461 We built the context by building a list with tree_cons'ing, but there is
10462 no easy counterpart in gimple tuples. It seems like far too much work
10463 for issuing exit/enter error messages. If someone really misses the
10464 distinct error message... patches welcome.
10468 /* Try to avoid confusing the user by producing and error message
10469 with correct "exit" or "enter" verbiage. We prefer "exit"
10470 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10471 if (branch_ctx
== NULL
)
10477 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10482 label_ctx
= TREE_CHAIN (label_ctx
);
10487 error ("invalid exit from OpenMP structured block");
10489 error ("invalid entry to OpenMP structured block");
10492 bool cilkplus_block
= false;
10496 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10497 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10499 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10500 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10501 cilkplus_block
= true;
10504 /* If it's obvious we have an invalid entry, be specific about the error. */
10505 if (branch_ctx
== NULL
)
10507 if (cilkplus_block
)
10508 error ("invalid entry to Cilk Plus structured block");
10510 error ("invalid entry to OpenMP structured block");
10514 /* Otherwise, be vague and lazy, but efficient. */
10515 if (cilkplus_block
)
10516 error ("invalid branch to/from a Cilk Plus structured block");
10518 error ("invalid branch to/from an OpenMP structured block");
10521 gsi_replace (gsi_p
, gimple_build_nop (), false);
10525 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10526 where each label is found. */
10529 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10530 struct walk_stmt_info
*wi
)
10532 gimple context
= (gimple
) wi
->info
;
10533 gimple inner_context
;
10534 gimple stmt
= gsi_stmt (*gsi_p
);
10536 *handled_ops_p
= true;
10538 switch (gimple_code (stmt
))
10542 case GIMPLE_OMP_PARALLEL
:
10543 case GIMPLE_OMP_TASK
:
10544 case GIMPLE_OMP_SECTIONS
:
10545 case GIMPLE_OMP_SINGLE
:
10546 case GIMPLE_OMP_SECTION
:
10547 case GIMPLE_OMP_MASTER
:
10548 case GIMPLE_OMP_ORDERED
:
10549 case GIMPLE_OMP_CRITICAL
:
10550 case GIMPLE_OMP_TARGET
:
10551 case GIMPLE_OMP_TEAMS
:
10552 case GIMPLE_OMP_TASKGROUP
:
10553 /* The minimal context here is just the current OMP construct. */
10554 inner_context
= stmt
;
10555 wi
->info
= inner_context
;
10556 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10557 wi
->info
= context
;
10560 case GIMPLE_OMP_FOR
:
10561 inner_context
= stmt
;
10562 wi
->info
= inner_context
;
10563 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10565 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10566 diagnose_sb_1
, NULL
, wi
);
10567 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10568 wi
->info
= context
;
10572 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10573 (splay_tree_value
) context
);
10583 /* Pass 2: Check each branch and see if its context differs from that of
10584 the destination label's context. */
10587 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10588 struct walk_stmt_info
*wi
)
10590 gimple context
= (gimple
) wi
->info
;
10592 gimple stmt
= gsi_stmt (*gsi_p
);
10594 *handled_ops_p
= true;
10596 switch (gimple_code (stmt
))
10600 case GIMPLE_OMP_PARALLEL
:
10601 case GIMPLE_OMP_TASK
:
10602 case GIMPLE_OMP_SECTIONS
:
10603 case GIMPLE_OMP_SINGLE
:
10604 case GIMPLE_OMP_SECTION
:
10605 case GIMPLE_OMP_MASTER
:
10606 case GIMPLE_OMP_ORDERED
:
10607 case GIMPLE_OMP_CRITICAL
:
10608 case GIMPLE_OMP_TARGET
:
10609 case GIMPLE_OMP_TEAMS
:
10610 case GIMPLE_OMP_TASKGROUP
:
10612 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10613 wi
->info
= context
;
10616 case GIMPLE_OMP_FOR
:
10618 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10620 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10621 diagnose_sb_2
, NULL
, wi
);
10622 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10623 wi
->info
= context
;
10628 tree lab
= gimple_cond_true_label (stmt
);
10631 n
= splay_tree_lookup (all_labels
,
10632 (splay_tree_key
) lab
);
10633 diagnose_sb_0 (gsi_p
, context
,
10634 n
? (gimple
) n
->value
: NULL
);
10636 lab
= gimple_cond_false_label (stmt
);
10639 n
= splay_tree_lookup (all_labels
,
10640 (splay_tree_key
) lab
);
10641 diagnose_sb_0 (gsi_p
, context
,
10642 n
? (gimple
) n
->value
: NULL
);
10649 tree lab
= gimple_goto_dest (stmt
);
10650 if (TREE_CODE (lab
) != LABEL_DECL
)
10653 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10654 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10658 case GIMPLE_SWITCH
:
10661 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10663 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10664 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10665 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10671 case GIMPLE_RETURN
:
10672 diagnose_sb_0 (gsi_p
, context
, NULL
);
10682 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10685 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10688 gimple last
= last_stmt (bb
);
10689 enum gimple_code code
= gimple_code (last
);
10690 struct omp_region
*cur_region
= *region
;
10691 bool fallthru
= false;
10695 case GIMPLE_OMP_PARALLEL
:
10696 case GIMPLE_OMP_TASK
:
10697 case GIMPLE_OMP_FOR
:
10698 case GIMPLE_OMP_SINGLE
:
10699 case GIMPLE_OMP_TEAMS
:
10700 case GIMPLE_OMP_MASTER
:
10701 case GIMPLE_OMP_TASKGROUP
:
10702 case GIMPLE_OMP_ORDERED
:
10703 case GIMPLE_OMP_CRITICAL
:
10704 case GIMPLE_OMP_SECTION
:
10705 cur_region
= new_omp_region (bb
, code
, cur_region
);
10709 case GIMPLE_OMP_TARGET
:
10710 cur_region
= new_omp_region (bb
, code
, cur_region
);
10712 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10713 cur_region
= cur_region
->outer
;
10716 case GIMPLE_OMP_SECTIONS
:
10717 cur_region
= new_omp_region (bb
, code
, cur_region
);
10721 case GIMPLE_OMP_SECTIONS_SWITCH
:
10725 case GIMPLE_OMP_ATOMIC_LOAD
:
10726 case GIMPLE_OMP_ATOMIC_STORE
:
10730 case GIMPLE_OMP_RETURN
:
10731 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10732 somewhere other than the next block. This will be
10734 cur_region
->exit
= bb
;
10735 if (cur_region
->type
== GIMPLE_OMP_TASK
)
10736 /* Add an edge corresponding to not scheduling the task
10738 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
10739 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10740 cur_region
= cur_region
->outer
;
10743 case GIMPLE_OMP_CONTINUE
:
10744 cur_region
->cont
= bb
;
10745 switch (cur_region
->type
)
10747 case GIMPLE_OMP_FOR
:
10748 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10749 succs edges as abnormal to prevent splitting
10751 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10752 /* Make the loopback edge. */
10753 make_edge (bb
, single_succ (cur_region
->entry
),
10756 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10757 corresponds to the case that the body of the loop
10758 is not executed at all. */
10759 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10760 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10764 case GIMPLE_OMP_SECTIONS
:
10765 /* Wire up the edges into and out of the nested sections. */
10767 basic_block switch_bb
= single_succ (cur_region
->entry
);
10769 struct omp_region
*i
;
10770 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10772 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10773 make_edge (switch_bb
, i
->entry
, 0);
10774 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10777 /* Make the loopback edge to the block with
10778 GIMPLE_OMP_SECTIONS_SWITCH. */
10779 make_edge (bb
, switch_bb
, 0);
10781 /* Make the edge from the switch to exit. */
10782 make_edge (switch_bb
, bb
->next_bb
, 0);
10787 case GIMPLE_OMP_TASK
:
10792 gcc_unreachable ();
10797 gcc_unreachable ();
10800 if (*region
!= cur_region
)
10802 *region
= cur_region
;
10804 *region_idx
= cur_region
->entry
->index
;
10812 static unsigned int
10813 diagnose_omp_structured_block_errors (void)
10815 struct walk_stmt_info wi
;
10816 gimple_seq body
= gimple_body (current_function_decl
);
10818 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10820 memset (&wi
, 0, sizeof (wi
));
10821 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10823 memset (&wi
, 0, sizeof (wi
));
10824 wi
.want_locations
= true;
10825 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10827 gimple_set_body (current_function_decl
, body
);
10829 splay_tree_delete (all_labels
);
10836 gate_diagnose_omp_blocks (void)
10838 return flag_openmp
|| flag_cilkplus
;
10843 const pass_data pass_data_diagnose_omp_blocks
=
10845 GIMPLE_PASS
, /* type */
10846 "*diagnose_omp_blocks", /* name */
10847 OPTGROUP_NONE
, /* optinfo_flags */
10848 true, /* has_gate */
10849 true, /* has_execute */
10850 TV_NONE
, /* tv_id */
10851 PROP_gimple_any
, /* properties_required */
10852 0, /* properties_provided */
10853 0, /* properties_destroyed */
10854 0, /* todo_flags_start */
10855 0, /* todo_flags_finish */
10858 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10861 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10862 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10865 /* opt_pass methods: */
10866 bool gate () { return gate_diagnose_omp_blocks (); }
10867 unsigned int execute () {
10868 return diagnose_omp_structured_block_errors ();
10871 }; // class pass_diagnose_omp_blocks
10873 } // anon namespace
10876 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10878 return new pass_diagnose_omp_blocks (ctxt
);
10881 /* SIMD clone supporting code. */
10883 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10884 of arguments to reserve space for. */
10886 static struct cgraph_simd_clone
*
10887 simd_clone_struct_alloc (int nargs
)
10889 struct cgraph_simd_clone
*clone_info
;
10890 size_t len
= (sizeof (struct cgraph_simd_clone
)
10891 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
10892 clone_info
= (struct cgraph_simd_clone
*)
10893 ggc_internal_cleared_alloc (len
);
10897 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10900 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
10901 struct cgraph_simd_clone
*from
)
10903 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
10904 + ((from
->nargs
- from
->inbranch
)
10905 * sizeof (struct cgraph_simd_clone_arg
))));
10908 /* Return vector of parameter types of function FNDECL. This uses
10909 TYPE_ARG_TYPES if available, otherwise falls back to types of
10910 DECL_ARGUMENTS types. */
10913 simd_clone_vector_of_formal_parm_types (tree fndecl
)
10915 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
10916 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
10917 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
10920 FOR_EACH_VEC_ELT (args
, i
, arg
)
10921 args
[i
] = TREE_TYPE (args
[i
]);
10925 /* Given a simd function in NODE, extract the simd specific
10926 information from the OMP clauses passed in CLAUSES, and return
10927 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10928 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10929 otherwise set to FALSE. */
10931 static struct cgraph_simd_clone
*
10932 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
10933 bool *inbranch_specified
)
10935 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
10938 *inbranch_specified
= false;
10940 n
= args
.length ();
10941 if (n
> 0 && args
.last () == void_type_node
)
10944 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10945 be cloned have a distinctive artificial label in addition to "omp
10949 && lookup_attribute ("cilk simd function",
10950 DECL_ATTRIBUTES (node
->decl
)));
10952 /* Allocate one more than needed just in case this is an in-branch
10953 clone which will require a mask argument. */
10954 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
10955 clone_info
->nargs
= n
;
10956 clone_info
->cilk_elemental
= cilk_clone
;
10963 clauses
= TREE_VALUE (clauses
);
10964 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
10967 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
10969 switch (OMP_CLAUSE_CODE (t
))
10971 case OMP_CLAUSE_INBRANCH
:
10972 clone_info
->inbranch
= 1;
10973 *inbranch_specified
= true;
10975 case OMP_CLAUSE_NOTINBRANCH
:
10976 clone_info
->inbranch
= 0;
10977 *inbranch_specified
= true;
10979 case OMP_CLAUSE_SIMDLEN
:
10980 clone_info
->simdlen
10981 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
10983 case OMP_CLAUSE_LINEAR
:
10985 tree decl
= OMP_CLAUSE_DECL (t
);
10986 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
10987 int argno
= TREE_INT_CST_LOW (decl
);
10988 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
10990 clone_info
->args
[argno
].arg_type
10991 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
10992 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10993 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
10994 && clone_info
->args
[argno
].linear_step
< n
);
10998 if (POINTER_TYPE_P (args
[argno
]))
10999 step
= fold_convert (ssizetype
, step
);
11000 if (!tree_fits_shwi_p (step
))
11002 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11003 "ignoring large linear step");
11007 else if (integer_zerop (step
))
11009 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11010 "ignoring zero linear step");
11016 clone_info
->args
[argno
].arg_type
11017 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
11018 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11023 case OMP_CLAUSE_UNIFORM
:
11025 tree decl
= OMP_CLAUSE_DECL (t
);
11026 int argno
= tree_to_uhwi (decl
);
11027 clone_info
->args
[argno
].arg_type
11028 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
11031 case OMP_CLAUSE_ALIGNED
:
11033 tree decl
= OMP_CLAUSE_DECL (t
);
11034 int argno
= tree_to_uhwi (decl
);
11035 clone_info
->args
[argno
].alignment
11036 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
11047 /* Given a SIMD clone in NODE, calculate the characteristic data
11048 type and return the coresponding type. The characteristic data
11049 type is computed as described in the Intel Vector ABI. */
11052 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
11053 struct cgraph_simd_clone
*clone_info
)
11055 tree type
= integer_type_node
;
11056 tree fndecl
= node
->decl
;
11058 /* a) For non-void function, the characteristic data type is the
11060 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
11061 type
= TREE_TYPE (TREE_TYPE (fndecl
));
11063 /* b) If the function has any non-uniform, non-linear parameters,
11064 then the characteristic data type is the type of the first
11068 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
11069 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
11070 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
11078 /* c) If the characteristic data type determined by a) or b) above
11079 is struct, union, or class type which is pass-by-value (except
11080 for the type that maps to the built-in complex data type), the
11081 characteristic data type is int. */
11082 if (RECORD_OR_UNION_TYPE_P (type
)
11083 && !aggregate_value_p (type
, NULL
)
11084 && TREE_CODE (type
) != COMPLEX_TYPE
)
11085 return integer_type_node
;
11087 /* d) If none of the above three classes is applicable, the
11088 characteristic data type is int. */
11092 /* e) For Intel Xeon Phi native and offload compilation, if the
11093 resulting characteristic data type is 8-bit or 16-bit integer
11094 data type, the characteristic data type is int. */
11095 /* Well, we don't handle Xeon Phi yet. */
11099 simd_clone_mangle (struct cgraph_node
*node
,
11100 struct cgraph_simd_clone
*clone_info
)
11102 char vecsize_mangle
= clone_info
->vecsize_mangle
;
11103 char mask
= clone_info
->inbranch
? 'M' : 'N';
11104 unsigned int simdlen
= clone_info
->simdlen
;
11108 gcc_assert (vecsize_mangle
&& simdlen
);
11110 pp_string (&pp
, "_ZGV");
11111 pp_character (&pp
, vecsize_mangle
);
11112 pp_character (&pp
, mask
);
11113 pp_decimal_int (&pp
, simdlen
);
11115 for (n
= 0; n
< clone_info
->nargs
; ++n
)
11117 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
11119 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
11120 pp_character (&pp
, 'u');
11121 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11123 gcc_assert (arg
.linear_step
!= 0);
11124 pp_character (&pp
, 'l');
11125 if (arg
.linear_step
> 1)
11126 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11127 else if (arg
.linear_step
< 0)
11129 pp_character (&pp
, 'n');
11130 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
11134 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11136 pp_character (&pp
, 's');
11137 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11140 pp_character (&pp
, 'v');
11143 pp_character (&pp
, 'a');
11144 pp_decimal_int (&pp
, arg
.alignment
);
11148 pp_underscore (&pp
);
11149 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
11152 pp_string (&pp
, str
);
11153 str
= pp_formatted_text (&pp
);
11155 /* If there already is a SIMD clone with the same mangled name, don't
11156 add another one. This can happen e.g. for
11157 #pragma omp declare simd
11158 #pragma omp declare simd simdlen(8)
11159 int foo (int, int);
11160 if the simdlen is assumed to be 8 for the first one, etc. */
11161 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11162 clone
= clone
->simdclone
->next_clone
)
11163 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11167 return get_identifier (str
);
11170 /* Create a simd clone of OLD_NODE and return it. */
11172 static struct cgraph_node
*
11173 simd_clone_create (struct cgraph_node
*old_node
)
11175 struct cgraph_node
*new_node
;
11176 if (old_node
->definition
)
11178 if (!cgraph_function_with_gimple_body_p (old_node
))
11180 cgraph_get_body (old_node
);
11181 new_node
= cgraph_function_versioning (old_node
, vNULL
, NULL
, NULL
,
11182 false, NULL
, NULL
, "simdclone");
11186 tree old_decl
= old_node
->decl
;
11187 tree new_decl
= copy_node (old_node
->decl
);
11188 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11189 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11190 SET_DECL_RTL (new_decl
, NULL
);
11191 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11192 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11194 = cgraph_copy_node_for_versioning (old_node
, new_decl
, vNULL
, NULL
);
11195 cgraph_call_function_insertion_hooks (new_node
);
11197 if (new_node
== NULL
)
11200 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11202 /* The function cgraph_function_versioning () will force the new
11203 symbol local. Undo this, and inherit external visability from
11205 new_node
->local
.local
= old_node
->local
.local
;
11206 new_node
->externally_visible
= old_node
->externally_visible
;
11211 /* Adjust the return type of the given function to its appropriate
11212 vector counterpart. Returns a simd array to be used throughout the
11213 function as a return value. */
11216 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11218 tree fndecl
= node
->decl
;
11219 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11220 unsigned int veclen
;
11223 /* Adjust the function return type. */
11224 if (orig_rettype
== void_type_node
)
11226 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11227 t
= TREE_TYPE (TREE_TYPE (fndecl
));
11228 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
11229 veclen
= node
->simdclone
->vecsize_int
;
11231 veclen
= node
->simdclone
->vecsize_float
;
11232 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
11233 if (veclen
> node
->simdclone
->simdlen
)
11234 veclen
= node
->simdclone
->simdlen
;
11235 if (POINTER_TYPE_P (t
))
11236 t
= pointer_sized_int_node
;
11237 if (veclen
== node
->simdclone
->simdlen
)
11238 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
11241 t
= build_vector_type (t
, veclen
);
11242 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11244 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11245 if (!node
->definition
)
11248 t
= DECL_RESULT (fndecl
);
11249 /* Adjust the DECL_RESULT. */
11250 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11251 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11254 tree atype
= build_array_type_nelts (orig_rettype
,
11255 node
->simdclone
->simdlen
);
11256 if (veclen
!= node
->simdclone
->simdlen
)
11257 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11259 /* Set up a SIMD array to use as the return value. */
11260 tree retval
= create_tmp_var_raw (atype
, "retval");
11261 gimple_add_tmp_var (retval
);
11265 /* Each vector argument has a corresponding array to be used locally
11266 as part of the eventual loop. Create such temporary array and
11269 PREFIX is the prefix to be used for the temporary.
11271 TYPE is the inner element type.
11273 SIMDLEN is the number of elements. */
11276 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11278 tree atype
= build_array_type_nelts (type
, simdlen
);
11279 tree avar
= create_tmp_var_raw (atype
, prefix
);
11280 gimple_add_tmp_var (avar
);
11284 /* Modify the function argument types to their corresponding vector
11285 counterparts if appropriate. Also, create one array for each simd
11286 argument to be used locally when using the function arguments as
11289 NODE is the function whose arguments are to be adjusted.
11291 Returns an adjustment vector that will be filled describing how the
11292 argument types will be adjusted. */
11294 static ipa_parm_adjustment_vec
11295 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11298 ipa_parm_adjustment_vec adjustments
;
11300 if (node
->definition
)
11301 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11303 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11304 adjustments
.create (args
.length ());
11305 unsigned i
, j
, veclen
;
11306 struct ipa_parm_adjustment adj
;
11307 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11309 memset (&adj
, 0, sizeof (adj
));
11310 tree parm
= args
[i
];
11311 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11312 adj
.base_index
= i
;
11315 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11316 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11318 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11320 /* No adjustment necessary for scalar arguments. */
11321 adj
.op
= IPA_PARM_OP_COPY
;
11325 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11326 veclen
= node
->simdclone
->vecsize_int
;
11328 veclen
= node
->simdclone
->vecsize_float
;
11329 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11330 if (veclen
> node
->simdclone
->simdlen
)
11331 veclen
= node
->simdclone
->simdlen
;
11332 adj
.arg_prefix
= "simd";
11333 if (POINTER_TYPE_P (parm_type
))
11334 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
11336 adj
.type
= build_vector_type (parm_type
, veclen
);
11337 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11338 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11340 adjustments
.safe_push (adj
);
11343 memset (&adj
, 0, sizeof (adj
));
11344 adj
.op
= IPA_PARM_OP_NEW
;
11345 adj
.arg_prefix
= "simd";
11346 adj
.base_index
= i
;
11347 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11351 if (node
->definition
)
11352 node
->simdclone
->args
[i
].simd_array
11353 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11354 parm_type
, node
->simdclone
->simdlen
);
11356 adjustments
.safe_push (adj
);
11359 if (node
->simdclone
->inbranch
)
11362 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11365 memset (&adj
, 0, sizeof (adj
));
11366 adj
.op
= IPA_PARM_OP_NEW
;
11367 adj
.arg_prefix
= "mask";
11369 adj
.base_index
= i
;
11370 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11371 veclen
= node
->simdclone
->vecsize_int
;
11373 veclen
= node
->simdclone
->vecsize_float
;
11374 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11375 if (veclen
> node
->simdclone
->simdlen
)
11376 veclen
= node
->simdclone
->simdlen
;
11377 if (POINTER_TYPE_P (base_type
))
11378 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
11380 adj
.type
= build_vector_type (base_type
, veclen
);
11381 adjustments
.safe_push (adj
);
11383 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11384 adjustments
.safe_push (adj
);
11386 /* We have previously allocated one extra entry for the mask. Use
11388 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11390 if (node
->definition
)
11392 sc
->args
[i
].orig_arg
11393 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11394 sc
->args
[i
].simd_array
11395 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11397 sc
->args
[i
].orig_type
= base_type
;
11398 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11401 if (node
->definition
)
11402 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11405 tree new_arg_types
= NULL_TREE
, new_reversed
;
11406 bool last_parm_void
= false;
11407 if (args
.length () > 0 && args
.last () == void_type_node
)
11408 last_parm_void
= true;
11410 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11411 j
= adjustments
.length ();
11412 for (i
= 0; i
< j
; i
++)
11414 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11416 if (adj
->op
== IPA_PARM_OP_COPY
)
11417 ptype
= args
[adj
->base_index
];
11420 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11422 new_reversed
= nreverse (new_arg_types
);
11423 if (last_parm_void
)
11426 TREE_CHAIN (new_arg_types
) = void_list_node
;
11428 new_reversed
= void_list_node
;
11431 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11432 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11433 TREE_TYPE (node
->decl
) = new_type
;
11435 adjustments
.release ();
11438 return adjustments
;
11441 /* Initialize and copy the function arguments in NODE to their
11442 corresponding local simd arrays. Returns a fresh gimple_seq with
11443 the instruction sequence generated. */
11446 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11447 ipa_parm_adjustment_vec adjustments
)
11449 gimple_seq seq
= NULL
;
11450 unsigned i
= 0, j
= 0, k
;
11452 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11454 arg
= DECL_CHAIN (arg
), i
++, j
++)
11456 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11459 node
->simdclone
->args
[i
].vector_arg
= arg
;
11461 tree array
= node
->simdclone
->args
[i
].simd_array
;
11462 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11464 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11465 tree ptr
= build_fold_addr_expr (array
);
11466 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11467 build_int_cst (ptype
, 0));
11468 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11469 gimplify_and_add (t
, &seq
);
11473 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11474 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11475 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11477 tree ptr
= build_fold_addr_expr (array
);
11481 arg
= DECL_CHAIN (arg
);
11485 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11486 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11487 build_int_cst (ptype
, k
* elemsize
));
11488 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11489 gimplify_and_add (t
, &seq
);
11496 /* Callback info for ipa_simd_modify_stmt_ops below. */
11498 struct modify_stmt_info
{
11499 ipa_parm_adjustment_vec adjustments
;
11501 /* True if the parent statement was modified by
11502 ipa_simd_modify_stmt_ops. */
11506 /* Callback for walk_gimple_op.
11508 Adjust operands from a given statement as specified in the
11509 adjustments vector in the callback data. */
11512 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11514 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11515 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11516 tree
*orig_tp
= tp
;
11517 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11518 tp
= &TREE_OPERAND (*tp
, 0);
11519 struct ipa_parm_adjustment
*cand
= NULL
;
11520 if (TREE_CODE (*tp
) == PARM_DECL
)
11521 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11525 *walk_subtrees
= 0;
11528 tree repl
= NULL_TREE
;
11530 repl
= unshare_expr (cand
->new_decl
);
11535 *walk_subtrees
= 0;
11536 bool modified
= info
->modified
;
11537 info
->modified
= false;
11538 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11539 if (!info
->modified
)
11541 info
->modified
= modified
;
11544 info
->modified
= modified
;
11553 repl
= build_fold_addr_expr (repl
);
11555 if (is_gimple_debug (info
->stmt
))
11557 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
11558 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
11559 DECL_ARTIFICIAL (vexpr
) = 1;
11560 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
11561 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
11566 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
11568 repl
= gimple_assign_lhs (stmt
);
11570 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11571 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11574 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11576 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11582 info
->modified
= true;
11586 /* Traverse the function body and perform all modifications as
11587 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11588 modified such that the replacement/reduction value will now be an
11589 offset into the corresponding simd_array.
11591 This function will replace all function argument uses with their
11592 corresponding simd array elements, and ajust the return values
11596 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11597 ipa_parm_adjustment_vec adjustments
,
11598 tree retval_array
, tree iter
)
11601 unsigned int i
, j
, l
;
11603 /* Re-use the adjustments array, but this time use it to replace
11604 every function argument use to an offset into the corresponding
11606 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11608 if (!node
->simdclone
->args
[i
].vector_arg
)
11611 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11612 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11613 adjustments
[j
].new_decl
11614 = build4 (ARRAY_REF
,
11616 node
->simdclone
->args
[i
].simd_array
,
11618 NULL_TREE
, NULL_TREE
);
11619 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11620 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11621 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11624 l
= adjustments
.length ();
11625 for (i
= 1; i
< num_ssa_names
; i
++)
11627 tree name
= ssa_name (i
);
11629 && SSA_NAME_VAR (name
)
11630 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11632 for (j
= 0; j
< l
; j
++)
11633 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11634 && adjustments
[j
].new_decl
)
11637 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11640 = copy_var_decl (adjustments
[j
].base
,
11641 DECL_NAME (adjustments
[j
].base
),
11642 TREE_TYPE (adjustments
[j
].base
));
11643 adjustments
[j
].new_ssa_base
= base_var
;
11646 base_var
= adjustments
[j
].new_ssa_base
;
11647 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11649 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11650 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11651 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11652 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11653 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11654 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11655 gimple stmt
= gimple_build_assign (name
, new_decl
);
11656 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11659 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11664 struct modify_stmt_info info
;
11665 info
.adjustments
= adjustments
;
11667 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11669 gimple_stmt_iterator gsi
;
11671 gsi
= gsi_start_bb (bb
);
11672 while (!gsi_end_p (gsi
))
11674 gimple stmt
= gsi_stmt (gsi
);
11676 struct walk_stmt_info wi
;
11678 memset (&wi
, 0, sizeof (wi
));
11679 info
.modified
= false;
11681 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11683 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11685 tree retval
= gimple_return_retval (stmt
);
11688 gsi_remove (&gsi
, true);
11692 /* Replace `return foo' with `retval_array[iter] = foo'. */
11693 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11694 retval_array
, iter
, NULL
, NULL
);
11695 stmt
= gimple_build_assign (ref
, retval
);
11696 gsi_replace (&gsi
, stmt
, true);
11697 info
.modified
= true;
11702 update_stmt (stmt
);
11703 if (maybe_clean_eh_stmt (stmt
))
11704 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11711 /* Adjust the argument types in NODE to their appropriate vector
11715 simd_clone_adjust (struct cgraph_node
*node
)
11717 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11719 targetm
.simd_clone
.adjust (node
);
11721 tree retval
= simd_clone_adjust_return_type (node
);
11722 ipa_parm_adjustment_vec adjustments
11723 = simd_clone_adjust_argument_types (node
);
11725 push_gimplify_context ();
11727 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11729 /* Adjust all uses of vector arguments accordingly. Adjust all
11730 return values accordingly. */
11731 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11732 tree iter1
= make_ssa_name (iter
, NULL
);
11733 tree iter2
= make_ssa_name (iter
, NULL
);
11734 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11736 /* Initialize the iteration variable. */
11737 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11738 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11739 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11740 /* Insert the SIMD array and iv initialization at function
11742 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11744 pop_gimplify_context (NULL
);
11746 /* Create a new BB right before the original exit BB, to hold the
11747 iteration increment and the condition/branch. */
11748 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11749 basic_block incr_bb
= create_empty_bb (orig_exit
);
11750 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
11751 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11752 flag. Set it now to be a FALLTHRU_EDGE. */
11753 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11754 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11755 for (unsigned i
= 0;
11756 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
11758 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
11759 redirect_edge_succ (e
, incr_bb
);
11761 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
11762 e
->probability
= REG_BR_PROB_BASE
;
11763 gsi
= gsi_last_bb (incr_bb
);
11764 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
11765 build_int_cst (unsigned_type_node
,
11767 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11769 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11770 struct loop
*loop
= alloc_loop ();
11771 cfun
->has_force_vect_loops
= true;
11772 loop
->safelen
= node
->simdclone
->simdlen
;
11773 loop
->force_vect
= true;
11774 loop
->header
= body_bb
;
11776 /* Branch around the body if the mask applies. */
11777 if (node
->simdclone
->inbranch
)
11779 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
11781 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
11782 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
11783 tree aref
= build4 (ARRAY_REF
,
11784 TREE_TYPE (TREE_TYPE (mask_array
)),
11787 g
= gimple_build_assign (mask
, aref
);
11788 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11789 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
11790 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
11792 aref
= build1 (VIEW_CONVERT_EXPR
,
11793 build_nonstandard_integer_type (bitsize
, 0), mask
);
11794 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
11795 g
= gimple_build_assign (mask
, aref
);
11796 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11799 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
11801 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11802 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
11803 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
11806 /* Generate the condition. */
11807 g
= gimple_build_cond (LT_EXPR
,
11809 build_int_cst (unsigned_type_node
,
11810 node
->simdclone
->simdlen
),
11812 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11813 e
= split_block (incr_bb
, gsi_stmt (gsi
));
11814 basic_block latch_bb
= e
->dest
;
11815 basic_block new_exit_bb
;
11816 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
11817 loop
->latch
= latch_bb
;
11819 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
11821 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
11822 /* The successor of incr_bb is already pointing to latch_bb; just
11824 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11825 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
11827 gimple phi
= create_phi_node (iter1
, body_bb
);
11828 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
11829 edge latch_edge
= single_succ_edge (latch_bb
);
11830 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
11832 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11834 /* Generate the new return. */
11835 gsi
= gsi_last_bb (new_exit_bb
);
11837 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
11838 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
11839 retval
= TREE_OPERAND (retval
, 0);
11842 retval
= build1 (VIEW_CONVERT_EXPR
,
11843 TREE_TYPE (TREE_TYPE (node
->decl
)),
11845 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
11846 false, GSI_CONTINUE_LINKING
);
11848 g
= gimple_build_return (retval
);
11849 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11851 /* Handle aligned clauses by replacing default defs of the aligned
11852 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11853 lhs. Handle linear by adding PHIs. */
11854 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
11855 if (node
->simdclone
->args
[i
].alignment
11856 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
11857 && (node
->simdclone
->args
[i
].alignment
11858 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
11859 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
11862 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
11863 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11864 tree def
= ssa_default_def (cfun
, orig_arg
);
11865 if (def
&& !has_zero_uses (def
))
11867 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
11868 gimple_seq seq
= NULL
;
11869 bool need_cvt
= false;
11871 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
11873 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
11876 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
11877 gimple_call_set_lhs (g
, t
);
11878 gimple_seq_add_stmt_without_update (&seq
, g
);
11881 t
= make_ssa_name (orig_arg
, NULL
);
11882 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
11883 gimple_call_lhs (g
),
11885 gimple_seq_add_stmt_without_update (&seq
, g
);
11887 gsi_insert_seq_on_edge_immediate
11888 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
11890 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11891 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
11893 cgraph_create_edge (node
, cgraph_get_create_node (fn
),
11894 call
, entry_bb
->count
, freq
);
11896 imm_use_iterator iter
;
11897 use_operand_p use_p
;
11899 tree repl
= gimple_get_lhs (g
);
11900 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11901 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
11904 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11905 SET_USE (use_p
, repl
);
11908 else if (node
->simdclone
->args
[i
].arg_type
11909 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11911 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11912 tree def
= ssa_default_def (cfun
, orig_arg
);
11913 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11914 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
11915 if (def
&& !has_zero_uses (def
))
11917 iter1
= make_ssa_name (orig_arg
, NULL
);
11918 iter2
= make_ssa_name (orig_arg
, NULL
);
11919 phi
= create_phi_node (iter1
, body_bb
);
11920 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
11921 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11922 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11923 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
11924 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11925 ? TREE_TYPE (orig_arg
) : sizetype
;
11927 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
11928 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
11929 gsi
= gsi_last_bb (incr_bb
);
11930 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11932 imm_use_iterator iter
;
11933 use_operand_p use_p
;
11935 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11936 if (use_stmt
== phi
)
11939 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11940 SET_USE (use_p
, iter1
);
11944 calculate_dominance_info (CDI_DOMINATORS
);
11945 add_loop (loop
, loop
->header
->loop_father
);
11946 update_ssa (TODO_update_ssa
);
11951 /* If the function in NODE is tagged as an elemental SIMD function,
11952 create the appropriate SIMD clones. */
11955 expand_simd_clones (struct cgraph_node
*node
)
11957 tree attr
= lookup_attribute ("omp declare simd",
11958 DECL_ATTRIBUTES (node
->decl
));
11959 if (attr
== NULL_TREE
11960 || node
->global
.inlined_to
11961 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
11965 #pragma omp declare simd
11967 in C, there we don't know the argument types at all. */
11968 if (!node
->definition
11969 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
11974 /* Start with parsing the "omp declare simd" attribute(s). */
11975 bool inbranch_clause_specified
;
11976 struct cgraph_simd_clone
*clone_info
11977 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
11978 &inbranch_clause_specified
);
11979 if (clone_info
== NULL
)
11982 int orig_simdlen
= clone_info
->simdlen
;
11983 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
11984 /* The target can return 0 (no simd clones should be created),
11985 1 (just one ISA of simd clones should be created) or higher
11986 count of ISA variants. In that case, clone_info is initialized
11987 for the first ISA variant. */
11989 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
11994 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11995 also create one inbranch and one !inbranch clone of it. */
11996 for (int i
= 0; i
< count
* 2; i
++)
11998 struct cgraph_simd_clone
*clone
= clone_info
;
11999 if (inbranch_clause_specified
&& (i
& 1) != 0)
12004 clone
= simd_clone_struct_alloc (clone_info
->nargs
12006 simd_clone_struct_copy (clone
, clone_info
);
12007 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
12008 and simd_clone_adjust_argument_types did to the first
12010 clone
->nargs
-= clone_info
->inbranch
;
12011 clone
->simdlen
= orig_simdlen
;
12012 /* And call the target hook again to get the right ISA. */
12013 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
12017 clone
->inbranch
= 1;
12020 /* simd_clone_mangle might fail if such a clone has been created
12022 tree id
= simd_clone_mangle (node
, clone
);
12023 if (id
== NULL_TREE
)
12026 /* Only when we are sure we want to create the clone actually
12027 clone the function (or definitions) or create another
12028 extern FUNCTION_DECL (for prototypes without definitions). */
12029 struct cgraph_node
*n
= simd_clone_create (node
);
12033 n
->simdclone
= clone
;
12034 clone
->origin
= node
;
12035 clone
->next_clone
= NULL
;
12036 if (node
->simd_clones
== NULL
)
12038 clone
->prev_clone
= n
;
12039 node
->simd_clones
= n
;
12043 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
12044 clone
->prev_clone
->simdclone
->next_clone
= n
;
12045 node
->simd_clones
->simdclone
->prev_clone
= n
;
12047 change_decl_assembler_name (n
->decl
, id
);
12048 /* And finally adjust the return type, parameters and for
12049 definitions also function body. */
12050 if (node
->definition
)
12051 simd_clone_adjust (n
);
12054 simd_clone_adjust_return_type (n
);
12055 simd_clone_adjust_argument_types (n
);
12059 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
12062 /* Entry point for IPA simd clone creation pass. */
12064 static unsigned int
12065 ipa_omp_simd_clone (void)
12067 struct cgraph_node
*node
;
12068 FOR_EACH_FUNCTION (node
)
12069 expand_simd_clones (node
);
12075 const pass_data pass_data_omp_simd_clone
=
12077 SIMPLE_IPA_PASS
, /* type */
12078 "simdclone", /* name */
12079 OPTGROUP_NONE
, /* optinfo_flags */
12080 true, /* has_gate */
12081 true, /* has_execute */
12082 TV_NONE
, /* tv_id */
12083 ( PROP_ssa
| PROP_cfg
), /* properties_required */
12084 0, /* properties_provided */
12085 0, /* properties_destroyed */
12086 0, /* todo_flags_start */
12087 0, /* todo_flags_finish */
12090 class pass_omp_simd_clone
: public simple_ipa_opt_pass
12093 pass_omp_simd_clone(gcc::context
*ctxt
)
12094 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
12097 /* opt_pass methods: */
12098 bool gate () { return ((flag_openmp
|| flag_openmp_simd
12099 || flag_cilkplus
|| (in_lto_p
&& !flag_wpa
))
12100 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
12102 unsigned int execute () { return ipa_omp_simd_clone (); }
12105 } // anon namespace
12107 simple_ipa_opt_pass
*
12108 make_pass_omp_simd_clone (gcc::context
*ctxt
)
12110 return new pass_omp_simd_clone (ctxt
);
12113 #include "gt-omp-low.h"