1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
208 static void scan_omp (gimple_seq
*, omp_context
*);
209 static tree
scan_omp_1_op (tree
*, int *, void *);
211 #define WALK_SUBSTMTS \
215 case GIMPLE_EH_FILTER: \
216 case GIMPLE_TRANSACTION: \
217 /* The sub-statements for these should be walked. */ \
218 *handled_ops_p = false; \
221 /* Convenience function for calling scan_omp_1_op on tree operands. */
224 scan_omp_op (tree
*tp
, omp_context
*ctx
)
226 struct walk_stmt_info wi
;
228 memset (&wi
, 0, sizeof (wi
));
230 wi
.want_locations
= true;
232 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
235 static void lower_omp (gimple_seq
*, omp_context
*);
236 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
237 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
239 /* Find an OpenMP clause of type KIND within CLAUSES. */
242 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
244 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
245 if (OMP_CLAUSE_CODE (clauses
) == kind
)
251 /* Return true if CTX is for an omp parallel. */
254 is_parallel_ctx (omp_context
*ctx
)
256 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
260 /* Return true if CTX is for an omp task. */
263 is_task_ctx (omp_context
*ctx
)
265 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
269 /* Return true if CTX is for an omp parallel or omp task. */
272 is_taskreg_ctx (omp_context
*ctx
)
274 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
275 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
279 /* Return true if REGION is a combined parallel+workshare region. */
282 is_combined_parallel (struct omp_region
*region
)
284 return region
->is_combined_parallel
;
288 /* Extract the header elements of parallel loop FOR_STMT and store
292 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
293 struct omp_for_data_loop
*loops
)
295 tree t
, var
, *collapse_iter
, *collapse_count
;
296 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
297 struct omp_for_data_loop
*loop
;
299 struct omp_for_data_loop dummy_loop
;
300 location_t loc
= gimple_location (for_stmt
);
301 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
302 bool distribute
= gimple_omp_for_kind (for_stmt
)
303 == GF_OMP_FOR_KIND_DISTRIBUTE
;
305 fd
->for_stmt
= for_stmt
;
307 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
308 if (fd
->collapse
> 1)
311 fd
->loops
= &fd
->loop
;
313 fd
->have_nowait
= distribute
|| simd
;
314 fd
->have_ordered
= false;
315 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
316 fd
->chunk_size
= NULL_TREE
;
317 collapse_iter
= NULL
;
318 collapse_count
= NULL
;
320 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
321 switch (OMP_CLAUSE_CODE (t
))
323 case OMP_CLAUSE_NOWAIT
:
324 fd
->have_nowait
= true;
326 case OMP_CLAUSE_ORDERED
:
327 fd
->have_ordered
= true;
329 case OMP_CLAUSE_SCHEDULE
:
330 gcc_assert (!distribute
);
331 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
332 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
334 case OMP_CLAUSE_DIST_SCHEDULE
:
335 gcc_assert (distribute
);
336 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
338 case OMP_CLAUSE_COLLAPSE
:
339 if (fd
->collapse
> 1)
341 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
342 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
349 /* FIXME: for now map schedule(auto) to schedule(static).
350 There should be analysis to determine whether all iterations
351 are approximately the same amount of work (then schedule(static)
352 is best) or if it varies (then schedule(dynamic,N) is better). */
353 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
355 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
356 gcc_assert (fd
->chunk_size
== NULL
);
358 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
359 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
360 gcc_assert (fd
->chunk_size
== NULL
);
361 else if (fd
->chunk_size
== NULL
)
363 /* We only need to compute a default chunk size for ordered
364 static loops and dynamic loops. */
365 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
367 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
368 ? integer_zero_node
: integer_one_node
;
371 for (i
= 0; i
< fd
->collapse
; i
++)
373 if (fd
->collapse
== 1)
375 else if (loops
!= NULL
)
380 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
381 gcc_assert (SSA_VAR_P (loop
->v
));
382 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
383 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
384 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
385 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
387 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
388 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
389 switch (loop
->cond_code
)
395 gcc_assert (gimple_omp_for_kind (for_stmt
)
396 == GF_OMP_FOR_KIND_CILKSIMD
);
399 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
400 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
402 loop
->n2
= fold_build2_loc (loc
,
403 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
404 build_int_cst (TREE_TYPE (loop
->n2
), 1));
405 loop
->cond_code
= LT_EXPR
;
408 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
409 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
411 loop
->n2
= fold_build2_loc (loc
,
412 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
413 build_int_cst (TREE_TYPE (loop
->n2
), 1));
414 loop
->cond_code
= GT_EXPR
;
420 t
= gimple_omp_for_incr (for_stmt
, i
);
421 gcc_assert (TREE_OPERAND (t
, 0) == var
);
422 switch (TREE_CODE (t
))
425 loop
->step
= TREE_OPERAND (t
, 1);
427 case POINTER_PLUS_EXPR
:
428 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
431 loop
->step
= TREE_OPERAND (t
, 1);
432 loop
->step
= fold_build1_loc (loc
,
433 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
441 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
442 && !fd
->have_ordered
))
444 if (fd
->collapse
== 1)
445 iter_type
= TREE_TYPE (loop
->v
);
447 || TYPE_PRECISION (iter_type
)
448 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
450 = build_nonstandard_integer_type
451 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
453 else if (iter_type
!= long_long_unsigned_type_node
)
455 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
456 iter_type
= long_long_unsigned_type_node
;
457 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
458 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
459 >= TYPE_PRECISION (iter_type
))
463 if (loop
->cond_code
== LT_EXPR
)
464 n
= fold_build2_loc (loc
,
465 PLUS_EXPR
, TREE_TYPE (loop
->v
),
466 loop
->n2
, loop
->step
);
469 if (TREE_CODE (n
) != INTEGER_CST
470 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
471 iter_type
= long_long_unsigned_type_node
;
473 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
474 > TYPE_PRECISION (iter_type
))
478 if (loop
->cond_code
== LT_EXPR
)
481 n2
= fold_build2_loc (loc
,
482 PLUS_EXPR
, TREE_TYPE (loop
->v
),
483 loop
->n2
, loop
->step
);
487 n1
= fold_build2_loc (loc
,
488 MINUS_EXPR
, TREE_TYPE (loop
->v
),
489 loop
->n2
, loop
->step
);
492 if (TREE_CODE (n1
) != INTEGER_CST
493 || TREE_CODE (n2
) != INTEGER_CST
494 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
495 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
496 iter_type
= long_long_unsigned_type_node
;
500 if (collapse_count
&& *collapse_count
== NULL
)
502 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
503 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
504 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
505 if (t
&& integer_zerop (t
))
506 count
= build_zero_cst (long_long_unsigned_type_node
);
507 else if ((i
== 0 || count
!= NULL_TREE
)
508 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
509 && TREE_CONSTANT (loop
->n1
)
510 && TREE_CONSTANT (loop
->n2
)
511 && TREE_CODE (loop
->step
) == INTEGER_CST
)
513 tree itype
= TREE_TYPE (loop
->v
);
515 if (POINTER_TYPE_P (itype
))
516 itype
= signed_type_for (itype
);
517 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
518 t
= fold_build2_loc (loc
,
520 fold_convert_loc (loc
, itype
, loop
->step
), t
);
521 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
522 fold_convert_loc (loc
, itype
, loop
->n2
));
523 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
524 fold_convert_loc (loc
, itype
, loop
->n1
));
525 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
526 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
527 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
528 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
529 fold_convert_loc (loc
, itype
,
532 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
533 fold_convert_loc (loc
, itype
, loop
->step
));
534 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
535 if (count
!= NULL_TREE
)
536 count
= fold_build2_loc (loc
,
537 MULT_EXPR
, long_long_unsigned_type_node
,
541 if (TREE_CODE (count
) != INTEGER_CST
)
544 else if (count
&& !integer_zerop (count
))
551 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
552 || fd
->have_ordered
))
554 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
555 iter_type
= long_long_unsigned_type_node
;
557 iter_type
= long_integer_type_node
;
559 else if (collapse_iter
&& *collapse_iter
!= NULL
)
560 iter_type
= TREE_TYPE (*collapse_iter
);
561 fd
->iter_type
= iter_type
;
562 if (collapse_iter
&& *collapse_iter
== NULL
)
563 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
564 if (collapse_count
&& *collapse_count
== NULL
)
567 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
569 *collapse_count
= create_tmp_var (iter_type
, ".count");
572 if (fd
->collapse
> 1)
574 fd
->loop
.v
= *collapse_iter
;
575 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
576 fd
->loop
.n2
= *collapse_count
;
577 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
578 fd
->loop
.cond_code
= LT_EXPR
;
583 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
584 is the immediate dominator of PAR_ENTRY_BB, return true if there
585 are no data dependencies that would prevent expanding the parallel
586 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
588 When expanding a combined parallel+workshare region, the call to
589 the child function may need additional arguments in the case of
590 GIMPLE_OMP_FOR regions. In some cases, these arguments are
591 computed out of variables passed in from the parent to the child
592 via 'struct .omp_data_s'. For instance:
594 #pragma omp parallel for schedule (guided, i * 4)
599 # BLOCK 2 (PAR_ENTRY_BB)
601 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
603 # BLOCK 3 (WS_ENTRY_BB)
604 .omp_data_i = &.omp_data_o;
605 D.1667 = .omp_data_i->i;
607 #pragma omp for schedule (guided, D.1598)
609 When we outline the parallel region, the call to the child function
610 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
611 that value is computed *after* the call site. So, in principle we
612 cannot do the transformation.
614 To see whether the code in WS_ENTRY_BB blocks the combined
615 parallel+workshare call, we collect all the variables used in the
616 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
617 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
620 FIXME. If we had the SSA form built at this point, we could merely
621 hoist the code in block 3 into block 2 and be done with it. But at
622 this point we don't have dataflow information and though we could
623 hack something up here, it is really not worth the aggravation. */
626 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
628 struct omp_for_data fd
;
629 gimple ws_stmt
= last_stmt (ws_entry_bb
);
631 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
634 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
636 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
638 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
640 if (fd
.iter_type
!= long_integer_type_node
)
643 /* FIXME. We give up too easily here. If any of these arguments
644 are not constants, they will likely involve variables that have
645 been mapped into fields of .omp_data_s for sharing with the child
646 function. With appropriate data flow, it would be possible to
648 if (!is_gimple_min_invariant (fd
.loop
.n1
)
649 || !is_gimple_min_invariant (fd
.loop
.n2
)
650 || !is_gimple_min_invariant (fd
.loop
.step
)
651 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
658 /* Collect additional arguments needed to emit a combined
659 parallel+workshare call. WS_STMT is the workshare directive being
662 static vec
<tree
, va_gc
> *
663 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
666 location_t loc
= gimple_location (ws_stmt
);
667 vec
<tree
, va_gc
> *ws_args
;
669 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
671 struct omp_for_data fd
;
674 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
678 if (gimple_omp_for_combined_into_p (ws_stmt
))
681 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
682 OMP_CLAUSE__LOOPTEMP_
);
684 n1
= OMP_CLAUSE_DECL (innerc
);
685 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
686 OMP_CLAUSE__LOOPTEMP_
);
688 n2
= OMP_CLAUSE_DECL (innerc
);
691 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
693 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
694 ws_args
->quick_push (t
);
696 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
697 ws_args
->quick_push (t
);
699 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
700 ws_args
->quick_push (t
);
704 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
705 ws_args
->quick_push (t
);
710 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
712 /* Number of sections is equal to the number of edges from the
713 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
714 the exit of the sections region. */
715 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
716 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
717 vec_alloc (ws_args
, 1);
718 ws_args
->quick_push (t
);
726 /* Discover whether REGION is a combined parallel+workshare region. */
729 determine_parallel_type (struct omp_region
*region
)
731 basic_block par_entry_bb
, par_exit_bb
;
732 basic_block ws_entry_bb
, ws_exit_bb
;
734 if (region
== NULL
|| region
->inner
== NULL
735 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
736 || region
->inner
->cont
== NULL
)
739 /* We only support parallel+for and parallel+sections. */
740 if (region
->type
!= GIMPLE_OMP_PARALLEL
741 || (region
->inner
->type
!= GIMPLE_OMP_FOR
742 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
745 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
746 WS_EXIT_BB -> PAR_EXIT_BB. */
747 par_entry_bb
= region
->entry
;
748 par_exit_bb
= region
->exit
;
749 ws_entry_bb
= region
->inner
->entry
;
750 ws_exit_bb
= region
->inner
->exit
;
752 if (single_succ (par_entry_bb
) == ws_entry_bb
753 && single_succ (ws_exit_bb
) == par_exit_bb
754 && workshare_safe_to_combine_p (ws_entry_bb
)
755 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
756 || (last_and_only_stmt (ws_entry_bb
)
757 && last_and_only_stmt (par_exit_bb
))))
759 gimple par_stmt
= last_stmt (par_entry_bb
);
760 gimple ws_stmt
= last_stmt (ws_entry_bb
);
762 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
764 /* If this is a combined parallel loop, we need to determine
765 whether or not to use the combined library calls. There
766 are two cases where we do not apply the transformation:
767 static loops and any kind of ordered loop. In the first
768 case, we already open code the loop so there is no need
769 to do anything else. In the latter case, the combined
770 parallel loop call would still need extra synchronization
771 to implement ordered semantics, so there would not be any
772 gain in using the combined call. */
773 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
774 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
776 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
777 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
779 region
->is_combined_parallel
= false;
780 region
->inner
->is_combined_parallel
= false;
785 region
->is_combined_parallel
= true;
786 region
->inner
->is_combined_parallel
= true;
787 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
792 /* Return true if EXPR is variable sized. */
795 is_variable_sized (const_tree expr
)
797 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
800 /* Return true if DECL is a reference type. */
803 is_reference (tree decl
)
805 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
808 /* Lookup variables in the decl or field splay trees. The "maybe" form
809 allows for the variable form to not have been entered, otherwise we
810 assert that the variable must have been entered. */
813 lookup_decl (tree var
, omp_context
*ctx
)
815 tree
*n
= ctx
->cb
.decl_map
->get (var
);
820 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
822 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
823 return n
? *n
: NULL_TREE
;
827 lookup_field (tree var
, omp_context
*ctx
)
830 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
831 return (tree
) n
->value
;
835 lookup_sfield (tree var
, omp_context
*ctx
)
838 n
= splay_tree_lookup (ctx
->sfield_map
839 ? ctx
->sfield_map
: ctx
->field_map
,
840 (splay_tree_key
) var
);
841 return (tree
) n
->value
;
845 maybe_lookup_field (tree var
, omp_context
*ctx
)
848 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
849 return n
? (tree
) n
->value
: NULL_TREE
;
852 /* Return true if DECL should be copied by pointer. SHARED_CTX is
853 the parallel context if DECL is to be shared. */
856 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
858 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
861 /* We can only use copy-in/copy-out semantics for shared variables
862 when we know the value is not accessible from an outer scope. */
865 /* ??? Trivially accessible from anywhere. But why would we even
866 be passing an address in this case? Should we simply assert
867 this to be false, or should we have a cleanup pass that removes
868 these from the list of mappings? */
869 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
872 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
873 without analyzing the expression whether or not its location
874 is accessible to anyone else. In the case of nested parallel
875 regions it certainly may be. */
876 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
879 /* Do not use copy-in/copy-out for variables that have their
881 if (TREE_ADDRESSABLE (decl
))
884 /* lower_send_shared_vars only uses copy-in, but not copy-out
886 if (TREE_READONLY (decl
)
887 || ((TREE_CODE (decl
) == RESULT_DECL
888 || TREE_CODE (decl
) == PARM_DECL
)
889 && DECL_BY_REFERENCE (decl
)))
892 /* Disallow copy-in/out in nested parallel if
893 decl is shared in outer parallel, otherwise
894 each thread could store the shared variable
895 in its own copy-in location, making the
896 variable no longer really shared. */
897 if (shared_ctx
->is_nested
)
901 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
902 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
909 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
910 c
; c
= OMP_CLAUSE_CHAIN (c
))
911 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
912 && OMP_CLAUSE_DECL (c
) == decl
)
916 goto maybe_mark_addressable_and_ret
;
920 /* For tasks avoid using copy-in/out. As tasks can be
921 deferred or executed in different thread, when GOMP_task
922 returns, the task hasn't necessarily terminated. */
923 if (is_task_ctx (shared_ctx
))
926 maybe_mark_addressable_and_ret
:
927 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
928 if (is_gimple_reg (outer
))
930 /* Taking address of OUTER in lower_send_shared_vars
931 might need regimplification of everything that uses the
933 if (!task_shared_vars
)
934 task_shared_vars
= BITMAP_ALLOC (NULL
);
935 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
936 TREE_ADDRESSABLE (outer
) = 1;
945 /* Construct a new automatic decl similar to VAR. */
948 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
950 tree copy
= copy_var_decl (var
, name
, type
);
952 DECL_CONTEXT (copy
) = current_function_decl
;
953 DECL_CHAIN (copy
) = ctx
->block_vars
;
954 ctx
->block_vars
= copy
;
960 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
962 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
965 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
968 omp_build_component_ref (tree obj
, tree field
)
970 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
971 if (TREE_THIS_VOLATILE (field
))
972 TREE_THIS_VOLATILE (ret
) |= 1;
973 if (TREE_READONLY (field
))
974 TREE_READONLY (ret
) |= 1;
978 /* Build tree nodes to access the field for VAR on the receiver side. */
981 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
983 tree x
, field
= lookup_field (var
, ctx
);
985 /* If the receiver record type was remapped in the child function,
986 remap the field into the new record type. */
987 x
= maybe_lookup_field (field
, ctx
);
991 x
= build_simple_mem_ref (ctx
->receiver_decl
);
992 x
= omp_build_component_ref (x
, field
);
994 x
= build_simple_mem_ref (x
);
999 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1000 of a parallel, this is a component reference; for workshare constructs
1001 this is some variable. */
1004 build_outer_var_ref (tree var
, omp_context
*ctx
)
1008 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1010 else if (is_variable_sized (var
))
1012 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1013 x
= build_outer_var_ref (x
, ctx
);
1014 x
= build_simple_mem_ref (x
);
1016 else if (is_taskreg_ctx (ctx
))
1018 bool by_ref
= use_pointer_for_field (var
, NULL
);
1019 x
= build_receiver_ref (var
, by_ref
, ctx
);
1021 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1022 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1024 /* #pragma omp simd isn't a worksharing construct, and can reference even
1025 private vars in its linear etc. clauses. */
1027 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1028 x
= lookup_decl (var
, ctx
->outer
);
1029 else if (ctx
->outer
)
1030 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1034 else if (ctx
->outer
)
1035 x
= lookup_decl (var
, ctx
->outer
);
1036 else if (is_reference (var
))
1037 /* This can happen with orphaned constructs. If var is reference, it is
1038 possible it is shared and as such valid. */
1043 if (is_reference (var
))
1044 x
= build_simple_mem_ref (x
);
1049 /* Build tree nodes to access the field for VAR on the sender side. */
1052 build_sender_ref (tree var
, omp_context
*ctx
)
1054 tree field
= lookup_sfield (var
, ctx
);
1055 return omp_build_component_ref (ctx
->sender_decl
, field
);
1058 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1061 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1063 tree field
, type
, sfield
= NULL_TREE
;
1065 gcc_assert ((mask
& 1) == 0
1066 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1067 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1068 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1070 type
= TREE_TYPE (var
);
1073 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1074 type
= build_pointer_type (build_pointer_type (type
));
1077 type
= build_pointer_type (type
);
1078 else if ((mask
& 3) == 1 && is_reference (var
))
1079 type
= TREE_TYPE (type
);
1081 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1082 FIELD_DECL
, DECL_NAME (var
), type
);
1084 /* Remember what variable this field was created for. This does have a
1085 side effect of making dwarf2out ignore this member, so for helpful
1086 debugging we clear it later in delete_omp_context. */
1087 DECL_ABSTRACT_ORIGIN (field
) = var
;
1088 if (type
== TREE_TYPE (var
))
1090 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1091 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1092 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1095 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1097 if ((mask
& 3) == 3)
1099 insert_field_into_struct (ctx
->record_type
, field
);
1100 if (ctx
->srecord_type
)
1102 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1103 FIELD_DECL
, DECL_NAME (var
), type
);
1104 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1105 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1106 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1107 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1108 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1113 if (ctx
->srecord_type
== NULL_TREE
)
1117 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1118 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1119 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1121 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1122 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1123 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1124 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1125 splay_tree_insert (ctx
->sfield_map
,
1126 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1127 (splay_tree_value
) sfield
);
1131 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1132 : ctx
->srecord_type
, field
);
1136 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1137 (splay_tree_value
) field
);
1138 if ((mask
& 2) && ctx
->sfield_map
)
1139 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1140 (splay_tree_value
) sfield
);
1144 install_var_local (tree var
, omp_context
*ctx
)
1146 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1147 insert_decl_map (&ctx
->cb
, var
, new_var
);
1151 /* Adjust the replacement for DECL in CTX for the new context. This means
1152 copying the DECL_VALUE_EXPR, and fixing up the type. */
1155 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1157 tree new_decl
, size
;
1159 new_decl
= lookup_decl (decl
, ctx
);
1161 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1163 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1164 && DECL_HAS_VALUE_EXPR_P (decl
))
1166 tree ve
= DECL_VALUE_EXPR (decl
);
1167 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1168 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1169 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1172 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1174 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1175 if (size
== error_mark_node
)
1176 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1177 DECL_SIZE (new_decl
) = size
;
1179 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1180 if (size
== error_mark_node
)
1181 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1182 DECL_SIZE_UNIT (new_decl
) = size
;
1186 /* The callback for remap_decl. Search all containing contexts for a
1187 mapping of the variable; this avoids having to duplicate the splay
1188 tree ahead of time. We know a mapping doesn't already exist in the
1189 given context. Create new mappings to implement default semantics. */
1192 omp_copy_decl (tree var
, copy_body_data
*cb
)
1194 omp_context
*ctx
= (omp_context
*) cb
;
1197 if (TREE_CODE (var
) == LABEL_DECL
)
1199 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1200 DECL_CONTEXT (new_var
) = current_function_decl
;
1201 insert_decl_map (&ctx
->cb
, var
, new_var
);
1205 while (!is_taskreg_ctx (ctx
))
1210 new_var
= maybe_lookup_decl (var
, ctx
);
1215 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1218 return error_mark_node
;
1222 /* Debugging dumps for parallel regions. */
1223 void dump_omp_region (FILE *, struct omp_region
*, int);
1224 void debug_omp_region (struct omp_region
*);
1225 void debug_all_omp_regions (void);
1227 /* Dump the parallel region tree rooted at REGION. */
1230 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1232 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1233 gimple_code_name
[region
->type
]);
1236 dump_omp_region (file
, region
->inner
, indent
+ 4);
1240 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1241 region
->cont
->index
);
1245 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1246 region
->exit
->index
);
1248 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1251 dump_omp_region (file
, region
->next
, indent
);
1255 debug_omp_region (struct omp_region
*region
)
1257 dump_omp_region (stderr
, region
, 0);
1261 debug_all_omp_regions (void)
1263 dump_omp_region (stderr
, root_omp_region
, 0);
1267 /* Create a new parallel region starting at STMT inside region PARENT. */
1269 static struct omp_region
*
1270 new_omp_region (basic_block bb
, enum gimple_code type
,
1271 struct omp_region
*parent
)
1273 struct omp_region
*region
= XCNEW (struct omp_region
);
1275 region
->outer
= parent
;
1277 region
->type
= type
;
1281 /* This is a nested region. Add it to the list of inner
1282 regions in PARENT. */
1283 region
->next
= parent
->inner
;
1284 parent
->inner
= region
;
1288 /* This is a toplevel region. Add it to the list of toplevel
1289 regions in ROOT_OMP_REGION. */
1290 region
->next
= root_omp_region
;
1291 root_omp_region
= region
;
1297 /* Release the memory associated with the region tree rooted at REGION. */
1300 free_omp_region_1 (struct omp_region
*region
)
1302 struct omp_region
*i
, *n
;
1304 for (i
= region
->inner
; i
; i
= n
)
1307 free_omp_region_1 (i
);
1313 /* Release the memory for the entire omp region tree. */
1316 free_omp_regions (void)
1318 struct omp_region
*r
, *n
;
1319 for (r
= root_omp_region
; r
; r
= n
)
1322 free_omp_region_1 (r
);
1324 root_omp_region
= NULL
;
1328 /* Create a new context, with OUTER_CTX being the surrounding context. */
1330 static omp_context
*
1331 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1333 omp_context
*ctx
= XCNEW (omp_context
);
1335 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1336 (splay_tree_value
) ctx
);
1341 ctx
->outer
= outer_ctx
;
1342 ctx
->cb
= outer_ctx
->cb
;
1343 ctx
->cb
.block
= NULL
;
1344 ctx
->depth
= outer_ctx
->depth
+ 1;
1348 ctx
->cb
.src_fn
= current_function_decl
;
1349 ctx
->cb
.dst_fn
= current_function_decl
;
1350 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1351 gcc_checking_assert (ctx
->cb
.src_node
);
1352 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1353 ctx
->cb
.src_cfun
= cfun
;
1354 ctx
->cb
.copy_decl
= omp_copy_decl
;
1355 ctx
->cb
.eh_lp_nr
= 0;
1356 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1360 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1365 static gimple_seq
maybe_catch_exception (gimple_seq
);
1367 /* Finalize task copyfn. */
1370 finalize_task_copyfn (gimple task_stmt
)
1372 struct function
*child_cfun
;
1374 gimple_seq seq
= NULL
, new_seq
;
1377 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1378 if (child_fn
== NULL_TREE
)
1381 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1382 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1384 push_cfun (child_cfun
);
1385 bind
= gimplify_body (child_fn
, false);
1386 gimple_seq_add_stmt (&seq
, bind
);
1387 new_seq
= maybe_catch_exception (seq
);
1390 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1392 gimple_seq_add_stmt (&seq
, bind
);
1394 gimple_set_body (child_fn
, seq
);
1397 /* Inform the callgraph about the new function. */
1398 cgraph_node::add_new_function (child_fn
, false);
1401 /* Destroy a omp_context data structures. Called through the splay tree
1402 value delete callback. */
1405 delete_omp_context (splay_tree_value value
)
1407 omp_context
*ctx
= (omp_context
*) value
;
1409 delete ctx
->cb
.decl_map
;
1412 splay_tree_delete (ctx
->field_map
);
1413 if (ctx
->sfield_map
)
1414 splay_tree_delete (ctx
->sfield_map
);
1416 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1417 it produces corrupt debug information. */
1418 if (ctx
->record_type
)
1421 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1422 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1424 if (ctx
->srecord_type
)
1427 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1428 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1431 if (is_task_ctx (ctx
))
1432 finalize_task_copyfn (ctx
->stmt
);
1437 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1441 fixup_child_record_type (omp_context
*ctx
)
1443 tree f
, type
= ctx
->record_type
;
1445 /* ??? It isn't sufficient to just call remap_type here, because
1446 variably_modified_type_p doesn't work the way we expect for
1447 record types. Testing each field for whether it needs remapping
1448 and creating a new record by hand works, however. */
1449 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1450 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1454 tree name
, new_fields
= NULL
;
1456 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1457 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1458 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1459 TYPE_DECL
, name
, type
);
1460 TYPE_NAME (type
) = name
;
1462 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1464 tree new_f
= copy_node (f
);
1465 DECL_CONTEXT (new_f
) = type
;
1466 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1467 DECL_CHAIN (new_f
) = new_fields
;
1468 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1469 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1471 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1475 /* Arrange to be able to look up the receiver field
1476 given the sender field. */
1477 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1478 (splay_tree_value
) new_f
);
1480 TYPE_FIELDS (type
) = nreverse (new_fields
);
1484 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1487 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1488 specified by CLAUSES. */
1491 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1494 bool scan_array_reductions
= false;
1496 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1500 switch (OMP_CLAUSE_CODE (c
))
1502 case OMP_CLAUSE_PRIVATE
:
1503 decl
= OMP_CLAUSE_DECL (c
);
1504 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1506 else if (!is_variable_sized (decl
))
1507 install_var_local (decl
, ctx
);
1510 case OMP_CLAUSE_SHARED
:
1511 decl
= OMP_CLAUSE_DECL (c
);
1512 /* Ignore shared directives in teams construct. */
1513 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1515 /* Global variables don't need to be copied,
1516 the receiver side will use them directly. */
1517 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1518 if (is_global_var (odecl
))
1520 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1523 gcc_assert (is_taskreg_ctx (ctx
));
1524 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1525 || !is_variable_sized (decl
));
1526 /* Global variables don't need to be copied,
1527 the receiver side will use them directly. */
1528 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1530 by_ref
= use_pointer_for_field (decl
, ctx
);
1531 if (! TREE_READONLY (decl
)
1532 || TREE_ADDRESSABLE (decl
)
1534 || is_reference (decl
))
1536 install_var_field (decl
, by_ref
, 3, ctx
);
1537 install_var_local (decl
, ctx
);
1540 /* We don't need to copy const scalar vars back. */
1541 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1544 case OMP_CLAUSE_LASTPRIVATE
:
1545 /* Let the corresponding firstprivate clause create
1547 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1551 case OMP_CLAUSE_FIRSTPRIVATE
:
1552 case OMP_CLAUSE_REDUCTION
:
1553 case OMP_CLAUSE_LINEAR
:
1554 decl
= OMP_CLAUSE_DECL (c
);
1556 if (is_variable_sized (decl
))
1558 if (is_task_ctx (ctx
))
1559 install_var_field (decl
, false, 1, ctx
);
1562 else if (is_taskreg_ctx (ctx
))
1565 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1566 by_ref
= use_pointer_for_field (decl
, NULL
);
1568 if (is_task_ctx (ctx
)
1569 && (global
|| by_ref
|| is_reference (decl
)))
1571 install_var_field (decl
, false, 1, ctx
);
1573 install_var_field (decl
, by_ref
, 2, ctx
);
1576 install_var_field (decl
, by_ref
, 3, ctx
);
1578 install_var_local (decl
, ctx
);
1581 case OMP_CLAUSE__LOOPTEMP_
:
1582 gcc_assert (is_parallel_ctx (ctx
));
1583 decl
= OMP_CLAUSE_DECL (c
);
1584 install_var_field (decl
, false, 3, ctx
);
1585 install_var_local (decl
, ctx
);
1588 case OMP_CLAUSE_COPYPRIVATE
:
1589 case OMP_CLAUSE_COPYIN
:
1590 decl
= OMP_CLAUSE_DECL (c
);
1591 by_ref
= use_pointer_for_field (decl
, NULL
);
1592 install_var_field (decl
, by_ref
, 3, ctx
);
1595 case OMP_CLAUSE_DEFAULT
:
1596 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1599 case OMP_CLAUSE_FINAL
:
1601 case OMP_CLAUSE_NUM_THREADS
:
1602 case OMP_CLAUSE_NUM_TEAMS
:
1603 case OMP_CLAUSE_THREAD_LIMIT
:
1604 case OMP_CLAUSE_DEVICE
:
1605 case OMP_CLAUSE_SCHEDULE
:
1606 case OMP_CLAUSE_DIST_SCHEDULE
:
1607 case OMP_CLAUSE_DEPEND
:
1609 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1613 case OMP_CLAUSE_FROM
:
1614 case OMP_CLAUSE_MAP
:
1616 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1617 decl
= OMP_CLAUSE_DECL (c
);
1618 /* Global variables with "omp declare target" attribute
1619 don't need to be copied, the receiver side will use them
1621 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1623 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1624 && lookup_attribute ("omp declare target",
1625 DECL_ATTRIBUTES (decl
)))
1627 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1628 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1630 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1631 #pragma omp target data, there is nothing to map for
1633 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1634 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1639 if (DECL_SIZE (decl
)
1640 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1642 tree decl2
= DECL_VALUE_EXPR (decl
);
1643 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1644 decl2
= TREE_OPERAND (decl2
, 0);
1645 gcc_assert (DECL_P (decl2
));
1646 install_var_field (decl2
, true, 3, ctx
);
1647 install_var_local (decl2
, ctx
);
1648 install_var_local (decl
, ctx
);
1652 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1653 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1654 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1655 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1656 install_var_field (decl
, true, 7, ctx
);
1658 install_var_field (decl
, true, 3, ctx
);
1659 if (gimple_omp_target_kind (ctx
->stmt
)
1660 == GF_OMP_TARGET_KIND_REGION
)
1661 install_var_local (decl
, ctx
);
1666 tree base
= get_base_address (decl
);
1667 tree nc
= OMP_CLAUSE_CHAIN (c
);
1670 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1671 && OMP_CLAUSE_DECL (nc
) == base
1672 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1673 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1675 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1676 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1682 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1683 decl
= OMP_CLAUSE_DECL (c
);
1685 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1686 (splay_tree_key
) decl
));
1688 = build_decl (OMP_CLAUSE_LOCATION (c
),
1689 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1690 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1691 insert_field_into_struct (ctx
->record_type
, field
);
1692 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1693 (splay_tree_value
) field
);
1698 case OMP_CLAUSE_NOWAIT
:
1699 case OMP_CLAUSE_ORDERED
:
1700 case OMP_CLAUSE_COLLAPSE
:
1701 case OMP_CLAUSE_UNTIED
:
1702 case OMP_CLAUSE_MERGEABLE
:
1703 case OMP_CLAUSE_PROC_BIND
:
1704 case OMP_CLAUSE_SAFELEN
:
1707 case OMP_CLAUSE_ALIGNED
:
1708 decl
= OMP_CLAUSE_DECL (c
);
1709 if (is_global_var (decl
)
1710 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1711 install_var_local (decl
, ctx
);
1719 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1721 switch (OMP_CLAUSE_CODE (c
))
1723 case OMP_CLAUSE_LASTPRIVATE
:
1724 /* Let the corresponding firstprivate clause create
1726 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1727 scan_array_reductions
= true;
1728 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1732 case OMP_CLAUSE_PRIVATE
:
1733 case OMP_CLAUSE_FIRSTPRIVATE
:
1734 case OMP_CLAUSE_REDUCTION
:
1735 case OMP_CLAUSE_LINEAR
:
1736 decl
= OMP_CLAUSE_DECL (c
);
1737 if (is_variable_sized (decl
))
1738 install_var_local (decl
, ctx
);
1739 fixup_remapped_decl (decl
, ctx
,
1740 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1741 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1742 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1743 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1744 scan_array_reductions
= true;
1745 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1746 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1747 scan_array_reductions
= true;
1750 case OMP_CLAUSE_SHARED
:
1751 /* Ignore shared directives in teams construct. */
1752 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1754 decl
= OMP_CLAUSE_DECL (c
);
1755 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1756 fixup_remapped_decl (decl
, ctx
, false);
1759 case OMP_CLAUSE_MAP
:
1760 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1762 decl
= OMP_CLAUSE_DECL (c
);
1764 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1765 && lookup_attribute ("omp declare target",
1766 DECL_ATTRIBUTES (decl
)))
1770 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1771 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1772 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1774 tree new_decl
= lookup_decl (decl
, ctx
);
1775 TREE_TYPE (new_decl
)
1776 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1778 else if (DECL_SIZE (decl
)
1779 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1781 tree decl2
= DECL_VALUE_EXPR (decl
);
1782 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1783 decl2
= TREE_OPERAND (decl2
, 0);
1784 gcc_assert (DECL_P (decl2
));
1785 fixup_remapped_decl (decl2
, ctx
, false);
1786 fixup_remapped_decl (decl
, ctx
, true);
1789 fixup_remapped_decl (decl
, ctx
, false);
1793 case OMP_CLAUSE_COPYPRIVATE
:
1794 case OMP_CLAUSE_COPYIN
:
1795 case OMP_CLAUSE_DEFAULT
:
1797 case OMP_CLAUSE_NUM_THREADS
:
1798 case OMP_CLAUSE_NUM_TEAMS
:
1799 case OMP_CLAUSE_THREAD_LIMIT
:
1800 case OMP_CLAUSE_DEVICE
:
1801 case OMP_CLAUSE_SCHEDULE
:
1802 case OMP_CLAUSE_DIST_SCHEDULE
:
1803 case OMP_CLAUSE_NOWAIT
:
1804 case OMP_CLAUSE_ORDERED
:
1805 case OMP_CLAUSE_COLLAPSE
:
1806 case OMP_CLAUSE_UNTIED
:
1807 case OMP_CLAUSE_FINAL
:
1808 case OMP_CLAUSE_MERGEABLE
:
1809 case OMP_CLAUSE_PROC_BIND
:
1810 case OMP_CLAUSE_SAFELEN
:
1811 case OMP_CLAUSE_ALIGNED
:
1812 case OMP_CLAUSE_DEPEND
:
1813 case OMP_CLAUSE__LOOPTEMP_
:
1815 case OMP_CLAUSE_FROM
:
1823 if (scan_array_reductions
)
1824 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1825 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1826 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1828 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1829 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1831 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1832 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1833 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1834 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1835 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1836 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1839 /* Create a new name for omp child function. Returns an identifier. */
1842 create_omp_child_function_name (bool task_copy
)
1844 return (clone_function_name (current_function_decl
,
1845 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1848 /* Build a decl for the omp child function. It'll not contain a body
1849 yet, just the bare decl. */
1852 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1854 tree decl
, type
, name
, t
;
1856 name
= create_omp_child_function_name (task_copy
);
1858 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1859 ptr_type_node
, NULL_TREE
);
1861 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1863 decl
= build_decl (gimple_location (ctx
->stmt
),
1864 FUNCTION_DECL
, name
, type
);
1867 ctx
->cb
.dst_fn
= decl
;
1869 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1871 TREE_STATIC (decl
) = 1;
1872 TREE_USED (decl
) = 1;
1873 DECL_ARTIFICIAL (decl
) = 1;
1874 DECL_IGNORED_P (decl
) = 0;
1875 TREE_PUBLIC (decl
) = 0;
1876 DECL_UNINLINABLE (decl
) = 1;
1877 DECL_EXTERNAL (decl
) = 0;
1878 DECL_CONTEXT (decl
) = NULL_TREE
;
1879 DECL_INITIAL (decl
) = make_node (BLOCK
);
1880 bool target_p
= false;
1881 if (lookup_attribute ("omp declare target",
1882 DECL_ATTRIBUTES (current_function_decl
)))
1887 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1888 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1889 && gimple_omp_target_kind (octx
->stmt
)
1890 == GF_OMP_TARGET_KIND_REGION
)
1897 DECL_ATTRIBUTES (decl
)
1898 = tree_cons (get_identifier ("omp declare target"),
1899 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1901 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1902 RESULT_DECL
, NULL_TREE
, void_type_node
);
1903 DECL_ARTIFICIAL (t
) = 1;
1904 DECL_IGNORED_P (t
) = 1;
1905 DECL_CONTEXT (t
) = decl
;
1906 DECL_RESULT (decl
) = t
;
1908 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1909 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1910 DECL_ARTIFICIAL (t
) = 1;
1911 DECL_NAMELESS (t
) = 1;
1912 DECL_ARG_TYPE (t
) = ptr_type_node
;
1913 DECL_CONTEXT (t
) = current_function_decl
;
1915 DECL_ARGUMENTS (decl
) = t
;
1917 ctx
->receiver_decl
= t
;
1920 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1921 PARM_DECL
, get_identifier (".omp_data_o"),
1923 DECL_ARTIFICIAL (t
) = 1;
1924 DECL_NAMELESS (t
) = 1;
1925 DECL_ARG_TYPE (t
) = ptr_type_node
;
1926 DECL_CONTEXT (t
) = current_function_decl
;
1928 TREE_ADDRESSABLE (t
) = 1;
1929 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1930 DECL_ARGUMENTS (decl
) = t
;
1933 /* Allocate memory for the function structure. The call to
1934 allocate_struct_function clobbers CFUN, so we need to restore
1936 push_struct_function (decl
);
1937 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1941 /* Callback for walk_gimple_seq. Check if combined parallel
1942 contains gimple_omp_for_combined_into_p OMP_FOR. */
1945 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1946 bool *handled_ops_p
,
1947 struct walk_stmt_info
*wi
)
1949 gimple stmt
= gsi_stmt (*gsi_p
);
1951 *handled_ops_p
= true;
1952 switch (gimple_code (stmt
))
1956 case GIMPLE_OMP_FOR
:
1957 if (gimple_omp_for_combined_into_p (stmt
)
1958 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1961 return integer_zero_node
;
1970 /* Scan an OpenMP parallel directive. */
1973 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1977 gimple stmt
= gsi_stmt (*gsi
);
1979 /* Ignore parallel directives with empty bodies, unless there
1980 are copyin clauses. */
1982 && empty_body_p (gimple_omp_body (stmt
))
1983 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1984 OMP_CLAUSE_COPYIN
) == NULL
)
1986 gsi_replace (gsi
, gimple_build_nop (), false);
1990 if (gimple_omp_parallel_combined_p (stmt
))
1993 struct walk_stmt_info wi
;
1995 memset (&wi
, 0, sizeof (wi
));
1997 walk_gimple_seq (gimple_omp_body (stmt
),
1998 find_combined_for
, NULL
, &wi
);
1999 for_stmt
= (gimple
) wi
.info
;
2002 struct omp_for_data fd
;
2003 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2004 /* We need two temporaries with fd.loop.v type (istart/iend)
2005 and then (fd.collapse - 1) temporaries with the same
2006 type for count2 ... countN-1 vars if not constant. */
2007 size_t count
= 2, i
;
2008 tree type
= fd
.iter_type
;
2010 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2011 count
+= fd
.collapse
- 1;
2012 for (i
= 0; i
< count
; i
++)
2014 tree temp
= create_tmp_var (type
, NULL
);
2015 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2016 OMP_CLAUSE__LOOPTEMP_
);
2017 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2018 OMP_CLAUSE_DECL (c
) = temp
;
2019 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2020 gimple_omp_parallel_set_clauses (stmt
, c
);
2025 ctx
= new_omp_context (stmt
, outer_ctx
);
2026 if (taskreg_nesting_level
> 1)
2027 ctx
->is_nested
= true;
2028 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2029 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2030 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2031 name
= create_tmp_var_name (".omp_data_s");
2032 name
= build_decl (gimple_location (stmt
),
2033 TYPE_DECL
, name
, ctx
->record_type
);
2034 DECL_ARTIFICIAL (name
) = 1;
2035 DECL_NAMELESS (name
) = 1;
2036 TYPE_NAME (ctx
->record_type
) = name
;
2037 create_omp_child_function (ctx
, false);
2038 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2040 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2041 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2043 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2044 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2047 layout_type (ctx
->record_type
);
2048 fixup_child_record_type (ctx
);
2052 /* Scan an OpenMP task directive. */
2055 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2059 gimple stmt
= gsi_stmt (*gsi
);
2060 location_t loc
= gimple_location (stmt
);
2062 /* Ignore task directives with empty bodies. */
2064 && empty_body_p (gimple_omp_body (stmt
)))
2066 gsi_replace (gsi
, gimple_build_nop (), false);
2070 ctx
= new_omp_context (stmt
, outer_ctx
);
2071 if (taskreg_nesting_level
> 1)
2072 ctx
->is_nested
= true;
2073 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2074 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2075 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2076 name
= create_tmp_var_name (".omp_data_s");
2077 name
= build_decl (gimple_location (stmt
),
2078 TYPE_DECL
, name
, ctx
->record_type
);
2079 DECL_ARTIFICIAL (name
) = 1;
2080 DECL_NAMELESS (name
) = 1;
2081 TYPE_NAME (ctx
->record_type
) = name
;
2082 create_omp_child_function (ctx
, false);
2083 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2085 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2087 if (ctx
->srecord_type
)
2089 name
= create_tmp_var_name (".omp_data_a");
2090 name
= build_decl (gimple_location (stmt
),
2091 TYPE_DECL
, name
, ctx
->srecord_type
);
2092 DECL_ARTIFICIAL (name
) = 1;
2093 DECL_NAMELESS (name
) = 1;
2094 TYPE_NAME (ctx
->srecord_type
) = name
;
2095 create_omp_child_function (ctx
, true);
2098 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2100 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2102 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2103 t
= build_int_cst (long_integer_type_node
, 0);
2104 gimple_omp_task_set_arg_size (stmt
, t
);
2105 t
= build_int_cst (long_integer_type_node
, 1);
2106 gimple_omp_task_set_arg_align (stmt
, t
);
2110 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2111 /* Move VLA fields to the end. */
2112 p
= &TYPE_FIELDS (ctx
->record_type
);
2114 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2115 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2118 *p
= TREE_CHAIN (*p
);
2119 TREE_CHAIN (*q
) = NULL_TREE
;
2120 q
= &TREE_CHAIN (*q
);
2123 p
= &DECL_CHAIN (*p
);
2125 layout_type (ctx
->record_type
);
2126 fixup_child_record_type (ctx
);
2127 if (ctx
->srecord_type
)
2128 layout_type (ctx
->srecord_type
);
2129 t
= fold_convert_loc (loc
, long_integer_type_node
,
2130 TYPE_SIZE_UNIT (ctx
->record_type
));
2131 gimple_omp_task_set_arg_size (stmt
, t
);
2132 t
= build_int_cst (long_integer_type_node
,
2133 TYPE_ALIGN_UNIT (ctx
->record_type
));
2134 gimple_omp_task_set_arg_align (stmt
, t
);
2139 /* Scan an OpenMP loop directive. */
2142 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2147 ctx
= new_omp_context (stmt
, outer_ctx
);
2149 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2151 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2152 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2154 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2155 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2156 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2157 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2159 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2162 /* Scan an OpenMP sections directive. */
2165 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2169 ctx
= new_omp_context (stmt
, outer_ctx
);
2170 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2171 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2174 /* Scan an OpenMP single directive. */
2177 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2182 ctx
= new_omp_context (stmt
, outer_ctx
);
2183 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2184 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2185 name
= create_tmp_var_name (".omp_copy_s");
2186 name
= build_decl (gimple_location (stmt
),
2187 TYPE_DECL
, name
, ctx
->record_type
);
2188 TYPE_NAME (ctx
->record_type
) = name
;
2190 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2191 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2193 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2194 ctx
->record_type
= NULL
;
2196 layout_type (ctx
->record_type
);
2199 /* Scan an OpenMP target{, data, update} directive. */
2202 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2206 int kind
= gimple_omp_target_kind (stmt
);
2208 ctx
= new_omp_context (stmt
, outer_ctx
);
2209 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2210 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2211 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2212 name
= create_tmp_var_name (".omp_data_t");
2213 name
= build_decl (gimple_location (stmt
),
2214 TYPE_DECL
, name
, ctx
->record_type
);
2215 DECL_ARTIFICIAL (name
) = 1;
2216 DECL_NAMELESS (name
) = 1;
2217 TYPE_NAME (ctx
->record_type
) = name
;
2218 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2220 create_omp_child_function (ctx
, false);
2221 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2224 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2225 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2227 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2228 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2231 TYPE_FIELDS (ctx
->record_type
)
2232 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2233 #ifdef ENABLE_CHECKING
2235 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2236 for (field
= TYPE_FIELDS (ctx
->record_type
);
2238 field
= DECL_CHAIN (field
))
2239 gcc_assert (DECL_ALIGN (field
) == align
);
2241 layout_type (ctx
->record_type
);
2242 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2243 fixup_child_record_type (ctx
);
2247 /* Scan an OpenMP teams directive. */
2250 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2252 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2253 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2254 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2257 /* Check OpenMP nesting restrictions. */
2259 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2263 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2264 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2266 error_at (gimple_location (stmt
),
2267 "OpenMP constructs may not be nested inside simd region");
2270 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2272 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2273 || (gimple_omp_for_kind (stmt
)
2274 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2275 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2277 error_at (gimple_location (stmt
),
2278 "only distribute or parallel constructs are allowed to "
2279 "be closely nested inside teams construct");
2284 switch (gimple_code (stmt
))
2286 case GIMPLE_OMP_FOR
:
2287 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2289 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2291 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2293 error_at (gimple_location (stmt
),
2294 "distribute construct must be closely nested inside "
2302 if (is_gimple_call (stmt
)
2303 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2304 == BUILT_IN_GOMP_CANCEL
2305 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2306 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2308 const char *bad
= NULL
;
2309 const char *kind
= NULL
;
2312 error_at (gimple_location (stmt
), "orphaned %qs construct",
2313 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2314 == BUILT_IN_GOMP_CANCEL
2315 ? "#pragma omp cancel"
2316 : "#pragma omp cancellation point");
2319 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2320 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2324 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2325 bad
= "#pragma omp parallel";
2326 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2327 == BUILT_IN_GOMP_CANCEL
2328 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2329 ctx
->cancellable
= true;
2333 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2334 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2335 bad
= "#pragma omp for";
2336 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2337 == BUILT_IN_GOMP_CANCEL
2338 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2340 ctx
->cancellable
= true;
2341 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2343 warning_at (gimple_location (stmt
), 0,
2344 "%<#pragma omp cancel for%> inside "
2345 "%<nowait%> for construct");
2346 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2347 OMP_CLAUSE_ORDERED
))
2348 warning_at (gimple_location (stmt
), 0,
2349 "%<#pragma omp cancel for%> inside "
2350 "%<ordered%> for construct");
2355 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2356 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2357 bad
= "#pragma omp sections";
2358 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2359 == BUILT_IN_GOMP_CANCEL
2360 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2362 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2364 ctx
->cancellable
= true;
2365 if (find_omp_clause (gimple_omp_sections_clauses
2368 warning_at (gimple_location (stmt
), 0,
2369 "%<#pragma omp cancel sections%> inside "
2370 "%<nowait%> sections construct");
2374 gcc_assert (ctx
->outer
2375 && gimple_code (ctx
->outer
->stmt
)
2376 == GIMPLE_OMP_SECTIONS
);
2377 ctx
->outer
->cancellable
= true;
2378 if (find_omp_clause (gimple_omp_sections_clauses
2381 warning_at (gimple_location (stmt
), 0,
2382 "%<#pragma omp cancel sections%> inside "
2383 "%<nowait%> sections construct");
2389 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2390 bad
= "#pragma omp task";
2392 ctx
->cancellable
= true;
2396 error_at (gimple_location (stmt
), "invalid arguments");
2401 error_at (gimple_location (stmt
),
2402 "%<%s %s%> construct not closely nested inside of %qs",
2403 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2404 == BUILT_IN_GOMP_CANCEL
2405 ? "#pragma omp cancel"
2406 : "#pragma omp cancellation point", kind
, bad
);
2411 case GIMPLE_OMP_SECTIONS
:
2412 case GIMPLE_OMP_SINGLE
:
2413 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2414 switch (gimple_code (ctx
->stmt
))
2416 case GIMPLE_OMP_FOR
:
2417 case GIMPLE_OMP_SECTIONS
:
2418 case GIMPLE_OMP_SINGLE
:
2419 case GIMPLE_OMP_ORDERED
:
2420 case GIMPLE_OMP_MASTER
:
2421 case GIMPLE_OMP_TASK
:
2422 case GIMPLE_OMP_CRITICAL
:
2423 if (is_gimple_call (stmt
))
2425 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2426 != BUILT_IN_GOMP_BARRIER
)
2428 error_at (gimple_location (stmt
),
2429 "barrier region may not be closely nested inside "
2430 "of work-sharing, critical, ordered, master or "
2431 "explicit task region");
2434 error_at (gimple_location (stmt
),
2435 "work-sharing region may not be closely nested inside "
2436 "of work-sharing, critical, ordered, master or explicit "
2439 case GIMPLE_OMP_PARALLEL
:
2445 case GIMPLE_OMP_MASTER
:
2446 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2447 switch (gimple_code (ctx
->stmt
))
2449 case GIMPLE_OMP_FOR
:
2450 case GIMPLE_OMP_SECTIONS
:
2451 case GIMPLE_OMP_SINGLE
:
2452 case GIMPLE_OMP_TASK
:
2453 error_at (gimple_location (stmt
),
2454 "master region may not be closely nested inside "
2455 "of work-sharing or explicit task region");
2457 case GIMPLE_OMP_PARALLEL
:
2463 case GIMPLE_OMP_ORDERED
:
2464 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2465 switch (gimple_code (ctx
->stmt
))
2467 case GIMPLE_OMP_CRITICAL
:
2468 case GIMPLE_OMP_TASK
:
2469 error_at (gimple_location (stmt
),
2470 "ordered region may not be closely nested inside "
2471 "of critical or explicit task region");
2473 case GIMPLE_OMP_FOR
:
2474 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2475 OMP_CLAUSE_ORDERED
) == NULL
)
2477 error_at (gimple_location (stmt
),
2478 "ordered region must be closely nested inside "
2479 "a loop region with an ordered clause");
2483 case GIMPLE_OMP_PARALLEL
:
2484 error_at (gimple_location (stmt
),
2485 "ordered region must be closely nested inside "
2486 "a loop region with an ordered clause");
2492 case GIMPLE_OMP_CRITICAL
:
2493 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2494 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2495 && (gimple_omp_critical_name (stmt
)
2496 == gimple_omp_critical_name (ctx
->stmt
)))
2498 error_at (gimple_location (stmt
),
2499 "critical region may not be nested inside a critical "
2500 "region with the same name");
2504 case GIMPLE_OMP_TEAMS
:
2506 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2507 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2509 error_at (gimple_location (stmt
),
2510 "teams construct not closely nested inside of target "
2515 case GIMPLE_OMP_TARGET
:
2516 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2517 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2518 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2521 switch (gimple_omp_target_kind (stmt
))
2523 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2524 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2525 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2526 default: gcc_unreachable ();
2528 warning_at (gimple_location (stmt
), 0,
2529 "%s construct inside of target region", name
);
2539 /* Helper function scan_omp.
2541 Callback for walk_tree or operators in walk_gimple_stmt used to
2542 scan for OpenMP directives in TP. */
2545 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2547 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2548 omp_context
*ctx
= (omp_context
*) wi
->info
;
2551 switch (TREE_CODE (t
))
2558 *tp
= remap_decl (t
, &ctx
->cb
);
2562 if (ctx
&& TYPE_P (t
))
2563 *tp
= remap_type (t
, &ctx
->cb
);
2564 else if (!DECL_P (t
))
2569 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2570 if (tem
!= TREE_TYPE (t
))
2572 if (TREE_CODE (t
) == INTEGER_CST
)
2573 *tp
= wide_int_to_tree (tem
, t
);
2575 TREE_TYPE (t
) = tem
;
2585 /* Return true if FNDECL is a setjmp or a longjmp. */
2588 setjmp_or_longjmp_p (const_tree fndecl
)
2590 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2591 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2592 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2595 tree declname
= DECL_NAME (fndecl
);
2598 const char *name
= IDENTIFIER_POINTER (declname
);
2599 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2603 /* Helper function for scan_omp.
2605 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2606 the current statement in GSI. */
2609 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2610 struct walk_stmt_info
*wi
)
2612 gimple stmt
= gsi_stmt (*gsi
);
2613 omp_context
*ctx
= (omp_context
*) wi
->info
;
2615 if (gimple_has_location (stmt
))
2616 input_location
= gimple_location (stmt
);
2618 /* Check the OpenMP nesting restrictions. */
2619 bool remove
= false;
2620 if (is_gimple_omp (stmt
))
2621 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2622 else if (is_gimple_call (stmt
))
2624 tree fndecl
= gimple_call_fndecl (stmt
);
2627 if (setjmp_or_longjmp_p (fndecl
)
2629 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2630 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2633 error_at (gimple_location (stmt
),
2634 "setjmp/longjmp inside simd construct");
2636 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2637 switch (DECL_FUNCTION_CODE (fndecl
))
2639 case BUILT_IN_GOMP_BARRIER
:
2640 case BUILT_IN_GOMP_CANCEL
:
2641 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2642 case BUILT_IN_GOMP_TASKYIELD
:
2643 case BUILT_IN_GOMP_TASKWAIT
:
2644 case BUILT_IN_GOMP_TASKGROUP_START
:
2645 case BUILT_IN_GOMP_TASKGROUP_END
:
2646 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2655 stmt
= gimple_build_nop ();
2656 gsi_replace (gsi
, stmt
, false);
2659 *handled_ops_p
= true;
2661 switch (gimple_code (stmt
))
2663 case GIMPLE_OMP_PARALLEL
:
2664 taskreg_nesting_level
++;
2665 scan_omp_parallel (gsi
, ctx
);
2666 taskreg_nesting_level
--;
2669 case GIMPLE_OMP_TASK
:
2670 taskreg_nesting_level
++;
2671 scan_omp_task (gsi
, ctx
);
2672 taskreg_nesting_level
--;
2675 case GIMPLE_OMP_FOR
:
2676 scan_omp_for (stmt
, ctx
);
2679 case GIMPLE_OMP_SECTIONS
:
2680 scan_omp_sections (stmt
, ctx
);
2683 case GIMPLE_OMP_SINGLE
:
2684 scan_omp_single (stmt
, ctx
);
2687 case GIMPLE_OMP_SECTION
:
2688 case GIMPLE_OMP_MASTER
:
2689 case GIMPLE_OMP_TASKGROUP
:
2690 case GIMPLE_OMP_ORDERED
:
2691 case GIMPLE_OMP_CRITICAL
:
2692 ctx
= new_omp_context (stmt
, ctx
);
2693 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2696 case GIMPLE_OMP_TARGET
:
2697 scan_omp_target (stmt
, ctx
);
2700 case GIMPLE_OMP_TEAMS
:
2701 scan_omp_teams (stmt
, ctx
);
2708 *handled_ops_p
= false;
2710 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2711 insert_decl_map (&ctx
->cb
, var
, var
);
2715 *handled_ops_p
= false;
2723 /* Scan all the statements starting at the current statement. CTX
2724 contains context information about the OpenMP directives and
2725 clauses found during the scan. */
2728 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2730 location_t saved_location
;
2731 struct walk_stmt_info wi
;
2733 memset (&wi
, 0, sizeof (wi
));
2735 wi
.want_locations
= true;
2737 saved_location
= input_location
;
2738 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2739 input_location
= saved_location
;
2742 /* Re-gimplification and code generation routines. */
2744 /* Build a call to GOMP_barrier. */
2747 build_omp_barrier (tree lhs
)
2749 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2750 : BUILT_IN_GOMP_BARRIER
);
2751 gimple g
= gimple_build_call (fndecl
, 0);
2753 gimple_call_set_lhs (g
, lhs
);
2757 /* If a context was created for STMT when it was scanned, return it. */
2759 static omp_context
*
2760 maybe_lookup_ctx (gimple stmt
)
2763 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2764 return n
? (omp_context
*) n
->value
: NULL
;
2768 /* Find the mapping for DECL in CTX or the immediately enclosing
2769 context that has a mapping for DECL.
2771 If CTX is a nested parallel directive, we may have to use the decl
2772 mappings created in CTX's parent context. Suppose that we have the
2773 following parallel nesting (variable UIDs showed for clarity):
2776 #omp parallel shared(iD.1562) -> outer parallel
2777 iD.1562 = iD.1562 + 1;
2779 #omp parallel shared (iD.1562) -> inner parallel
2780 iD.1562 = iD.1562 - 1;
2782 Each parallel structure will create a distinct .omp_data_s structure
2783 for copying iD.1562 in/out of the directive:
2785 outer parallel .omp_data_s.1.i -> iD.1562
2786 inner parallel .omp_data_s.2.i -> iD.1562
2788 A shared variable mapping will produce a copy-out operation before
2789 the parallel directive and a copy-in operation after it. So, in
2790 this case we would have:
2793 .omp_data_o.1.i = iD.1562;
2794 #omp parallel shared(iD.1562) -> outer parallel
2795 .omp_data_i.1 = &.omp_data_o.1
2796 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2798 .omp_data_o.2.i = iD.1562; -> **
2799 #omp parallel shared(iD.1562) -> inner parallel
2800 .omp_data_i.2 = &.omp_data_o.2
2801 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2804 ** This is a problem. The symbol iD.1562 cannot be referenced
2805 inside the body of the outer parallel region. But since we are
2806 emitting this copy operation while expanding the inner parallel
2807 directive, we need to access the CTX structure of the outer
2808 parallel directive to get the correct mapping:
2810 .omp_data_o.2.i = .omp_data_i.1->i
2812 Since there may be other workshare or parallel directives enclosing
2813 the parallel directive, it may be necessary to walk up the context
2814 parent chain. This is not a problem in general because nested
2815 parallelism happens only rarely. */
2818 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2823 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2824 t
= maybe_lookup_decl (decl
, up
);
2826 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2828 return t
? t
: decl
;
2832 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2833 in outer contexts. */
2836 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2841 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2842 t
= maybe_lookup_decl (decl
, up
);
2844 return t
? t
: decl
;
2848 /* Construct the initialization value for reduction CLAUSE. */
2851 omp_reduction_init (tree clause
, tree type
)
2853 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2854 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2861 case TRUTH_ORIF_EXPR
:
2862 case TRUTH_XOR_EXPR
:
2864 return build_zero_cst (type
);
2867 case TRUTH_AND_EXPR
:
2868 case TRUTH_ANDIF_EXPR
:
2870 return fold_convert_loc (loc
, type
, integer_one_node
);
2873 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2876 if (SCALAR_FLOAT_TYPE_P (type
))
2878 REAL_VALUE_TYPE max
, min
;
2879 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2882 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2885 real_maxval (&min
, 1, TYPE_MODE (type
));
2886 return build_real (type
, min
);
2890 gcc_assert (INTEGRAL_TYPE_P (type
));
2891 return TYPE_MIN_VALUE (type
);
2895 if (SCALAR_FLOAT_TYPE_P (type
))
2897 REAL_VALUE_TYPE max
;
2898 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2901 real_maxval (&max
, 0, TYPE_MODE (type
));
2902 return build_real (type
, max
);
2906 gcc_assert (INTEGRAL_TYPE_P (type
));
2907 return TYPE_MAX_VALUE (type
);
2915 /* Return alignment to be assumed for var in CLAUSE, which should be
2916 OMP_CLAUSE_ALIGNED. */
2919 omp_clause_aligned_alignment (tree clause
)
2921 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2922 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2924 /* Otherwise return implementation defined alignment. */
2925 unsigned int al
= 1;
2926 enum machine_mode mode
, vmode
;
2927 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2929 vs
= 1 << floor_log2 (vs
);
2930 static enum mode_class classes
[]
2931 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2932 for (int i
= 0; i
< 4; i
+= 2)
2933 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2935 mode
= GET_MODE_WIDER_MODE (mode
))
2937 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2938 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2941 && GET_MODE_SIZE (vmode
) < vs
2942 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2943 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2945 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2946 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2948 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2949 / GET_MODE_SIZE (mode
));
2950 if (TYPE_MODE (type
) != vmode
)
2952 if (TYPE_ALIGN_UNIT (type
) > al
)
2953 al
= TYPE_ALIGN_UNIT (type
);
2955 return build_int_cst (integer_type_node
, al
);
2958 /* Return maximum possible vectorization factor for the target. */
2965 || !flag_tree_loop_optimize
2966 || (!flag_tree_loop_vectorize
2967 && (global_options_set
.x_flag_tree_loop_vectorize
2968 || global_options_set
.x_flag_tree_vectorize
)))
2971 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2974 vs
= 1 << floor_log2 (vs
);
2977 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2978 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2979 return GET_MODE_NUNITS (vqimode
);
2983 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2987 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2988 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2992 max_vf
= omp_max_vf ();
2995 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2996 OMP_CLAUSE_SAFELEN
);
2997 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
2999 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3001 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3005 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3006 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3012 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3013 tree avar
= create_tmp_var_raw (atype
, NULL
);
3014 if (TREE_ADDRESSABLE (new_var
))
3015 TREE_ADDRESSABLE (avar
) = 1;
3016 DECL_ATTRIBUTES (avar
)
3017 = tree_cons (get_identifier ("omp simd array"), NULL
,
3018 DECL_ATTRIBUTES (avar
));
3019 gimple_add_tmp_var (avar
);
3020 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3021 NULL_TREE
, NULL_TREE
);
3022 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3023 NULL_TREE
, NULL_TREE
);
3024 if (DECL_P (new_var
))
3026 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3027 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3032 /* Helper function of lower_rec_input_clauses. For a reference
3033 in simd reduction, add an underlying variable it will reference. */
3036 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3038 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3039 if (TREE_CONSTANT (z
))
3041 const char *name
= NULL
;
3042 if (DECL_NAME (new_vard
))
3043 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3045 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3046 gimple_add_tmp_var (z
);
3047 TREE_ADDRESSABLE (z
) = 1;
3048 z
= build_fold_addr_expr_loc (loc
, z
);
3049 gimplify_assign (new_vard
, z
, ilist
);
3053 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3054 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3055 private variables. Initialization statements go in ILIST, while calls
3056 to destructors go in DLIST. */
3059 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3060 omp_context
*ctx
, struct omp_for_data
*fd
)
3062 tree c
, dtor
, copyin_seq
, x
, ptr
;
3063 bool copyin_by_ref
= false;
3064 bool lastprivate_firstprivate
= false;
3065 bool reduction_omp_orig_ref
= false;
3067 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3068 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3070 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3071 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3072 gimple_seq llist
[2] = { NULL
, NULL
};
3076 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3077 with data sharing clauses referencing variable sized vars. That
3078 is unnecessarily hard to support and very unlikely to result in
3079 vectorized code anyway. */
3081 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3082 switch (OMP_CLAUSE_CODE (c
))
3084 case OMP_CLAUSE_LINEAR
:
3085 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3088 case OMP_CLAUSE_REDUCTION
:
3089 case OMP_CLAUSE_PRIVATE
:
3090 case OMP_CLAUSE_FIRSTPRIVATE
:
3091 case OMP_CLAUSE_LASTPRIVATE
:
3092 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3099 /* Do all the fixed sized types in the first pass, and the variable sized
3100 types in the second pass. This makes sure that the scalar arguments to
3101 the variable sized types are processed before we use them in the
3102 variable sized operations. */
3103 for (pass
= 0; pass
< 2; ++pass
)
3105 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3107 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3110 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3114 case OMP_CLAUSE_PRIVATE
:
3115 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3118 case OMP_CLAUSE_SHARED
:
3119 /* Ignore shared directives in teams construct. */
3120 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3122 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3124 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3127 case OMP_CLAUSE_FIRSTPRIVATE
:
3128 case OMP_CLAUSE_COPYIN
:
3129 case OMP_CLAUSE_LINEAR
:
3131 case OMP_CLAUSE_REDUCTION
:
3132 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3133 reduction_omp_orig_ref
= true;
3135 case OMP_CLAUSE__LOOPTEMP_
:
3136 /* Handle _looptemp_ clauses only on parallel. */
3140 case OMP_CLAUSE_LASTPRIVATE
:
3141 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3143 lastprivate_firstprivate
= true;
3147 /* Even without corresponding firstprivate, if
3148 decl is Fortran allocatable, it needs outer var
3151 && lang_hooks
.decls
.omp_private_outer_ref
3152 (OMP_CLAUSE_DECL (c
)))
3153 lastprivate_firstprivate
= true;
3155 case OMP_CLAUSE_ALIGNED
:
3158 var
= OMP_CLAUSE_DECL (c
);
3159 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3160 && !is_global_var (var
))
3162 new_var
= maybe_lookup_decl (var
, ctx
);
3163 if (new_var
== NULL_TREE
)
3164 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3165 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3166 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3167 omp_clause_aligned_alignment (c
));
3168 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3169 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3170 gimplify_and_add (x
, ilist
);
3172 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3173 && is_global_var (var
))
3175 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3176 new_var
= lookup_decl (var
, ctx
);
3177 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3178 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3179 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3180 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3181 omp_clause_aligned_alignment (c
));
3182 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3183 x
= create_tmp_var (ptype
, NULL
);
3184 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3185 gimplify_and_add (t
, ilist
);
3186 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3187 SET_DECL_VALUE_EXPR (new_var
, t
);
3188 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3195 new_var
= var
= OMP_CLAUSE_DECL (c
);
3196 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3197 new_var
= lookup_decl (var
, ctx
);
3199 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3204 else if (is_variable_sized (var
))
3206 /* For variable sized types, we need to allocate the
3207 actual storage here. Call alloca and store the
3208 result in the pointer decl that we created elsewhere. */
3212 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3217 ptr
= DECL_VALUE_EXPR (new_var
);
3218 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3219 ptr
= TREE_OPERAND (ptr
, 0);
3220 gcc_assert (DECL_P (ptr
));
3221 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3223 /* void *tmp = __builtin_alloca */
3224 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3225 stmt
= gimple_build_call (atmp
, 1, x
);
3226 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3227 gimple_add_tmp_var (tmp
);
3228 gimple_call_set_lhs (stmt
, tmp
);
3230 gimple_seq_add_stmt (ilist
, stmt
);
3232 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3233 gimplify_assign (ptr
, x
, ilist
);
3236 else if (is_reference (var
))
3238 /* For references that are being privatized for Fortran,
3239 allocate new backing storage for the new pointer
3240 variable. This allows us to avoid changing all the
3241 code that expects a pointer to something that expects
3242 a direct variable. */
3246 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3247 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3249 x
= build_receiver_ref (var
, false, ctx
);
3250 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3252 else if (TREE_CONSTANT (x
))
3254 /* For reduction in SIMD loop, defer adding the
3255 initialization of the reference, because if we decide
3256 to use SIMD array for it, the initilization could cause
3258 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3262 const char *name
= NULL
;
3263 if (DECL_NAME (var
))
3264 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3266 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3268 gimple_add_tmp_var (x
);
3269 TREE_ADDRESSABLE (x
) = 1;
3270 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3275 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3276 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3281 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3282 gimplify_assign (new_var
, x
, ilist
);
3285 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3287 else if (c_kind
== OMP_CLAUSE_REDUCTION
3288 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3296 switch (OMP_CLAUSE_CODE (c
))
3298 case OMP_CLAUSE_SHARED
:
3299 /* Ignore shared directives in teams construct. */
3300 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3302 /* Shared global vars are just accessed directly. */
3303 if (is_global_var (new_var
))
3305 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3306 needs to be delayed until after fixup_child_record_type so
3307 that we get the correct type during the dereference. */
3308 by_ref
= use_pointer_for_field (var
, ctx
);
3309 x
= build_receiver_ref (var
, by_ref
, ctx
);
3310 SET_DECL_VALUE_EXPR (new_var
, x
);
3311 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3313 /* ??? If VAR is not passed by reference, and the variable
3314 hasn't been initialized yet, then we'll get a warning for
3315 the store into the omp_data_s structure. Ideally, we'd be
3316 able to notice this and not store anything at all, but
3317 we're generating code too early. Suppress the warning. */
3319 TREE_NO_WARNING (var
) = 1;
3322 case OMP_CLAUSE_LASTPRIVATE
:
3323 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3327 case OMP_CLAUSE_PRIVATE
:
3328 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3329 x
= build_outer_var_ref (var
, ctx
);
3330 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3332 if (is_task_ctx (ctx
))
3333 x
= build_receiver_ref (var
, false, ctx
);
3335 x
= build_outer_var_ref (var
, ctx
);
3341 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3344 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3345 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3346 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3347 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3348 idx
, lane
, ivar
, lvar
))
3351 x
= lang_hooks
.decls
.omp_clause_default_ctor
3352 (c
, unshare_expr (ivar
), x
);
3354 gimplify_and_add (x
, &llist
[0]);
3357 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3360 gimple_seq tseq
= NULL
;
3363 gimplify_stmt (&dtor
, &tseq
);
3364 gimple_seq_add_seq (&llist
[1], tseq
);
3371 gimplify_and_add (nx
, ilist
);
3375 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3378 gimple_seq tseq
= NULL
;
3381 gimplify_stmt (&dtor
, &tseq
);
3382 gimple_seq_add_seq (dlist
, tseq
);
3386 case OMP_CLAUSE_LINEAR
:
3387 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3388 goto do_firstprivate
;
3389 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3392 x
= build_outer_var_ref (var
, ctx
);
3395 case OMP_CLAUSE_FIRSTPRIVATE
:
3396 if (is_task_ctx (ctx
))
3398 if (is_reference (var
) || is_variable_sized (var
))
3400 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3402 || use_pointer_for_field (var
, NULL
))
3404 x
= build_receiver_ref (var
, false, ctx
);
3405 SET_DECL_VALUE_EXPR (new_var
, x
);
3406 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3411 x
= build_outer_var_ref (var
, ctx
);
3414 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3415 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3417 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3418 tree stept
= TREE_TYPE (t
);
3419 tree ct
= find_omp_clause (clauses
,
3420 OMP_CLAUSE__LOOPTEMP_
);
3422 tree l
= OMP_CLAUSE_DECL (ct
);
3423 tree n1
= fd
->loop
.n1
;
3424 tree step
= fd
->loop
.step
;
3425 tree itype
= TREE_TYPE (l
);
3426 if (POINTER_TYPE_P (itype
))
3427 itype
= signed_type_for (itype
);
3428 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3429 if (TYPE_UNSIGNED (itype
)
3430 && fd
->loop
.cond_code
== GT_EXPR
)
3431 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3432 fold_build1 (NEGATE_EXPR
, itype
, l
),
3433 fold_build1 (NEGATE_EXPR
,
3436 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3437 t
= fold_build2 (MULT_EXPR
, stept
,
3438 fold_convert (stept
, l
), t
);
3440 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3442 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3444 gimplify_and_add (x
, ilist
);
3448 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3449 x
= fold_build2 (POINTER_PLUS_EXPR
,
3450 TREE_TYPE (x
), x
, t
);
3452 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3455 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3456 || TREE_ADDRESSABLE (new_var
))
3457 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3458 idx
, lane
, ivar
, lvar
))
3460 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3462 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3463 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3464 gimplify_and_add (x
, ilist
);
3465 gimple_stmt_iterator gsi
3466 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3468 = gimple_build_assign (unshare_expr (lvar
), iv
);
3469 gsi_insert_before_without_update (&gsi
, g
,
3471 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3472 enum tree_code code
= PLUS_EXPR
;
3473 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3474 code
= POINTER_PLUS_EXPR
;
3475 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3476 gsi_insert_before_without_update (&gsi
, g
,
3480 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3481 (c
, unshare_expr (ivar
), x
);
3482 gimplify_and_add (x
, &llist
[0]);
3483 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3486 gimple_seq tseq
= NULL
;
3489 gimplify_stmt (&dtor
, &tseq
);
3490 gimple_seq_add_seq (&llist
[1], tseq
);
3495 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3496 gimplify_and_add (x
, ilist
);
3499 case OMP_CLAUSE__LOOPTEMP_
:
3500 gcc_assert (is_parallel_ctx (ctx
));
3501 x
= build_outer_var_ref (var
, ctx
);
3502 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3503 gimplify_and_add (x
, ilist
);
3506 case OMP_CLAUSE_COPYIN
:
3507 by_ref
= use_pointer_for_field (var
, NULL
);
3508 x
= build_receiver_ref (var
, by_ref
, ctx
);
3509 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3510 append_to_statement_list (x
, ©in_seq
);
3511 copyin_by_ref
|= by_ref
;
3514 case OMP_CLAUSE_REDUCTION
:
3515 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3517 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3519 x
= build_outer_var_ref (var
, ctx
);
3521 if (is_reference (var
)
3522 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3524 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3525 SET_DECL_VALUE_EXPR (placeholder
, x
);
3526 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3527 tree new_vard
= new_var
;
3528 if (is_reference (var
))
3530 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3531 new_vard
= TREE_OPERAND (new_var
, 0);
3532 gcc_assert (DECL_P (new_vard
));
3535 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3536 idx
, lane
, ivar
, lvar
))
3538 if (new_vard
== new_var
)
3540 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3541 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3545 SET_DECL_VALUE_EXPR (new_vard
,
3546 build_fold_addr_expr (ivar
));
3547 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3549 x
= lang_hooks
.decls
.omp_clause_default_ctor
3550 (c
, unshare_expr (ivar
),
3551 build_outer_var_ref (var
, ctx
));
3553 gimplify_and_add (x
, &llist
[0]);
3554 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3556 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3557 lower_omp (&tseq
, ctx
);
3558 gimple_seq_add_seq (&llist
[0], tseq
);
3560 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3561 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3562 lower_omp (&tseq
, ctx
);
3563 gimple_seq_add_seq (&llist
[1], tseq
);
3564 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3565 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3566 if (new_vard
== new_var
)
3567 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3569 SET_DECL_VALUE_EXPR (new_vard
,
3570 build_fold_addr_expr (lvar
));
3571 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3576 gimplify_stmt (&dtor
, &tseq
);
3577 gimple_seq_add_seq (&llist
[1], tseq
);
3581 /* If this is a reference to constant size reduction var
3582 with placeholder, we haven't emitted the initializer
3583 for it because it is undesirable if SIMD arrays are used.
3584 But if they aren't used, we need to emit the deferred
3585 initialization now. */
3586 else if (is_reference (var
) && is_simd
)
3587 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3588 x
= lang_hooks
.decls
.omp_clause_default_ctor
3589 (c
, unshare_expr (new_var
),
3590 build_outer_var_ref (var
, ctx
));
3592 gimplify_and_add (x
, ilist
);
3593 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3595 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3596 lower_omp (&tseq
, ctx
);
3597 gimple_seq_add_seq (ilist
, tseq
);
3599 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3602 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3603 lower_omp (&tseq
, ctx
);
3604 gimple_seq_add_seq (dlist
, tseq
);
3605 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3607 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3612 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3613 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3614 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3616 /* reduction(-:var) sums up the partial results, so it
3617 acts identically to reduction(+:var). */
3618 if (code
== MINUS_EXPR
)
3621 tree new_vard
= new_var
;
3622 if (is_simd
&& is_reference (var
))
3624 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3625 new_vard
= TREE_OPERAND (new_var
, 0);
3626 gcc_assert (DECL_P (new_vard
));
3629 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3630 idx
, lane
, ivar
, lvar
))
3632 tree ref
= build_outer_var_ref (var
, ctx
);
3634 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3636 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3637 ref
= build_outer_var_ref (var
, ctx
);
3638 gimplify_assign (ref
, x
, &llist
[1]);
3640 if (new_vard
!= new_var
)
3642 SET_DECL_VALUE_EXPR (new_vard
,
3643 build_fold_addr_expr (lvar
));
3644 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3649 if (is_reference (var
) && is_simd
)
3650 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3651 gimplify_assign (new_var
, x
, ilist
);
3654 tree ref
= build_outer_var_ref (var
, ctx
);
3656 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3657 ref
= build_outer_var_ref (var
, ctx
);
3658 gimplify_assign (ref
, x
, dlist
);
3672 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3673 /* Don't want uninit warnings on simduid, it is always uninitialized,
3674 but we use it not for the value, but for the DECL_UID only. */
3675 TREE_NO_WARNING (uid
) = 1;
3677 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3678 gimple_call_set_lhs (g
, lane
);
3679 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3680 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3681 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3682 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3683 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3684 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3685 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3686 build_int_cst (unsigned_type_node
, 0),
3688 gimple_seq_add_stmt (ilist
, g
);
3689 for (int i
= 0; i
< 2; i
++)
3692 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3693 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3694 gimple_call_set_lhs (g
, vf
);
3695 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3696 gimple_seq_add_stmt (seq
, g
);
3697 tree t
= build_int_cst (unsigned_type_node
, 0);
3698 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3699 gimple_seq_add_stmt (seq
, g
);
3700 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3701 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3702 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3703 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3704 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3705 gimple_seq_add_seq (seq
, llist
[i
]);
3706 t
= build_int_cst (unsigned_type_node
, 1);
3707 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3708 gimple_seq_add_stmt (seq
, g
);
3709 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3710 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3711 gimple_seq_add_stmt (seq
, g
);
3712 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3716 /* The copyin sequence is not to be executed by the main thread, since
3717 that would result in self-copies. Perhaps not visible to scalars,
3718 but it certainly is to C++ operator=. */
3721 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3723 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3724 build_int_cst (TREE_TYPE (x
), 0));
3725 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3726 gimplify_and_add (x
, ilist
);
3729 /* If any copyin variable is passed by reference, we must ensure the
3730 master thread doesn't modify it before it is copied over in all
3731 threads. Similarly for variables in both firstprivate and
3732 lastprivate clauses we need to ensure the lastprivate copying
3733 happens after firstprivate copying in all threads. And similarly
3734 for UDRs if initializer expression refers to omp_orig. */
3735 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3737 /* Don't add any barrier for #pragma omp simd or
3738 #pragma omp distribute. */
3739 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3740 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3741 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3744 /* If max_vf is non-zero, then we can use only a vectorization factor
3745 up to the max_vf we chose. So stick it into the safelen clause. */
3748 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3749 OMP_CLAUSE_SAFELEN
);
3751 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3752 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3755 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3756 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3758 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3759 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3765 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3766 both parallel and workshare constructs. PREDICATE may be NULL if it's
3770 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3773 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3774 bool par_clauses
= false;
3775 tree simduid
= NULL
, lastlane
= NULL
;
3777 /* Early exit if there are no lastprivate or linear clauses. */
3778 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3779 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3780 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3781 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3783 if (clauses
== NULL
)
3785 /* If this was a workshare clause, see if it had been combined
3786 with its parallel. In that case, look for the clauses on the
3787 parallel statement itself. */
3788 if (is_parallel_ctx (ctx
))
3792 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3795 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3796 OMP_CLAUSE_LASTPRIVATE
);
3797 if (clauses
== NULL
)
3805 tree label_true
, arm1
, arm2
;
3807 label
= create_artificial_label (UNKNOWN_LOCATION
);
3808 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3809 arm1
= TREE_OPERAND (predicate
, 0);
3810 arm2
= TREE_OPERAND (predicate
, 1);
3811 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3812 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3813 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3815 gimple_seq_add_stmt (stmt_list
, stmt
);
3816 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3819 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3820 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3822 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3824 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3827 for (c
= clauses
; c
;)
3830 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3832 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3833 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3834 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3836 var
= OMP_CLAUSE_DECL (c
);
3837 new_var
= lookup_decl (var
, ctx
);
3839 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3841 tree val
= DECL_VALUE_EXPR (new_var
);
3842 if (TREE_CODE (val
) == ARRAY_REF
3843 && VAR_P (TREE_OPERAND (val
, 0))
3844 && lookup_attribute ("omp simd array",
3845 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3848 if (lastlane
== NULL
)
3850 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3852 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3854 TREE_OPERAND (val
, 1));
3855 gimple_call_set_lhs (g
, lastlane
);
3856 gimple_seq_add_stmt (stmt_list
, g
);
3858 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3859 TREE_OPERAND (val
, 0), lastlane
,
3860 NULL_TREE
, NULL_TREE
);
3864 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3865 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3867 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3868 gimple_seq_add_seq (stmt_list
,
3869 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3870 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3872 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3873 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
3875 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
3876 gimple_seq_add_seq (stmt_list
,
3877 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
3878 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
3881 x
= build_outer_var_ref (var
, ctx
);
3882 if (is_reference (var
))
3883 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3884 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3885 gimplify_and_add (x
, stmt_list
);
3887 c
= OMP_CLAUSE_CHAIN (c
);
3888 if (c
== NULL
&& !par_clauses
)
3890 /* If this was a workshare clause, see if it had been combined
3891 with its parallel. In that case, continue looking for the
3892 clauses also on the parallel statement itself. */
3893 if (is_parallel_ctx (ctx
))
3897 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3900 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3901 OMP_CLAUSE_LASTPRIVATE
);
3907 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3911 /* Generate code to implement the REDUCTION clauses. */
3914 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3916 gimple_seq sub_seq
= NULL
;
3921 /* SIMD reductions are handled in lower_rec_input_clauses. */
3922 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3923 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3926 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3927 update in that case, otherwise use a lock. */
3928 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3929 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3931 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3933 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3943 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3945 tree var
, ref
, new_var
;
3946 enum tree_code code
;
3947 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3949 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3952 var
= OMP_CLAUSE_DECL (c
);
3953 new_var
= lookup_decl (var
, ctx
);
3954 if (is_reference (var
))
3955 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3956 ref
= build_outer_var_ref (var
, ctx
);
3957 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3959 /* reduction(-:var) sums up the partial results, so it acts
3960 identically to reduction(+:var). */
3961 if (code
== MINUS_EXPR
)
3966 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3968 addr
= save_expr (addr
);
3969 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3970 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3971 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3972 gimplify_and_add (x
, stmt_seqp
);
3976 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3978 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3980 if (is_reference (var
)
3981 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3983 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3984 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3985 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3986 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3987 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3988 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3989 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3993 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3994 ref
= build_outer_var_ref (var
, ctx
);
3995 gimplify_assign (ref
, x
, &sub_seq
);
3999 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4001 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4003 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4005 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4007 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4011 /* Generate code to implement the COPYPRIVATE clauses. */
4014 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4019 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4021 tree var
, new_var
, ref
, x
;
4023 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4025 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4028 var
= OMP_CLAUSE_DECL (c
);
4029 by_ref
= use_pointer_for_field (var
, NULL
);
4031 ref
= build_sender_ref (var
, ctx
);
4032 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4035 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4036 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4038 gimplify_assign (ref
, x
, slist
);
4040 ref
= build_receiver_ref (var
, false, ctx
);
4043 ref
= fold_convert_loc (clause_loc
,
4044 build_pointer_type (TREE_TYPE (new_var
)),
4046 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4048 if (is_reference (var
))
4050 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4051 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4052 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4054 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4055 gimplify_and_add (x
, rlist
);
4060 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4061 and REDUCTION from the sender (aka parent) side. */
4064 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4069 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4071 tree val
, ref
, x
, var
;
4072 bool by_ref
, do_in
= false, do_out
= false;
4073 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4075 switch (OMP_CLAUSE_CODE (c
))
4077 case OMP_CLAUSE_PRIVATE
:
4078 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4081 case OMP_CLAUSE_FIRSTPRIVATE
:
4082 case OMP_CLAUSE_COPYIN
:
4083 case OMP_CLAUSE_LASTPRIVATE
:
4084 case OMP_CLAUSE_REDUCTION
:
4085 case OMP_CLAUSE__LOOPTEMP_
:
4091 val
= OMP_CLAUSE_DECL (c
);
4092 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4094 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4095 && is_global_var (var
))
4097 if (is_variable_sized (val
))
4099 by_ref
= use_pointer_for_field (val
, NULL
);
4101 switch (OMP_CLAUSE_CODE (c
))
4103 case OMP_CLAUSE_PRIVATE
:
4104 case OMP_CLAUSE_FIRSTPRIVATE
:
4105 case OMP_CLAUSE_COPYIN
:
4106 case OMP_CLAUSE__LOOPTEMP_
:
4110 case OMP_CLAUSE_LASTPRIVATE
:
4111 if (by_ref
|| is_reference (val
))
4113 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4120 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4125 case OMP_CLAUSE_REDUCTION
:
4127 do_out
= !(by_ref
|| is_reference (val
));
4136 ref
= build_sender_ref (val
, ctx
);
4137 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4138 gimplify_assign (ref
, x
, ilist
);
4139 if (is_task_ctx (ctx
))
4140 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4145 ref
= build_sender_ref (val
, ctx
);
4146 gimplify_assign (var
, ref
, olist
);
4151 /* Generate code to implement SHARED from the sender (aka parent)
4152 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4153 list things that got automatically shared. */
4156 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4158 tree var
, ovar
, nvar
, f
, x
, record_type
;
4160 if (ctx
->record_type
== NULL
)
4163 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4164 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4166 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4167 nvar
= maybe_lookup_decl (ovar
, ctx
);
4168 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4171 /* If CTX is a nested parallel directive. Find the immediately
4172 enclosing parallel or workshare construct that contains a
4173 mapping for OVAR. */
4174 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4176 if (use_pointer_for_field (ovar
, ctx
))
4178 x
= build_sender_ref (ovar
, ctx
);
4179 var
= build_fold_addr_expr (var
);
4180 gimplify_assign (x
, var
, ilist
);
4184 x
= build_sender_ref (ovar
, ctx
);
4185 gimplify_assign (x
, var
, ilist
);
4187 if (!TREE_READONLY (var
)
4188 /* We don't need to receive a new reference to a result
4189 or parm decl. In fact we may not store to it as we will
4190 invalidate any pending RSO and generate wrong gimple
4192 && !((TREE_CODE (var
) == RESULT_DECL
4193 || TREE_CODE (var
) == PARM_DECL
)
4194 && DECL_BY_REFERENCE (var
)))
4196 x
= build_sender_ref (ovar
, ctx
);
4197 gimplify_assign (var
, x
, olist
);
4204 /* A convenience function to build an empty GIMPLE_COND with just the
4208 gimple_build_cond_empty (tree cond
)
4210 enum tree_code pred_code
;
4213 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4214 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4218 /* Build the function calls to GOMP_parallel_start etc to actually
4219 generate the parallel operation. REGION is the parallel region
4220 being expanded. BB is the block where to insert the code. WS_ARGS
4221 will be set if this is a call to a combined parallel+workshare
4222 construct, it contains the list of additional arguments needed by
4223 the workshare construct. */
4226 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4227 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4229 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4230 gimple_stmt_iterator gsi
;
4232 enum built_in_function start_ix
;
4234 location_t clause_loc
;
4235 vec
<tree
, va_gc
> *args
;
4237 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4239 /* Determine what flavor of GOMP_parallel we will be
4241 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4242 if (is_combined_parallel (region
))
4244 switch (region
->inner
->type
)
4246 case GIMPLE_OMP_FOR
:
4247 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4248 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4249 + (region
->inner
->sched_kind
4250 == OMP_CLAUSE_SCHEDULE_RUNTIME
4251 ? 3 : region
->inner
->sched_kind
));
4252 start_ix
= (enum built_in_function
)start_ix2
;
4254 case GIMPLE_OMP_SECTIONS
:
4255 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4262 /* By default, the value of NUM_THREADS is zero (selected at run time)
4263 and there is no conditional. */
4265 val
= build_int_cst (unsigned_type_node
, 0);
4266 flags
= build_int_cst (unsigned_type_node
, 0);
4268 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4270 cond
= OMP_CLAUSE_IF_EXPR (c
);
4272 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4275 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4276 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4279 clause_loc
= gimple_location (entry_stmt
);
4281 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4283 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4285 /* Ensure 'val' is of the correct type. */
4286 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4288 /* If we found the clause 'if (cond)', build either
4289 (cond != 0) or (cond ? val : 1u). */
4292 cond
= gimple_boolify (cond
);
4294 if (integer_zerop (val
))
4295 val
= fold_build2_loc (clause_loc
,
4296 EQ_EXPR
, unsigned_type_node
, cond
,
4297 build_int_cst (TREE_TYPE (cond
), 0));
4300 basic_block cond_bb
, then_bb
, else_bb
;
4301 edge e
, e_then
, e_else
;
4302 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4304 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4305 if (gimple_in_ssa_p (cfun
))
4307 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4308 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4309 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4318 e
= split_block (bb
, NULL
);
4323 then_bb
= create_empty_bb (cond_bb
);
4324 else_bb
= create_empty_bb (then_bb
);
4325 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4326 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4328 stmt
= gimple_build_cond_empty (cond
);
4329 gsi
= gsi_start_bb (cond_bb
);
4330 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4332 gsi
= gsi_start_bb (then_bb
);
4333 stmt
= gimple_build_assign (tmp_then
, val
);
4334 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4336 gsi
= gsi_start_bb (else_bb
);
4337 stmt
= gimple_build_assign
4338 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4339 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4341 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4342 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4343 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4344 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4345 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4346 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4348 if (gimple_in_ssa_p (cfun
))
4350 gimple phi
= create_phi_node (tmp_join
, bb
);
4351 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4352 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4358 gsi
= gsi_start_bb (bb
);
4359 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4360 false, GSI_CONTINUE_LINKING
);
4363 gsi
= gsi_last_bb (bb
);
4364 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4366 t1
= null_pointer_node
;
4368 t1
= build_fold_addr_expr (t
);
4369 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4371 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4372 args
->quick_push (t2
);
4373 args
->quick_push (t1
);
4374 args
->quick_push (val
);
4376 args
->splice (*ws_args
);
4377 args
->quick_push (flags
);
4379 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4380 builtin_decl_explicit (start_ix
), args
);
4382 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4383 false, GSI_CONTINUE_LINKING
);
4387 /* Build the function call to GOMP_task to actually
4388 generate the task operation. BB is the block where to insert the code. */
4391 expand_task_call (basic_block bb
, gimple entry_stmt
)
4393 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4394 gimple_stmt_iterator gsi
;
4395 location_t loc
= gimple_location (entry_stmt
);
4397 clauses
= gimple_omp_task_clauses (entry_stmt
);
4399 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4401 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4403 cond
= boolean_true_node
;
4405 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4406 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4407 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4408 flags
= build_int_cst (unsigned_type_node
,
4409 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4411 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4414 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4415 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4416 build_int_cst (unsigned_type_node
, 2),
4417 build_int_cst (unsigned_type_node
, 0));
4418 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4421 depend
= OMP_CLAUSE_DECL (depend
);
4423 depend
= build_int_cst (ptr_type_node
, 0);
4425 gsi
= gsi_last_bb (bb
);
4426 t
= gimple_omp_task_data_arg (entry_stmt
);
4428 t2
= null_pointer_node
;
4430 t2
= build_fold_addr_expr_loc (loc
, t
);
4431 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4432 t
= gimple_omp_task_copy_fn (entry_stmt
);
4434 t3
= null_pointer_node
;
4436 t3
= build_fold_addr_expr_loc (loc
, t
);
4438 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4440 gimple_omp_task_arg_size (entry_stmt
),
4441 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4444 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4445 false, GSI_CONTINUE_LINKING
);
4449 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4450 catch handler and return it. This prevents programs from violating the
4451 structured block semantics with throws. */
4454 maybe_catch_exception (gimple_seq body
)
4459 if (!flag_exceptions
)
4462 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4463 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4465 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4467 g
= gimple_build_eh_must_not_throw (decl
);
4468 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4471 return gimple_seq_alloc_with_stmt (g
);
4474 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4477 vec2chain (vec
<tree
, va_gc
> *v
)
4479 tree chain
= NULL_TREE
, t
;
4482 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4484 DECL_CHAIN (t
) = chain
;
4492 /* Remove barriers in REGION->EXIT's block. Note that this is only
4493 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4494 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4495 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4499 remove_exit_barrier (struct omp_region
*region
)
4501 gimple_stmt_iterator gsi
;
4502 basic_block exit_bb
;
4506 int any_addressable_vars
= -1;
4508 exit_bb
= region
->exit
;
4510 /* If the parallel region doesn't return, we don't have REGION->EXIT
4515 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4516 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4517 statements that can appear in between are extremely limited -- no
4518 memory operations at all. Here, we allow nothing at all, so the
4519 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4520 gsi
= gsi_last_bb (exit_bb
);
4521 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4523 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4526 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4528 gsi
= gsi_last_bb (e
->src
);
4529 if (gsi_end_p (gsi
))
4531 stmt
= gsi_stmt (gsi
);
4532 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4533 && !gimple_omp_return_nowait_p (stmt
))
4535 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4536 in many cases. If there could be tasks queued, the barrier
4537 might be needed to let the tasks run before some local
4538 variable of the parallel that the task uses as shared
4539 runs out of scope. The task can be spawned either
4540 from within current function (this would be easy to check)
4541 or from some function it calls and gets passed an address
4542 of such a variable. */
4543 if (any_addressable_vars
< 0)
4545 gimple parallel_stmt
= last_stmt (region
->entry
);
4546 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4547 tree local_decls
, block
, decl
;
4550 any_addressable_vars
= 0;
4551 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4552 if (TREE_ADDRESSABLE (decl
))
4554 any_addressable_vars
= 1;
4557 for (block
= gimple_block (stmt
);
4558 !any_addressable_vars
4560 && TREE_CODE (block
) == BLOCK
;
4561 block
= BLOCK_SUPERCONTEXT (block
))
4563 for (local_decls
= BLOCK_VARS (block
);
4565 local_decls
= DECL_CHAIN (local_decls
))
4566 if (TREE_ADDRESSABLE (local_decls
))
4568 any_addressable_vars
= 1;
4571 if (block
== gimple_block (parallel_stmt
))
4575 if (!any_addressable_vars
)
4576 gimple_omp_return_set_nowait (stmt
);
4582 remove_exit_barriers (struct omp_region
*region
)
4584 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4585 remove_exit_barrier (region
);
4589 region
= region
->inner
;
4590 remove_exit_barriers (region
);
4591 while (region
->next
)
4593 region
= region
->next
;
4594 remove_exit_barriers (region
);
4599 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4600 calls. These can't be declared as const functions, but
4601 within one parallel body they are constant, so they can be
4602 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4603 which are declared const. Similarly for task body, except
4604 that in untied task omp_get_thread_num () can change at any task
4605 scheduling point. */
4608 optimize_omp_library_calls (gimple entry_stmt
)
4611 gimple_stmt_iterator gsi
;
4612 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4613 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4614 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4615 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4616 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4617 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4618 OMP_CLAUSE_UNTIED
) != NULL
);
4620 FOR_EACH_BB_FN (bb
, cfun
)
4621 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4623 gimple call
= gsi_stmt (gsi
);
4626 if (is_gimple_call (call
)
4627 && (decl
= gimple_call_fndecl (call
))
4628 && DECL_EXTERNAL (decl
)
4629 && TREE_PUBLIC (decl
)
4630 && DECL_INITIAL (decl
) == NULL
)
4634 if (DECL_NAME (decl
) == thr_num_id
)
4636 /* In #pragma omp task untied omp_get_thread_num () can change
4637 during the execution of the task region. */
4640 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4642 else if (DECL_NAME (decl
) == num_thr_id
)
4643 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4647 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4648 || gimple_call_num_args (call
) != 0)
4651 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4654 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4655 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4656 TREE_TYPE (TREE_TYPE (built_in
))))
4659 gimple_call_set_fndecl (call
, built_in
);
4664 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4668 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4672 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4673 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4676 if (TREE_CODE (t
) == ADDR_EXPR
)
4677 recompute_tree_invariant_for_addr_expr (t
);
4679 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4683 /* Prepend TO = FROM assignment before *GSI_P. */
4686 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4688 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4689 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4690 true, GSI_SAME_STMT
);
4691 gimple stmt
= gimple_build_assign (to
, from
);
4692 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4693 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4694 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4696 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4697 gimple_regimplify_operands (stmt
, &gsi
);
4701 /* Expand the OpenMP parallel or task directive starting at REGION. */
4704 expand_omp_taskreg (struct omp_region
*region
)
4706 basic_block entry_bb
, exit_bb
, new_bb
;
4707 struct function
*child_cfun
;
4708 tree child_fn
, block
, t
;
4709 gimple_stmt_iterator gsi
;
4710 gimple entry_stmt
, stmt
;
4712 vec
<tree
, va_gc
> *ws_args
;
4714 entry_stmt
= last_stmt (region
->entry
);
4715 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4716 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4718 entry_bb
= region
->entry
;
4719 exit_bb
= region
->exit
;
4721 if (is_combined_parallel (region
))
4722 ws_args
= region
->ws_args
;
4726 if (child_cfun
->cfg
)
4728 /* Due to inlining, it may happen that we have already outlined
4729 the region, in which case all we need to do is make the
4730 sub-graph unreachable and emit the parallel call. */
4731 edge entry_succ_e
, exit_succ_e
;
4733 entry_succ_e
= single_succ_edge (entry_bb
);
4735 gsi
= gsi_last_bb (entry_bb
);
4736 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4737 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4738 gsi_remove (&gsi
, true);
4743 exit_succ_e
= single_succ_edge (exit_bb
);
4744 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4746 remove_edge_and_dominated_blocks (entry_succ_e
);
4750 unsigned srcidx
, dstidx
, num
;
4752 /* If the parallel region needs data sent from the parent
4753 function, then the very first statement (except possible
4754 tree profile counter updates) of the parallel body
4755 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4756 &.OMP_DATA_O is passed as an argument to the child function,
4757 we need to replace it with the argument as seen by the child
4760 In most cases, this will end up being the identity assignment
4761 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4762 a function call that has been inlined, the original PARM_DECL
4763 .OMP_DATA_I may have been converted into a different local
4764 variable. In which case, we need to keep the assignment. */
4765 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4767 basic_block entry_succ_bb
= single_succ (entry_bb
);
4769 gimple parcopy_stmt
= NULL
;
4771 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4775 gcc_assert (!gsi_end_p (gsi
));
4776 stmt
= gsi_stmt (gsi
);
4777 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4780 if (gimple_num_ops (stmt
) == 2)
4782 tree arg
= gimple_assign_rhs1 (stmt
);
4784 /* We're ignore the subcode because we're
4785 effectively doing a STRIP_NOPS. */
4787 if (TREE_CODE (arg
) == ADDR_EXPR
4788 && TREE_OPERAND (arg
, 0)
4789 == gimple_omp_taskreg_data_arg (entry_stmt
))
4791 parcopy_stmt
= stmt
;
4797 gcc_assert (parcopy_stmt
!= NULL
);
4798 arg
= DECL_ARGUMENTS (child_fn
);
4800 if (!gimple_in_ssa_p (cfun
))
4802 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4803 gsi_remove (&gsi
, true);
4806 /* ?? Is setting the subcode really necessary ?? */
4807 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4808 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4813 /* If we are in ssa form, we must load the value from the default
4814 definition of the argument. That should not be defined now,
4815 since the argument is not used uninitialized. */
4816 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4817 narg
= make_ssa_name (arg
, gimple_build_nop ());
4818 set_ssa_default_def (cfun
, arg
, narg
);
4819 /* ?? Is setting the subcode really necessary ?? */
4820 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4821 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4822 update_stmt (parcopy_stmt
);
4826 /* Declare local variables needed in CHILD_CFUN. */
4827 block
= DECL_INITIAL (child_fn
);
4828 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4829 /* The gimplifier could record temporaries in parallel/task block
4830 rather than in containing function's local_decls chain,
4831 which would mean cgraph missed finalizing them. Do it now. */
4832 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4833 if (TREE_CODE (t
) == VAR_DECL
4835 && !DECL_EXTERNAL (t
))
4836 varpool_node::finalize_decl (t
);
4837 DECL_SAVED_TREE (child_fn
) = NULL
;
4838 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4839 gimple_set_body (child_fn
, NULL
);
4840 TREE_USED (block
) = 1;
4842 /* Reset DECL_CONTEXT on function arguments. */
4843 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4844 DECL_CONTEXT (t
) = child_fn
;
4846 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4847 so that it can be moved to the child function. */
4848 gsi
= gsi_last_bb (entry_bb
);
4849 stmt
= gsi_stmt (gsi
);
4850 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4851 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4852 gsi_remove (&gsi
, true);
4853 e
= split_block (entry_bb
, stmt
);
4855 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4857 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4860 gsi
= gsi_last_bb (exit_bb
);
4861 gcc_assert (!gsi_end_p (gsi
)
4862 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4863 stmt
= gimple_build_return (NULL
);
4864 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4865 gsi_remove (&gsi
, true);
4868 /* Move the parallel region into CHILD_CFUN. */
4870 if (gimple_in_ssa_p (cfun
))
4872 init_tree_ssa (child_cfun
);
4873 init_ssa_operands (child_cfun
);
4874 child_cfun
->gimple_df
->in_ssa_p
= true;
4878 block
= gimple_block (entry_stmt
);
4880 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4882 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4883 /* When the OMP expansion process cannot guarantee an up-to-date
4884 loop tree arrange for the child function to fixup loops. */
4885 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4886 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4888 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4889 num
= vec_safe_length (child_cfun
->local_decls
);
4890 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4892 t
= (*child_cfun
->local_decls
)[srcidx
];
4893 if (DECL_CONTEXT (t
) == cfun
->decl
)
4895 if (srcidx
!= dstidx
)
4896 (*child_cfun
->local_decls
)[dstidx
] = t
;
4900 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4902 /* Inform the callgraph about the new function. */
4903 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4904 cgraph_node::add_new_function (child_fn
, true);
4906 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4907 fixed in a following pass. */
4908 push_cfun (child_cfun
);
4910 optimize_omp_library_calls (entry_stmt
);
4911 rebuild_cgraph_edges ();
4913 /* Some EH regions might become dead, see PR34608. If
4914 pass_cleanup_cfg isn't the first pass to happen with the
4915 new child, these dead EH edges might cause problems.
4916 Clean them up now. */
4917 if (flag_exceptions
)
4920 bool changed
= false;
4922 FOR_EACH_BB_FN (bb
, cfun
)
4923 changed
|= gimple_purge_dead_eh_edges (bb
);
4925 cleanup_tree_cfg ();
4927 if (gimple_in_ssa_p (cfun
))
4928 update_ssa (TODO_update_ssa
);
4932 /* Emit a library call to launch the children threads. */
4933 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4934 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4936 expand_task_call (new_bb
, entry_stmt
);
4937 if (gimple_in_ssa_p (cfun
))
4938 update_ssa (TODO_update_ssa_only_virtuals
);
4942 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4943 of the combined collapse > 1 loop constructs, generate code like:
4944 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4949 count3 = (adj + N32 - N31) / STEP3;
4950 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4955 count2 = (adj + N22 - N21) / STEP2;
4956 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4961 count1 = (adj + N12 - N11) / STEP1;
4962 count = count1 * count2 * count3;
4963 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4965 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4966 of the combined loop constructs, just initialize COUNTS array
4967 from the _looptemp_ clauses. */
4969 /* NOTE: It *could* be better to moosh all of the BBs together,
4970 creating one larger BB with all the computation and the unexpected
4971 jump at the end. I.e.
4973 bool zero3, zero2, zero1, zero;
4976 count3 = (N32 - N31) /[cl] STEP3;
4978 count2 = (N22 - N21) /[cl] STEP2;
4980 count1 = (N12 - N11) /[cl] STEP1;
4981 zero = zero3 || zero2 || zero1;
4982 count = count1 * count2 * count3;
4983 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4985 After all, we expect the zero=false, and thus we expect to have to
4986 evaluate all of the comparison expressions, so short-circuiting
4987 oughtn't be a win. Since the condition isn't protecting a
4988 denominator, we're not concerned about divide-by-zero, so we can
4989 fully evaluate count even if a numerator turned out to be wrong.
4991 It seems like putting this all together would create much better
4992 scheduling opportunities, and less pressure on the chip's branch
4996 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4997 basic_block
&entry_bb
, tree
*counts
,
4998 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4999 basic_block
&l2_dom_bb
)
5001 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5006 /* Collapsed loops need work for expansion into SSA form. */
5007 gcc_assert (!gimple_in_ssa_p (cfun
));
5009 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5010 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5012 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5013 isn't supposed to be handled, as the inner loop doesn't
5015 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5016 OMP_CLAUSE__LOOPTEMP_
);
5017 gcc_assert (innerc
);
5018 for (i
= 0; i
< fd
->collapse
; i
++)
5020 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5021 OMP_CLAUSE__LOOPTEMP_
);
5022 gcc_assert (innerc
);
5024 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5026 counts
[0] = NULL_TREE
;
5031 for (i
= 0; i
< fd
->collapse
; i
++)
5033 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5035 if (SSA_VAR_P (fd
->loop
.n2
)
5036 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5037 fold_convert (itype
, fd
->loops
[i
].n1
),
5038 fold_convert (itype
, fd
->loops
[i
].n2
)))
5039 == NULL_TREE
|| !integer_onep (t
)))
5042 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5043 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5044 true, GSI_SAME_STMT
);
5045 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5046 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5047 true, GSI_SAME_STMT
);
5048 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5049 NULL_TREE
, NULL_TREE
);
5050 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5051 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5052 expand_omp_regimplify_p
, NULL
, NULL
)
5053 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5054 expand_omp_regimplify_p
, NULL
, NULL
))
5056 *gsi
= gsi_for_stmt (stmt
);
5057 gimple_regimplify_operands (stmt
, gsi
);
5059 e
= split_block (entry_bb
, stmt
);
5060 if (zero_iter_bb
== NULL
)
5062 first_zero_iter
= i
;
5063 zero_iter_bb
= create_empty_bb (entry_bb
);
5064 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5065 *gsi
= gsi_after_labels (zero_iter_bb
);
5066 stmt
= gimple_build_assign (fd
->loop
.n2
,
5067 build_zero_cst (type
));
5068 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5069 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5072 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5073 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5074 e
->flags
= EDGE_TRUE_VALUE
;
5075 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5076 if (l2_dom_bb
== NULL
)
5077 l2_dom_bb
= entry_bb
;
5079 *gsi
= gsi_last_bb (entry_bb
);
5082 if (POINTER_TYPE_P (itype
))
5083 itype
= signed_type_for (itype
);
5084 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5086 t
= fold_build2 (PLUS_EXPR
, itype
,
5087 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5088 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5089 fold_convert (itype
, fd
->loops
[i
].n2
));
5090 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5091 fold_convert (itype
, fd
->loops
[i
].n1
));
5092 /* ?? We could probably use CEIL_DIV_EXPR instead of
5093 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5094 generate the same code in the end because generically we
5095 don't know that the values involved must be negative for
5097 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5098 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5099 fold_build1 (NEGATE_EXPR
, itype
, t
),
5100 fold_build1 (NEGATE_EXPR
, itype
,
5101 fold_convert (itype
,
5102 fd
->loops
[i
].step
)));
5104 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5105 fold_convert (itype
, fd
->loops
[i
].step
));
5106 t
= fold_convert (type
, t
);
5107 if (TREE_CODE (t
) == INTEGER_CST
)
5111 counts
[i
] = create_tmp_reg (type
, ".count");
5112 expand_omp_build_assign (gsi
, counts
[i
], t
);
5114 if (SSA_VAR_P (fd
->loop
.n2
))
5119 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5120 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5126 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5128 V3 = N31 + (T % count3) * STEP3;
5130 V2 = N21 + (T % count2) * STEP2;
5132 V1 = N11 + T * STEP1;
5133 if this loop doesn't have an inner loop construct combined with it.
5134 If it does have an inner loop construct combined with it and the
5135 iteration count isn't known constant, store values from counts array
5136 into its _looptemp_ temporaries instead. */
5139 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5140 tree
*counts
, gimple inner_stmt
, tree startvar
)
5143 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5145 /* If fd->loop.n2 is constant, then no propagation of the counts
5146 is needed, they are constant. */
5147 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5150 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5151 ? gimple_omp_parallel_clauses (inner_stmt
)
5152 : gimple_omp_for_clauses (inner_stmt
);
5153 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5154 isn't supposed to be handled, as the inner loop doesn't
5156 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5157 gcc_assert (innerc
);
5158 for (i
= 0; i
< fd
->collapse
; i
++)
5160 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5161 OMP_CLAUSE__LOOPTEMP_
);
5162 gcc_assert (innerc
);
5165 tree tem
= OMP_CLAUSE_DECL (innerc
);
5166 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5167 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5168 false, GSI_CONTINUE_LINKING
);
5169 gimple stmt
= gimple_build_assign (tem
, t
);
5170 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5176 tree type
= TREE_TYPE (fd
->loop
.v
);
5177 tree tem
= create_tmp_reg (type
, ".tem");
5178 gimple stmt
= gimple_build_assign (tem
, startvar
);
5179 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5181 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5183 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5185 if (POINTER_TYPE_P (vtype
))
5186 itype
= signed_type_for (vtype
);
5188 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5191 t
= fold_convert (itype
, t
);
5192 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5193 fold_convert (itype
, fd
->loops
[i
].step
));
5194 if (POINTER_TYPE_P (vtype
))
5195 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5197 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5198 t
= force_gimple_operand_gsi (gsi
, t
,
5199 DECL_P (fd
->loops
[i
].v
)
5200 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5202 GSI_CONTINUE_LINKING
);
5203 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5204 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5207 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5208 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5209 false, GSI_CONTINUE_LINKING
);
5210 stmt
= gimple_build_assign (tem
, t
);
5211 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5217 /* Helper function for expand_omp_for_*. Generate code like:
5220 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5224 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5231 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5232 basic_block body_bb
)
5234 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5236 gimple_stmt_iterator gsi
;
5242 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5244 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5246 bb
= create_empty_bb (last_bb
);
5247 add_bb_to_loop (bb
, last_bb
->loop_father
);
5248 gsi
= gsi_start_bb (bb
);
5250 if (i
< fd
->collapse
- 1)
5252 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5253 e
->probability
= REG_BR_PROB_BASE
/ 8;
5255 t
= fd
->loops
[i
+ 1].n1
;
5256 t
= force_gimple_operand_gsi (&gsi
, t
,
5257 DECL_P (fd
->loops
[i
+ 1].v
)
5258 && TREE_ADDRESSABLE (fd
->loops
[i
5261 GSI_CONTINUE_LINKING
);
5262 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5263 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5268 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5270 if (POINTER_TYPE_P (vtype
))
5271 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5273 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5274 t
= force_gimple_operand_gsi (&gsi
, t
,
5275 DECL_P (fd
->loops
[i
].v
)
5276 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5277 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5278 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5279 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5283 t
= fd
->loops
[i
].n2
;
5284 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5285 false, GSI_CONTINUE_LINKING
);
5286 tree v
= fd
->loops
[i
].v
;
5287 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5288 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5289 false, GSI_CONTINUE_LINKING
);
5290 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5291 stmt
= gimple_build_cond_empty (t
);
5292 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5293 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5294 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5297 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5305 /* A subroutine of expand_omp_for. Generate code for a parallel
5306 loop with any schedule. Given parameters:
5308 for (V = N1; V cond N2; V += STEP) BODY;
5310 where COND is "<" or ">", we generate pseudocode
5312 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5313 if (more) goto L0; else goto L3;
5320 if (V cond iend) goto L1; else goto L2;
5322 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5325 If this is a combined omp parallel loop, instead of the call to
5326 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5327 If this is gimple_omp_for_combined_p loop, then instead of assigning
5328 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5329 inner GIMPLE_OMP_FOR and V += STEP; and
5330 if (V cond iend) goto L1; else goto L2; are removed.
5332 For collapsed loops, given parameters:
5334 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5335 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5336 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5339 we generate pseudocode
5341 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5346 count3 = (adj + N32 - N31) / STEP3;
5347 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5352 count2 = (adj + N22 - N21) / STEP2;
5353 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5358 count1 = (adj + N12 - N11) / STEP1;
5359 count = count1 * count2 * count3;
5364 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5365 if (more) goto L0; else goto L3;
5369 V3 = N31 + (T % count3) * STEP3;
5371 V2 = N21 + (T % count2) * STEP2;
5373 V1 = N11 + T * STEP1;
5378 if (V < iend) goto L10; else goto L2;
5381 if (V3 cond3 N32) goto L1; else goto L11;
5385 if (V2 cond2 N22) goto L1; else goto L12;
5391 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5397 expand_omp_for_generic (struct omp_region
*region
,
5398 struct omp_for_data
*fd
,
5399 enum built_in_function start_fn
,
5400 enum built_in_function next_fn
,
5403 tree type
, istart0
, iend0
, iend
;
5404 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5405 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5406 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5407 gimple_stmt_iterator gsi
;
5409 bool in_combined_parallel
= is_combined_parallel (region
);
5410 bool broken_loop
= region
->cont
== NULL
;
5412 tree
*counts
= NULL
;
5415 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5416 gcc_assert (fd
->iter_type
== long_integer_type_node
5417 || !in_combined_parallel
);
5419 type
= TREE_TYPE (fd
->loop
.v
);
5420 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5421 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5422 TREE_ADDRESSABLE (istart0
) = 1;
5423 TREE_ADDRESSABLE (iend0
) = 1;
5425 /* See if we need to bias by LLONG_MIN. */
5426 if (fd
->iter_type
== long_long_unsigned_type_node
5427 && TREE_CODE (type
) == INTEGER_TYPE
5428 && !TYPE_UNSIGNED (type
))
5432 if (fd
->loop
.cond_code
== LT_EXPR
)
5435 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5439 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5442 if (TREE_CODE (n1
) != INTEGER_CST
5443 || TREE_CODE (n2
) != INTEGER_CST
5444 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5445 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5448 entry_bb
= region
->entry
;
5449 cont_bb
= region
->cont
;
5451 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5452 gcc_assert (broken_loop
5453 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5454 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5455 l1_bb
= single_succ (l0_bb
);
5458 l2_bb
= create_empty_bb (cont_bb
);
5459 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5460 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5464 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5465 exit_bb
= region
->exit
;
5467 gsi
= gsi_last_bb (entry_bb
);
5469 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5470 if (fd
->collapse
> 1)
5472 int first_zero_iter
= -1;
5473 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5475 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5476 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5477 zero_iter_bb
, first_zero_iter
,
5482 /* Some counts[i] vars might be uninitialized if
5483 some loop has zero iterations. But the body shouldn't
5484 be executed in that case, so just avoid uninit warnings. */
5485 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5486 if (SSA_VAR_P (counts
[i
]))
5487 TREE_NO_WARNING (counts
[i
]) = 1;
5489 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5491 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5492 gsi
= gsi_last_bb (entry_bb
);
5493 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5494 get_immediate_dominator (CDI_DOMINATORS
,
5498 if (in_combined_parallel
)
5500 /* In a combined parallel loop, emit a call to
5501 GOMP_loop_foo_next. */
5502 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5503 build_fold_addr_expr (istart0
),
5504 build_fold_addr_expr (iend0
));
5508 tree t0
, t1
, t2
, t3
, t4
;
5509 /* If this is not a combined parallel loop, emit a call to
5510 GOMP_loop_foo_start in ENTRY_BB. */
5511 t4
= build_fold_addr_expr (iend0
);
5512 t3
= build_fold_addr_expr (istart0
);
5513 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5516 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5518 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5519 OMP_CLAUSE__LOOPTEMP_
);
5520 gcc_assert (innerc
);
5521 t0
= OMP_CLAUSE_DECL (innerc
);
5522 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5523 OMP_CLAUSE__LOOPTEMP_
);
5524 gcc_assert (innerc
);
5525 t1
= OMP_CLAUSE_DECL (innerc
);
5527 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5528 && TYPE_PRECISION (TREE_TYPE (t0
))
5529 != TYPE_PRECISION (fd
->iter_type
))
5531 /* Avoid casting pointers to integer of a different size. */
5532 tree itype
= signed_type_for (type
);
5533 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5534 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5538 t1
= fold_convert (fd
->iter_type
, t1
);
5539 t0
= fold_convert (fd
->iter_type
, t0
);
5543 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5544 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5546 if (fd
->iter_type
== long_integer_type_node
)
5550 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5551 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5552 6, t0
, t1
, t2
, t
, t3
, t4
);
5555 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5556 5, t0
, t1
, t2
, t3
, t4
);
5564 /* The GOMP_loop_ull_*start functions have additional boolean
5565 argument, true for < loops and false for > loops.
5566 In Fortran, the C bool type can be different from
5567 boolean_type_node. */
5568 bfn_decl
= builtin_decl_explicit (start_fn
);
5569 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5570 t5
= build_int_cst (c_bool_type
,
5571 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5574 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5575 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5576 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5579 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5580 6, t5
, t0
, t1
, t2
, t3
, t4
);
5583 if (TREE_TYPE (t
) != boolean_type_node
)
5584 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5585 t
, build_int_cst (TREE_TYPE (t
), 0));
5586 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5587 true, GSI_SAME_STMT
);
5588 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5590 /* Remove the GIMPLE_OMP_FOR statement. */
5591 gsi_remove (&gsi
, true);
5593 /* Iteration setup for sequential loop goes in L0_BB. */
5594 tree startvar
= fd
->loop
.v
;
5595 tree endvar
= NULL_TREE
;
5597 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5599 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5600 && gimple_omp_for_kind (inner_stmt
)
5601 == GF_OMP_FOR_KIND_SIMD
);
5602 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5603 OMP_CLAUSE__LOOPTEMP_
);
5604 gcc_assert (innerc
);
5605 startvar
= OMP_CLAUSE_DECL (innerc
);
5606 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5607 OMP_CLAUSE__LOOPTEMP_
);
5608 gcc_assert (innerc
);
5609 endvar
= OMP_CLAUSE_DECL (innerc
);
5612 gsi
= gsi_start_bb (l0_bb
);
5615 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5616 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5617 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5618 t
= fold_convert (TREE_TYPE (startvar
), t
);
5619 t
= force_gimple_operand_gsi (&gsi
, t
,
5621 && TREE_ADDRESSABLE (startvar
),
5622 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5623 stmt
= gimple_build_assign (startvar
, t
);
5624 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5628 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5629 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5630 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5631 t
= fold_convert (TREE_TYPE (startvar
), t
);
5632 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5633 false, GSI_CONTINUE_LINKING
);
5636 stmt
= gimple_build_assign (endvar
, iend
);
5637 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5638 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5639 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5641 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5643 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5645 if (fd
->collapse
> 1)
5646 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5650 /* Code to control the increment and predicate for the sequential
5651 loop goes in the CONT_BB. */
5652 gsi
= gsi_last_bb (cont_bb
);
5653 stmt
= gsi_stmt (gsi
);
5654 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5655 vmain
= gimple_omp_continue_control_use (stmt
);
5656 vback
= gimple_omp_continue_control_def (stmt
);
5658 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5660 if (POINTER_TYPE_P (type
))
5661 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5663 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5664 t
= force_gimple_operand_gsi (&gsi
, t
,
5666 && TREE_ADDRESSABLE (vback
),
5667 NULL_TREE
, true, GSI_SAME_STMT
);
5668 stmt
= gimple_build_assign (vback
, t
);
5669 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5671 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5672 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5674 stmt
= gimple_build_cond_empty (t
);
5675 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5678 /* Remove GIMPLE_OMP_CONTINUE. */
5679 gsi_remove (&gsi
, true);
5681 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5682 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5684 /* Emit code to get the next parallel iteration in L2_BB. */
5685 gsi
= gsi_start_bb (l2_bb
);
5687 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5688 build_fold_addr_expr (istart0
),
5689 build_fold_addr_expr (iend0
));
5690 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5691 false, GSI_CONTINUE_LINKING
);
5692 if (TREE_TYPE (t
) != boolean_type_node
)
5693 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5694 t
, build_int_cst (TREE_TYPE (t
), 0));
5695 stmt
= gimple_build_cond_empty (t
);
5696 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5699 /* Add the loop cleanup function. */
5700 gsi
= gsi_last_bb (exit_bb
);
5701 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5702 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5703 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5704 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5706 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5707 stmt
= gimple_build_call (t
, 0);
5708 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5709 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5710 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5711 gsi_remove (&gsi
, true);
5713 /* Connect the new blocks. */
5714 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5715 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5721 e
= find_edge (cont_bb
, l3_bb
);
5722 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5724 phis
= phi_nodes (l3_bb
);
5725 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5727 gimple phi
= gsi_stmt (gsi
);
5728 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5729 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5733 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5734 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5735 e
= find_edge (cont_bb
, l1_bb
);
5736 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5741 else if (fd
->collapse
> 1)
5744 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5747 e
->flags
= EDGE_TRUE_VALUE
;
5750 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5751 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5755 e
= find_edge (cont_bb
, l2_bb
);
5756 e
->flags
= EDGE_FALLTHRU
;
5758 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5760 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5761 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5762 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5763 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5764 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5765 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5766 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5767 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5769 struct loop
*outer_loop
= alloc_loop ();
5770 outer_loop
->header
= l0_bb
;
5771 outer_loop
->latch
= l2_bb
;
5772 add_loop (outer_loop
, l0_bb
->loop_father
);
5774 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5776 struct loop
*loop
= alloc_loop ();
5777 loop
->header
= l1_bb
;
5778 /* The loop may have multiple latches. */
5779 add_loop (loop
, outer_loop
);
5785 /* A subroutine of expand_omp_for. Generate code for a parallel
5786 loop with static schedule and no specified chunk size. Given
5789 for (V = N1; V cond N2; V += STEP) BODY;
5791 where COND is "<" or ">", we generate pseudocode
5793 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5798 if ((__typeof (V)) -1 > 0 && cond is >)
5799 n = -(adj + N2 - N1) / -STEP;
5801 n = (adj + N2 - N1) / STEP;
5804 if (threadid < tt) goto L3; else goto L4;
5809 s0 = q * threadid + tt;
5812 if (s0 >= e0) goto L2; else goto L0;
5818 if (V cond e) goto L1;
5823 expand_omp_for_static_nochunk (struct omp_region
*region
,
5824 struct omp_for_data
*fd
,
5827 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5828 tree type
, itype
, vmain
, vback
;
5829 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5830 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5832 gimple_stmt_iterator gsi
;
5835 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5836 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5837 bool broken_loop
= region
->cont
== NULL
;
5838 tree
*counts
= NULL
;
5841 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5842 if (POINTER_TYPE_P (type
))
5843 itype
= signed_type_for (type
);
5845 entry_bb
= region
->entry
;
5846 cont_bb
= region
->cont
;
5847 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5848 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5849 gcc_assert (broken_loop
5850 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5851 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5852 body_bb
= single_succ (seq_start_bb
);
5855 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5856 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5858 exit_bb
= region
->exit
;
5860 /* Iteration space partitioning goes in ENTRY_BB. */
5861 gsi
= gsi_last_bb (entry_bb
);
5862 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5864 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5866 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5867 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5870 if (fd
->collapse
> 1)
5872 int first_zero_iter
= -1;
5873 basic_block l2_dom_bb
= NULL
;
5875 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5876 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5877 fin_bb
, first_zero_iter
,
5881 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5882 t
= integer_one_node
;
5884 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5885 fold_convert (type
, fd
->loop
.n1
),
5886 fold_convert (type
, fd
->loop
.n2
));
5887 if (fd
->collapse
== 1
5888 && TYPE_UNSIGNED (type
)
5889 && (t
== NULL_TREE
|| !integer_onep (t
)))
5891 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5892 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5893 true, GSI_SAME_STMT
);
5894 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5895 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5896 true, GSI_SAME_STMT
);
5897 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5898 NULL_TREE
, NULL_TREE
);
5899 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5900 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5901 expand_omp_regimplify_p
, NULL
, NULL
)
5902 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5903 expand_omp_regimplify_p
, NULL
, NULL
))
5905 gsi
= gsi_for_stmt (stmt
);
5906 gimple_regimplify_operands (stmt
, &gsi
);
5908 ep
= split_block (entry_bb
, stmt
);
5909 ep
->flags
= EDGE_TRUE_VALUE
;
5910 entry_bb
= ep
->dest
;
5911 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5912 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5913 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5914 if (gimple_in_ssa_p (cfun
))
5916 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5917 for (gsi
= gsi_start_phis (fin_bb
);
5918 !gsi_end_p (gsi
); gsi_next (&gsi
))
5920 gimple phi
= gsi_stmt (gsi
);
5921 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5922 ep
, UNKNOWN_LOCATION
);
5925 gsi
= gsi_last_bb (entry_bb
);
5928 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5929 t
= fold_convert (itype
, t
);
5930 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5931 true, GSI_SAME_STMT
);
5933 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5934 t
= fold_convert (itype
, t
);
5935 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5936 true, GSI_SAME_STMT
);
5940 step
= fd
->loop
.step
;
5941 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5943 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5944 OMP_CLAUSE__LOOPTEMP_
);
5945 gcc_assert (innerc
);
5946 n1
= OMP_CLAUSE_DECL (innerc
);
5947 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5948 OMP_CLAUSE__LOOPTEMP_
);
5949 gcc_assert (innerc
);
5950 n2
= OMP_CLAUSE_DECL (innerc
);
5952 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5953 true, NULL_TREE
, true, GSI_SAME_STMT
);
5954 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5955 true, NULL_TREE
, true, GSI_SAME_STMT
);
5956 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5957 true, NULL_TREE
, true, GSI_SAME_STMT
);
5959 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5960 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5961 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5962 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5963 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5964 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5965 fold_build1 (NEGATE_EXPR
, itype
, t
),
5966 fold_build1 (NEGATE_EXPR
, itype
, step
));
5968 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5969 t
= fold_convert (itype
, t
);
5970 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5972 q
= create_tmp_reg (itype
, "q");
5973 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5974 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5975 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5977 tt
= create_tmp_reg (itype
, "tt");
5978 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5979 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5980 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5982 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5983 stmt
= gimple_build_cond_empty (t
);
5984 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5986 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5987 gsi
= gsi_last_bb (second_bb
);
5988 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5990 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5992 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5993 build_int_cst (itype
, 1));
5994 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5996 third_bb
= split_block (second_bb
, stmt
)->dest
;
5997 gsi
= gsi_last_bb (third_bb
);
5998 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6000 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6001 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6002 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6004 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6005 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6007 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6008 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6010 /* Remove the GIMPLE_OMP_FOR statement. */
6011 gsi_remove (&gsi
, true);
6013 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6014 gsi
= gsi_start_bb (seq_start_bb
);
6016 tree startvar
= fd
->loop
.v
;
6017 tree endvar
= NULL_TREE
;
6019 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6021 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6022 ? gimple_omp_parallel_clauses (inner_stmt
)
6023 : gimple_omp_for_clauses (inner_stmt
);
6024 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6025 gcc_assert (innerc
);
6026 startvar
= OMP_CLAUSE_DECL (innerc
);
6027 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6028 OMP_CLAUSE__LOOPTEMP_
);
6029 gcc_assert (innerc
);
6030 endvar
= OMP_CLAUSE_DECL (innerc
);
6032 t
= fold_convert (itype
, s0
);
6033 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6034 if (POINTER_TYPE_P (type
))
6035 t
= fold_build_pointer_plus (n1
, t
);
6037 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6038 t
= fold_convert (TREE_TYPE (startvar
), t
);
6039 t
= force_gimple_operand_gsi (&gsi
, t
,
6041 && TREE_ADDRESSABLE (startvar
),
6042 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6043 stmt
= gimple_build_assign (startvar
, t
);
6044 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6046 t
= fold_convert (itype
, e0
);
6047 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6048 if (POINTER_TYPE_P (type
))
6049 t
= fold_build_pointer_plus (n1
, t
);
6051 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6052 t
= fold_convert (TREE_TYPE (startvar
), t
);
6053 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6054 false, GSI_CONTINUE_LINKING
);
6057 stmt
= gimple_build_assign (endvar
, e
);
6058 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6059 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6060 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6062 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6064 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6066 if (fd
->collapse
> 1)
6067 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6071 /* The code controlling the sequential loop replaces the
6072 GIMPLE_OMP_CONTINUE. */
6073 gsi
= gsi_last_bb (cont_bb
);
6074 stmt
= gsi_stmt (gsi
);
6075 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6076 vmain
= gimple_omp_continue_control_use (stmt
);
6077 vback
= gimple_omp_continue_control_def (stmt
);
6079 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6081 if (POINTER_TYPE_P (type
))
6082 t
= fold_build_pointer_plus (vmain
, step
);
6084 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6085 t
= force_gimple_operand_gsi (&gsi
, t
,
6087 && TREE_ADDRESSABLE (vback
),
6088 NULL_TREE
, true, GSI_SAME_STMT
);
6089 stmt
= gimple_build_assign (vback
, t
);
6090 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6092 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6093 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6095 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6098 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6099 gsi_remove (&gsi
, true);
6101 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6102 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6105 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6106 gsi
= gsi_last_bb (exit_bb
);
6107 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6109 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6110 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6112 gsi_remove (&gsi
, true);
6114 /* Connect all the blocks. */
6115 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6116 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6117 ep
= find_edge (entry_bb
, second_bb
);
6118 ep
->flags
= EDGE_TRUE_VALUE
;
6119 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6120 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6121 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6125 ep
= find_edge (cont_bb
, body_bb
);
6126 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6131 else if (fd
->collapse
> 1)
6134 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6137 ep
->flags
= EDGE_TRUE_VALUE
;
6138 find_edge (cont_bb
, fin_bb
)->flags
6139 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6142 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6143 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6144 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6146 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6147 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6148 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6149 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6151 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6153 struct loop
*loop
= alloc_loop ();
6154 loop
->header
= body_bb
;
6155 if (collapse_bb
== NULL
)
6156 loop
->latch
= cont_bb
;
6157 add_loop (loop
, body_bb
->loop_father
);
6162 /* A subroutine of expand_omp_for. Generate code for a parallel
6163 loop with static schedule and a specified chunk size. Given
6166 for (V = N1; V cond N2; V += STEP) BODY;
6168 where COND is "<" or ">", we generate pseudocode
6170 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6175 if ((__typeof (V)) -1 > 0 && cond is >)
6176 n = -(adj + N2 - N1) / -STEP;
6178 n = (adj + N2 - N1) / STEP;
6180 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6181 here so that V is defined
6182 if the loop is not entered
6184 s0 = (trip * nthreads + threadid) * CHUNK;
6185 e0 = min(s0 + CHUNK, n);
6186 if (s0 < n) goto L1; else goto L4;
6193 if (V cond e) goto L2; else goto L3;
6201 expand_omp_for_static_chunk (struct omp_region
*region
,
6202 struct omp_for_data
*fd
, gimple inner_stmt
)
6204 tree n
, s0
, e0
, e
, t
;
6205 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6206 tree type
, itype
, vmain
, vback
, vextra
;
6207 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6208 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6209 gimple_stmt_iterator gsi
;
6212 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6213 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6214 bool broken_loop
= region
->cont
== NULL
;
6215 tree
*counts
= NULL
;
6218 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6219 if (POINTER_TYPE_P (type
))
6220 itype
= signed_type_for (type
);
6222 entry_bb
= region
->entry
;
6223 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6225 iter_part_bb
= se
->dest
;
6226 cont_bb
= region
->cont
;
6227 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6228 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6229 gcc_assert (broken_loop
6230 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6231 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6232 body_bb
= single_succ (seq_start_bb
);
6235 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6236 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6237 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6239 exit_bb
= region
->exit
;
6241 /* Trip and adjustment setup goes in ENTRY_BB. */
6242 gsi
= gsi_last_bb (entry_bb
);
6243 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6245 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6247 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6248 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6251 if (fd
->collapse
> 1)
6253 int first_zero_iter
= -1;
6254 basic_block l2_dom_bb
= NULL
;
6256 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6257 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6258 fin_bb
, first_zero_iter
,
6262 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6263 t
= integer_one_node
;
6265 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6266 fold_convert (type
, fd
->loop
.n1
),
6267 fold_convert (type
, fd
->loop
.n2
));
6268 if (fd
->collapse
== 1
6269 && TYPE_UNSIGNED (type
)
6270 && (t
== NULL_TREE
|| !integer_onep (t
)))
6272 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6273 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6274 true, GSI_SAME_STMT
);
6275 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6276 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6277 true, GSI_SAME_STMT
);
6278 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6279 NULL_TREE
, NULL_TREE
);
6280 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6281 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6282 expand_omp_regimplify_p
, NULL
, NULL
)
6283 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6284 expand_omp_regimplify_p
, NULL
, NULL
))
6286 gsi
= gsi_for_stmt (stmt
);
6287 gimple_regimplify_operands (stmt
, &gsi
);
6289 se
= split_block (entry_bb
, stmt
);
6290 se
->flags
= EDGE_TRUE_VALUE
;
6291 entry_bb
= se
->dest
;
6292 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6293 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6294 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6295 if (gimple_in_ssa_p (cfun
))
6297 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6298 for (gsi
= gsi_start_phis (fin_bb
);
6299 !gsi_end_p (gsi
); gsi_next (&gsi
))
6301 gimple phi
= gsi_stmt (gsi
);
6302 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6303 se
, UNKNOWN_LOCATION
);
6306 gsi
= gsi_last_bb (entry_bb
);
6309 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6310 t
= fold_convert (itype
, t
);
6311 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6312 true, GSI_SAME_STMT
);
6314 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6315 t
= fold_convert (itype
, t
);
6316 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6317 true, GSI_SAME_STMT
);
6321 step
= fd
->loop
.step
;
6322 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6324 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6325 OMP_CLAUSE__LOOPTEMP_
);
6326 gcc_assert (innerc
);
6327 n1
= OMP_CLAUSE_DECL (innerc
);
6328 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6329 OMP_CLAUSE__LOOPTEMP_
);
6330 gcc_assert (innerc
);
6331 n2
= OMP_CLAUSE_DECL (innerc
);
6333 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6334 true, NULL_TREE
, true, GSI_SAME_STMT
);
6335 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6336 true, NULL_TREE
, true, GSI_SAME_STMT
);
6337 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6338 true, NULL_TREE
, true, GSI_SAME_STMT
);
6340 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6341 true, NULL_TREE
, true, GSI_SAME_STMT
);
6343 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6344 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6345 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6346 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6347 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6348 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6349 fold_build1 (NEGATE_EXPR
, itype
, t
),
6350 fold_build1 (NEGATE_EXPR
, itype
, step
));
6352 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6353 t
= fold_convert (itype
, t
);
6354 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6355 true, GSI_SAME_STMT
);
6357 trip_var
= create_tmp_reg (itype
, ".trip");
6358 if (gimple_in_ssa_p (cfun
))
6360 trip_init
= make_ssa_name (trip_var
, NULL
);
6361 trip_main
= make_ssa_name (trip_var
, NULL
);
6362 trip_back
= make_ssa_name (trip_var
, NULL
);
6366 trip_init
= trip_var
;
6367 trip_main
= trip_var
;
6368 trip_back
= trip_var
;
6371 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6372 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6374 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6375 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6376 if (POINTER_TYPE_P (type
))
6377 t
= fold_build_pointer_plus (n1
, t
);
6379 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6380 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6381 true, GSI_SAME_STMT
);
6383 /* Remove the GIMPLE_OMP_FOR. */
6384 gsi_remove (&gsi
, true);
6386 /* Iteration space partitioning goes in ITER_PART_BB. */
6387 gsi
= gsi_last_bb (iter_part_bb
);
6389 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6390 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6391 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6392 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6393 false, GSI_CONTINUE_LINKING
);
6395 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6396 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6397 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6398 false, GSI_CONTINUE_LINKING
);
6400 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6401 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6403 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6404 gsi
= gsi_start_bb (seq_start_bb
);
6406 tree startvar
= fd
->loop
.v
;
6407 tree endvar
= NULL_TREE
;
6409 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6411 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6412 ? gimple_omp_parallel_clauses (inner_stmt
)
6413 : gimple_omp_for_clauses (inner_stmt
);
6414 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6415 gcc_assert (innerc
);
6416 startvar
= OMP_CLAUSE_DECL (innerc
);
6417 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6418 OMP_CLAUSE__LOOPTEMP_
);
6419 gcc_assert (innerc
);
6420 endvar
= OMP_CLAUSE_DECL (innerc
);
6423 t
= fold_convert (itype
, s0
);
6424 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6425 if (POINTER_TYPE_P (type
))
6426 t
= fold_build_pointer_plus (n1
, t
);
6428 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6429 t
= fold_convert (TREE_TYPE (startvar
), t
);
6430 t
= force_gimple_operand_gsi (&gsi
, t
,
6432 && TREE_ADDRESSABLE (startvar
),
6433 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6434 stmt
= gimple_build_assign (startvar
, t
);
6435 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6437 t
= fold_convert (itype
, e0
);
6438 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6439 if (POINTER_TYPE_P (type
))
6440 t
= fold_build_pointer_plus (n1
, t
);
6442 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6443 t
= fold_convert (TREE_TYPE (startvar
), t
);
6444 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6445 false, GSI_CONTINUE_LINKING
);
6448 stmt
= gimple_build_assign (endvar
, e
);
6449 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6450 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6451 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6453 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6455 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6457 if (fd
->collapse
> 1)
6458 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6462 /* The code controlling the sequential loop goes in CONT_BB,
6463 replacing the GIMPLE_OMP_CONTINUE. */
6464 gsi
= gsi_last_bb (cont_bb
);
6465 stmt
= gsi_stmt (gsi
);
6466 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6467 vmain
= gimple_omp_continue_control_use (stmt
);
6468 vback
= gimple_omp_continue_control_def (stmt
);
6470 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6472 if (POINTER_TYPE_P (type
))
6473 t
= fold_build_pointer_plus (vmain
, step
);
6475 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6476 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6477 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6478 true, GSI_SAME_STMT
);
6479 stmt
= gimple_build_assign (vback
, t
);
6480 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6482 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6483 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6485 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6488 /* Remove GIMPLE_OMP_CONTINUE. */
6489 gsi_remove (&gsi
, true);
6491 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6492 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6494 /* Trip update code goes into TRIP_UPDATE_BB. */
6495 gsi
= gsi_start_bb (trip_update_bb
);
6497 t
= build_int_cst (itype
, 1);
6498 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6499 stmt
= gimple_build_assign (trip_back
, t
);
6500 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6503 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6504 gsi
= gsi_last_bb (exit_bb
);
6505 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6507 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6508 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6510 gsi_remove (&gsi
, true);
6512 /* Connect the new blocks. */
6513 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6514 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6518 se
= find_edge (cont_bb
, body_bb
);
6519 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6524 else if (fd
->collapse
> 1)
6527 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6530 se
->flags
= EDGE_TRUE_VALUE
;
6531 find_edge (cont_bb
, trip_update_bb
)->flags
6532 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6534 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6537 if (gimple_in_ssa_p (cfun
))
6539 gimple_stmt_iterator psi
;
6545 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6547 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6548 remove arguments of the phi nodes in fin_bb. We need to create
6549 appropriate phi nodes in iter_part_bb instead. */
6550 se
= single_pred_edge (fin_bb
);
6551 re
= single_succ_edge (trip_update_bb
);
6552 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
6553 ene
= single_succ_edge (entry_bb
);
6555 psi
= gsi_start_phis (fin_bb
);
6556 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6557 gsi_next (&psi
), ++i
)
6560 source_location locus
;
6562 phi
= gsi_stmt (psi
);
6563 t
= gimple_phi_result (phi
);
6564 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6565 nphi
= create_phi_node (t
, iter_part_bb
);
6567 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6568 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6570 /* A special case -- fd->loop.v is not yet computed in
6571 iter_part_bb, we need to use vextra instead. */
6572 if (t
== fd
->loop
.v
)
6574 add_phi_arg (nphi
, t
, ene
, locus
);
6575 locus
= redirect_edge_var_map_location (vm
);
6576 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6578 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6579 redirect_edge_var_map_clear (re
);
6582 psi
= gsi_start_phis (fin_bb
);
6583 if (gsi_end_p (psi
))
6585 remove_phi_node (&psi
, false);
6588 /* Make phi node for trip. */
6589 phi
= create_phi_node (trip_main
, iter_part_bb
);
6590 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6592 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6597 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6598 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6599 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6600 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6601 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6602 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6603 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6604 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6605 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6609 struct loop
*trip_loop
= alloc_loop ();
6610 trip_loop
->header
= iter_part_bb
;
6611 trip_loop
->latch
= trip_update_bb
;
6612 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6614 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6616 struct loop
*loop
= alloc_loop ();
6617 loop
->header
= body_bb
;
6618 if (collapse_bb
== NULL
)
6619 loop
->latch
= cont_bb
;
6620 add_loop (loop
, trip_loop
);
6626 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6627 loop. Given parameters:
6629 for (V = N1; V cond N2; V += STEP) BODY;
6631 where COND is "<" or ">", we generate pseudocode
6639 if (V cond N2) goto L0; else goto L2;
6642 For collapsed loops, given parameters:
6644 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6645 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6646 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6649 we generate pseudocode
6655 count3 = (adj + N32 - N31) / STEP3;
6660 count2 = (adj + N22 - N21) / STEP2;
6665 count1 = (adj + N12 - N11) / STEP1;
6666 count = count1 * count2 * count3;
6676 V2 += (V3 cond3 N32) ? 0 : STEP2;
6677 V3 = (V3 cond3 N32) ? V3 : N31;
6678 V1 += (V2 cond2 N22) ? 0 : STEP1;
6679 V2 = (V2 cond2 N22) ? V2 : N21;
6681 if (V < count) goto L0; else goto L2;
6687 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6690 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6691 gimple_stmt_iterator gsi
;
6693 bool broken_loop
= region
->cont
== NULL
;
6695 tree
*counts
= NULL
;
6697 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6698 OMP_CLAUSE_SAFELEN
);
6699 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6700 OMP_CLAUSE__SIMDUID_
);
6703 type
= TREE_TYPE (fd
->loop
.v
);
6704 entry_bb
= region
->entry
;
6705 cont_bb
= region
->cont
;
6706 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6707 gcc_assert (broken_loop
6708 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6709 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6712 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6713 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6714 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6715 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6719 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6720 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6721 l2_bb
= single_succ (l1_bb
);
6723 exit_bb
= region
->exit
;
6726 gsi
= gsi_last_bb (entry_bb
);
6728 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6729 /* Not needed in SSA form right now. */
6730 gcc_assert (!gimple_in_ssa_p (cfun
));
6731 if (fd
->collapse
> 1)
6733 int first_zero_iter
= -1;
6734 basic_block zero_iter_bb
= l2_bb
;
6736 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6737 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6738 zero_iter_bb
, first_zero_iter
,
6741 if (l2_dom_bb
== NULL
)
6746 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6748 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6749 OMP_CLAUSE__LOOPTEMP_
);
6750 gcc_assert (innerc
);
6751 n1
= OMP_CLAUSE_DECL (innerc
);
6752 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6753 OMP_CLAUSE__LOOPTEMP_
);
6754 gcc_assert (innerc
);
6755 n2
= OMP_CLAUSE_DECL (innerc
);
6756 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6757 fold_convert (type
, n1
));
6758 if (fd
->collapse
> 1)
6761 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6767 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6768 fold_convert (type
, fd
->loop
.n1
));
6769 if (fd
->collapse
> 1)
6770 for (i
= 0; i
< fd
->collapse
; i
++)
6772 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6773 if (POINTER_TYPE_P (itype
))
6774 itype
= signed_type_for (itype
);
6775 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6776 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6780 /* Remove the GIMPLE_OMP_FOR statement. */
6781 gsi_remove (&gsi
, true);
6785 /* Code to control the increment goes in the CONT_BB. */
6786 gsi
= gsi_last_bb (cont_bb
);
6787 stmt
= gsi_stmt (gsi
);
6788 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6790 if (POINTER_TYPE_P (type
))
6791 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6793 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6794 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6796 if (fd
->collapse
> 1)
6798 i
= fd
->collapse
- 1;
6799 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6801 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6802 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6806 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6808 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6811 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6813 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6815 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6816 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6817 if (POINTER_TYPE_P (itype2
))
6818 itype2
= signed_type_for (itype2
);
6819 t
= build3 (COND_EXPR
, itype2
,
6820 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6822 fold_convert (itype
, fd
->loops
[i
].n2
)),
6823 build_int_cst (itype2
, 0),
6824 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6825 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6826 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6828 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6829 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6831 t
= build3 (COND_EXPR
, itype
,
6832 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6834 fold_convert (itype
, fd
->loops
[i
].n2
)),
6836 fold_convert (itype
, fd
->loops
[i
].n1
));
6837 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6841 /* Remove GIMPLE_OMP_CONTINUE. */
6842 gsi_remove (&gsi
, true);
6845 /* Emit the condition in L1_BB. */
6846 gsi
= gsi_start_bb (l1_bb
);
6848 t
= fold_convert (type
, n2
);
6849 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6850 false, GSI_CONTINUE_LINKING
);
6851 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6852 stmt
= gimple_build_cond_empty (t
);
6853 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6854 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6856 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6859 gsi
= gsi_for_stmt (stmt
);
6860 gimple_regimplify_operands (stmt
, &gsi
);
6863 /* Remove GIMPLE_OMP_RETURN. */
6864 gsi
= gsi_last_bb (exit_bb
);
6865 gsi_remove (&gsi
, true);
6867 /* Connect the new blocks. */
6868 remove_edge (FALLTHRU_EDGE (entry_bb
));
6872 remove_edge (BRANCH_EDGE (entry_bb
));
6873 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6875 e
= BRANCH_EDGE (l1_bb
);
6876 ne
= FALLTHRU_EDGE (l1_bb
);
6877 e
->flags
= EDGE_TRUE_VALUE
;
6881 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6883 ne
= single_succ_edge (l1_bb
);
6884 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6887 ne
->flags
= EDGE_FALSE_VALUE
;
6888 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6889 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6891 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6892 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6893 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6897 struct loop
*loop
= alloc_loop ();
6898 loop
->header
= l1_bb
;
6899 loop
->latch
= cont_bb
;
6900 add_loop (loop
, l1_bb
->loop_father
);
6901 if (safelen
== NULL_TREE
)
6902 loop
->safelen
= INT_MAX
;
6905 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6906 if (TREE_CODE (safelen
) != INTEGER_CST
)
6908 else if (!tree_fits_uhwi_p (safelen
)
6909 || tree_to_uhwi (safelen
) > INT_MAX
)
6910 loop
->safelen
= INT_MAX
;
6912 loop
->safelen
= tree_to_uhwi (safelen
);
6913 if (loop
->safelen
== 1)
6918 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6919 cfun
->has_simduid_loops
= true;
6921 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6923 if ((flag_tree_loop_vectorize
6924 || (!global_options_set
.x_flag_tree_loop_vectorize
6925 && !global_options_set
.x_flag_tree_vectorize
))
6926 && flag_tree_loop_optimize
6927 && loop
->safelen
> 1)
6929 loop
->force_vectorize
= true;
6930 cfun
->has_force_vectorize_loops
= true;
6936 /* Expand the OpenMP loop defined by REGION. */
6939 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6941 struct omp_for_data fd
;
6942 struct omp_for_data_loop
*loops
;
6945 = (struct omp_for_data_loop
*)
6946 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6947 * sizeof (struct omp_for_data_loop
));
6948 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6949 region
->sched_kind
= fd
.sched_kind
;
6951 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6952 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6953 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6956 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6957 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6958 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6961 /* If there isn't a continue then this is a degerate case where
6962 the introduction of abnormal edges during lowering will prevent
6963 original loops from being detected. Fix that up. */
6964 loops_state_set (LOOPS_NEED_FIXUP
);
6966 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
6967 expand_omp_simd (region
, &fd
);
6968 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6969 && !fd
.have_ordered
)
6971 if (fd
.chunk_size
== NULL
)
6972 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6974 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6978 int fn_index
, start_ix
, next_ix
;
6980 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6981 == GF_OMP_FOR_KIND_FOR
);
6982 if (fd
.chunk_size
== NULL
6983 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6984 fd
.chunk_size
= integer_zero_node
;
6985 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6986 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6987 ? 3 : fd
.sched_kind
;
6988 fn_index
+= fd
.have_ordered
* 4;
6989 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6990 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6991 if (fd
.iter_type
== long_long_unsigned_type_node
)
6993 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6994 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6995 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6996 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6998 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6999 (enum built_in_function
) next_ix
, inner_stmt
);
7002 if (gimple_in_ssa_p (cfun
))
7003 update_ssa (TODO_update_ssa_only_virtuals
);
7007 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7009 v = GOMP_sections_start (n);
7026 v = GOMP_sections_next ();
7031 If this is a combined parallel sections, replace the call to
7032 GOMP_sections_start with call to GOMP_sections_next. */
7035 expand_omp_sections (struct omp_region
*region
)
7037 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7039 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7040 gimple_stmt_iterator si
, switch_si
;
7041 gimple sections_stmt
, stmt
, cont
;
7044 struct omp_region
*inner
;
7046 bool exit_reachable
= region
->cont
!= NULL
;
7048 gcc_assert (region
->exit
!= NULL
);
7049 entry_bb
= region
->entry
;
7050 l0_bb
= single_succ (entry_bb
);
7051 l1_bb
= region
->cont
;
7052 l2_bb
= region
->exit
;
7053 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7054 l2
= gimple_block_label (l2_bb
);
7057 /* This can happen if there are reductions. */
7058 len
= EDGE_COUNT (l0_bb
->succs
);
7059 gcc_assert (len
> 0);
7060 e
= EDGE_SUCC (l0_bb
, len
- 1);
7061 si
= gsi_last_bb (e
->dest
);
7064 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7065 l2
= gimple_block_label (e
->dest
);
7067 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7069 si
= gsi_last_bb (e
->dest
);
7071 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7073 l2
= gimple_block_label (e
->dest
);
7079 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7081 default_bb
= create_empty_bb (l0_bb
);
7083 /* We will build a switch() with enough cases for all the
7084 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7085 and a default case to abort if something goes wrong. */
7086 len
= EDGE_COUNT (l0_bb
->succs
);
7088 /* Use vec::quick_push on label_vec throughout, since we know the size
7090 auto_vec
<tree
> label_vec (len
);
7092 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7093 GIMPLE_OMP_SECTIONS statement. */
7094 si
= gsi_last_bb (entry_bb
);
7095 sections_stmt
= gsi_stmt (si
);
7096 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7097 vin
= gimple_omp_sections_control (sections_stmt
);
7098 if (!is_combined_parallel (region
))
7100 /* If we are not inside a combined parallel+sections region,
7101 call GOMP_sections_start. */
7102 t
= build_int_cst (unsigned_type_node
, len
- 1);
7103 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7104 stmt
= gimple_build_call (u
, 1, t
);
7108 /* Otherwise, call GOMP_sections_next. */
7109 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7110 stmt
= gimple_build_call (u
, 0);
7112 gimple_call_set_lhs (stmt
, vin
);
7113 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7114 gsi_remove (&si
, true);
7116 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7118 switch_si
= gsi_last_bb (l0_bb
);
7119 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7122 cont
= last_stmt (l1_bb
);
7123 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7124 vmain
= gimple_omp_continue_control_use (cont
);
7125 vnext
= gimple_omp_continue_control_def (cont
);
7133 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7134 label_vec
.quick_push (t
);
7137 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7138 for (inner
= region
->inner
, casei
= 1;
7140 inner
= inner
->next
, i
++, casei
++)
7142 basic_block s_entry_bb
, s_exit_bb
;
7144 /* Skip optional reduction region. */
7145 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7152 s_entry_bb
= inner
->entry
;
7153 s_exit_bb
= inner
->exit
;
7155 t
= gimple_block_label (s_entry_bb
);
7156 u
= build_int_cst (unsigned_type_node
, casei
);
7157 u
= build_case_label (u
, NULL
, t
);
7158 label_vec
.quick_push (u
);
7160 si
= gsi_last_bb (s_entry_bb
);
7161 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7162 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7163 gsi_remove (&si
, true);
7164 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7166 if (s_exit_bb
== NULL
)
7169 si
= gsi_last_bb (s_exit_bb
);
7170 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7171 gsi_remove (&si
, true);
7173 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7176 /* Error handling code goes in DEFAULT_BB. */
7177 t
= gimple_block_label (default_bb
);
7178 u
= build_case_label (NULL
, NULL
, t
);
7179 make_edge (l0_bb
, default_bb
, 0);
7180 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7182 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7183 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7184 gsi_remove (&switch_si
, true);
7186 si
= gsi_start_bb (default_bb
);
7187 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7188 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7194 /* Code to get the next section goes in L1_BB. */
7195 si
= gsi_last_bb (l1_bb
);
7196 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7198 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7199 stmt
= gimple_build_call (bfn_decl
, 0);
7200 gimple_call_set_lhs (stmt
, vnext
);
7201 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7202 gsi_remove (&si
, true);
7204 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7207 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7208 si
= gsi_last_bb (l2_bb
);
7209 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7210 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7211 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7212 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7214 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7215 stmt
= gimple_build_call (t
, 0);
7216 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7217 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7218 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7219 gsi_remove (&si
, true);
7221 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7225 /* Expand code for an OpenMP single directive. We've already expanded
7226 much of the code, here we simply place the GOMP_barrier call. */
7229 expand_omp_single (struct omp_region
*region
)
7231 basic_block entry_bb
, exit_bb
;
7232 gimple_stmt_iterator si
;
7234 entry_bb
= region
->entry
;
7235 exit_bb
= region
->exit
;
7237 si
= gsi_last_bb (entry_bb
);
7238 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7239 gsi_remove (&si
, true);
7240 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7242 si
= gsi_last_bb (exit_bb
);
7243 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7245 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7246 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7248 gsi_remove (&si
, true);
7249 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7253 /* Generic expansion for OpenMP synchronization directives: master,
7254 ordered and critical. All we need to do here is remove the entry
7255 and exit markers for REGION. */
7258 expand_omp_synch (struct omp_region
*region
)
7260 basic_block entry_bb
, exit_bb
;
7261 gimple_stmt_iterator si
;
7263 entry_bb
= region
->entry
;
7264 exit_bb
= region
->exit
;
7266 si
= gsi_last_bb (entry_bb
);
7267 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7268 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7269 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7270 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7271 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7272 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7273 gsi_remove (&si
, true);
7274 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7278 si
= gsi_last_bb (exit_bb
);
7279 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7280 gsi_remove (&si
, true);
7281 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7285 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7286 operation as a normal volatile load. */
7289 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7290 tree loaded_val
, int index
)
7292 enum built_in_function tmpbase
;
7293 gimple_stmt_iterator gsi
;
7294 basic_block store_bb
;
7297 tree decl
, call
, type
, itype
;
7299 gsi
= gsi_last_bb (load_bb
);
7300 stmt
= gsi_stmt (gsi
);
7301 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7302 loc
= gimple_location (stmt
);
7304 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7305 is smaller than word size, then expand_atomic_load assumes that the load
7306 is atomic. We could avoid the builtin entirely in this case. */
7308 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7309 decl
= builtin_decl_explicit (tmpbase
);
7310 if (decl
== NULL_TREE
)
7313 type
= TREE_TYPE (loaded_val
);
7314 itype
= TREE_TYPE (TREE_TYPE (decl
));
7316 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7317 build_int_cst (NULL
,
7318 gimple_omp_atomic_seq_cst_p (stmt
)
7320 : MEMMODEL_RELAXED
));
7321 if (!useless_type_conversion_p (type
, itype
))
7322 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7323 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7325 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7326 gsi_remove (&gsi
, true);
7328 store_bb
= single_succ (load_bb
);
7329 gsi
= gsi_last_bb (store_bb
);
7330 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7331 gsi_remove (&gsi
, true);
7333 if (gimple_in_ssa_p (cfun
))
7334 update_ssa (TODO_update_ssa_no_phi
);
7339 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7340 operation as a normal volatile store. */
7343 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7344 tree loaded_val
, tree stored_val
, int index
)
7346 enum built_in_function tmpbase
;
7347 gimple_stmt_iterator gsi
;
7348 basic_block store_bb
= single_succ (load_bb
);
7351 tree decl
, call
, type
, itype
;
7352 enum machine_mode imode
;
7355 gsi
= gsi_last_bb (load_bb
);
7356 stmt
= gsi_stmt (gsi
);
7357 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7359 /* If the load value is needed, then this isn't a store but an exchange. */
7360 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7362 gsi
= gsi_last_bb (store_bb
);
7363 stmt
= gsi_stmt (gsi
);
7364 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7365 loc
= gimple_location (stmt
);
7367 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7368 is smaller than word size, then expand_atomic_store assumes that the store
7369 is atomic. We could avoid the builtin entirely in this case. */
7371 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7372 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7373 decl
= builtin_decl_explicit (tmpbase
);
7374 if (decl
== NULL_TREE
)
7377 type
= TREE_TYPE (stored_val
);
7379 /* Dig out the type of the function's second argument. */
7380 itype
= TREE_TYPE (decl
);
7381 itype
= TYPE_ARG_TYPES (itype
);
7382 itype
= TREE_CHAIN (itype
);
7383 itype
= TREE_VALUE (itype
);
7384 imode
= TYPE_MODE (itype
);
7386 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7389 if (!useless_type_conversion_p (itype
, type
))
7390 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7391 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7392 build_int_cst (NULL
,
7393 gimple_omp_atomic_seq_cst_p (stmt
)
7395 : MEMMODEL_RELAXED
));
7398 if (!useless_type_conversion_p (type
, itype
))
7399 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7400 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7403 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7404 gsi_remove (&gsi
, true);
7406 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7407 gsi
= gsi_last_bb (load_bb
);
7408 gsi_remove (&gsi
, true);
7410 if (gimple_in_ssa_p (cfun
))
7411 update_ssa (TODO_update_ssa_no_phi
);
7416 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7417 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7418 size of the data type, and thus usable to find the index of the builtin
7419 decl. Returns false if the expression is not of the proper form. */
7422 expand_omp_atomic_fetch_op (basic_block load_bb
,
7423 tree addr
, tree loaded_val
,
7424 tree stored_val
, int index
)
7426 enum built_in_function oldbase
, newbase
, tmpbase
;
7427 tree decl
, itype
, call
;
7429 basic_block store_bb
= single_succ (load_bb
);
7430 gimple_stmt_iterator gsi
;
7433 enum tree_code code
;
7434 bool need_old
, need_new
;
7435 enum machine_mode imode
;
7438 /* We expect to find the following sequences:
7441 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7444 val = tmp OP something; (or: something OP tmp)
7445 GIMPLE_OMP_STORE (val)
7447 ???FIXME: Allow a more flexible sequence.
7448 Perhaps use data flow to pick the statements.
7452 gsi
= gsi_after_labels (store_bb
);
7453 stmt
= gsi_stmt (gsi
);
7454 loc
= gimple_location (stmt
);
7455 if (!is_gimple_assign (stmt
))
7458 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7460 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7461 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7462 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7463 gcc_checking_assert (!need_old
|| !need_new
);
7465 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7468 /* Check for one of the supported fetch-op operations. */
7469 code
= gimple_assign_rhs_code (stmt
);
7473 case POINTER_PLUS_EXPR
:
7474 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7475 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7478 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7479 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7482 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7483 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7486 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7487 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7490 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7491 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7497 /* Make sure the expression is of the proper form. */
7498 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7499 rhs
= gimple_assign_rhs2 (stmt
);
7500 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7501 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7502 rhs
= gimple_assign_rhs1 (stmt
);
7506 tmpbase
= ((enum built_in_function
)
7507 ((need_new
? newbase
: oldbase
) + index
+ 1));
7508 decl
= builtin_decl_explicit (tmpbase
);
7509 if (decl
== NULL_TREE
)
7511 itype
= TREE_TYPE (TREE_TYPE (decl
));
7512 imode
= TYPE_MODE (itype
);
7514 /* We could test all of the various optabs involved, but the fact of the
7515 matter is that (with the exception of i486 vs i586 and xadd) all targets
7516 that support any atomic operaton optab also implements compare-and-swap.
7517 Let optabs.c take care of expanding any compare-and-swap loop. */
7518 if (!can_compare_and_swap_p (imode
, true))
7521 gsi
= gsi_last_bb (load_bb
);
7522 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7524 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7525 It only requires that the operation happen atomically. Thus we can
7526 use the RELAXED memory model. */
7527 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7528 fold_convert_loc (loc
, itype
, rhs
),
7529 build_int_cst (NULL
,
7530 seq_cst
? MEMMODEL_SEQ_CST
7531 : MEMMODEL_RELAXED
));
7533 if (need_old
|| need_new
)
7535 lhs
= need_old
? loaded_val
: stored_val
;
7536 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7537 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7540 call
= fold_convert_loc (loc
, void_type_node
, call
);
7541 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7542 gsi_remove (&gsi
, true);
7544 gsi
= gsi_last_bb (store_bb
);
7545 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7546 gsi_remove (&gsi
, true);
7547 gsi
= gsi_last_bb (store_bb
);
7548 gsi_remove (&gsi
, true);
7550 if (gimple_in_ssa_p (cfun
))
7551 update_ssa (TODO_update_ssa_no_phi
);
7556 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7560 newval = rhs; // with oldval replacing *addr in rhs
7561 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7562 if (oldval != newval)
7565 INDEX is log2 of the size of the data type, and thus usable to find the
7566 index of the builtin decl. */
7569 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7570 tree addr
, tree loaded_val
, tree stored_val
,
7573 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7574 tree type
, itype
, cmpxchg
, iaddr
;
7575 gimple_stmt_iterator si
;
7576 basic_block loop_header
= single_succ (load_bb
);
7579 enum built_in_function fncode
;
7581 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7582 order to use the RELAXED memory model effectively. */
7583 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7585 cmpxchg
= builtin_decl_explicit (fncode
);
7586 if (cmpxchg
== NULL_TREE
)
7588 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7589 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7591 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7594 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7595 si
= gsi_last_bb (load_bb
);
7596 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7598 /* For floating-point values, we'll need to view-convert them to integers
7599 so that we can perform the atomic compare and swap. Simplify the
7600 following code by always setting up the "i"ntegral variables. */
7601 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7605 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7608 = force_gimple_operand_gsi (&si
,
7609 fold_convert (TREE_TYPE (iaddr
), addr
),
7610 false, NULL_TREE
, true, GSI_SAME_STMT
);
7611 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7612 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7613 loadedi
= create_tmp_var (itype
, NULL
);
7614 if (gimple_in_ssa_p (cfun
))
7615 loadedi
= make_ssa_name (loadedi
, NULL
);
7620 loadedi
= loaded_val
;
7623 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7624 tree loaddecl
= builtin_decl_explicit (fncode
);
7627 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
7628 build_call_expr (loaddecl
, 2, iaddr
,
7629 build_int_cst (NULL_TREE
,
7630 MEMMODEL_RELAXED
)));
7632 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
7633 build_int_cst (TREE_TYPE (iaddr
), 0));
7636 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
7639 /* Move the value to the LOADEDI temporary. */
7640 if (gimple_in_ssa_p (cfun
))
7642 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7643 phi
= create_phi_node (loadedi
, loop_header
);
7644 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7648 gsi_insert_before (&si
,
7649 gimple_build_assign (loadedi
, initial
),
7651 if (loadedi
!= loaded_val
)
7653 gimple_stmt_iterator gsi2
;
7656 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7657 gsi2
= gsi_start_bb (loop_header
);
7658 if (gimple_in_ssa_p (cfun
))
7661 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7662 true, GSI_SAME_STMT
);
7663 stmt
= gimple_build_assign (loaded_val
, x
);
7664 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7668 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7669 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7670 true, GSI_SAME_STMT
);
7673 gsi_remove (&si
, true);
7675 si
= gsi_last_bb (store_bb
);
7676 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7679 storedi
= stored_val
;
7682 force_gimple_operand_gsi (&si
,
7683 build1 (VIEW_CONVERT_EXPR
, itype
,
7684 stored_val
), true, NULL_TREE
, true,
7687 /* Build the compare&swap statement. */
7688 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7689 new_storedi
= force_gimple_operand_gsi (&si
,
7690 fold_convert (TREE_TYPE (loadedi
),
7693 true, GSI_SAME_STMT
);
7695 if (gimple_in_ssa_p (cfun
))
7699 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7700 stmt
= gimple_build_assign (old_vali
, loadedi
);
7701 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7703 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7704 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7707 /* Note that we always perform the comparison as an integer, even for
7708 floating point. This allows the atomic operation to properly
7709 succeed even with NaNs and -0.0. */
7710 stmt
= gimple_build_cond_empty
7711 (build2 (NE_EXPR
, boolean_type_node
,
7712 new_storedi
, old_vali
));
7713 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7716 e
= single_succ_edge (store_bb
);
7717 e
->flags
&= ~EDGE_FALLTHRU
;
7718 e
->flags
|= EDGE_FALSE_VALUE
;
7720 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7722 /* Copy the new value to loadedi (we already did that before the condition
7723 if we are not in SSA). */
7724 if (gimple_in_ssa_p (cfun
))
7726 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7727 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7730 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7731 gsi_remove (&si
, true);
7733 struct loop
*loop
= alloc_loop ();
7734 loop
->header
= loop_header
;
7735 loop
->latch
= store_bb
;
7736 add_loop (loop
, loop_header
->loop_father
);
7738 if (gimple_in_ssa_p (cfun
))
7739 update_ssa (TODO_update_ssa_no_phi
);
7744 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7746 GOMP_atomic_start ();
7750 The result is not globally atomic, but works so long as all parallel
7751 references are within #pragma omp atomic directives. According to
7752 responses received from omp@openmp.org, appears to be within spec.
7753 Which makes sense, since that's how several other compilers handle
7754 this situation as well.
7755 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7756 expanding. STORED_VAL is the operand of the matching
7757 GIMPLE_OMP_ATOMIC_STORE.
7760 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7764 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7769 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7770 tree addr
, tree loaded_val
, tree stored_val
)
7772 gimple_stmt_iterator si
;
7776 si
= gsi_last_bb (load_bb
);
7777 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7779 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7780 t
= build_call_expr (t
, 0);
7781 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7783 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7784 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7785 gsi_remove (&si
, true);
7787 si
= gsi_last_bb (store_bb
);
7788 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7790 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7792 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7794 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7795 t
= build_call_expr (t
, 0);
7796 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7797 gsi_remove (&si
, true);
7799 if (gimple_in_ssa_p (cfun
))
7800 update_ssa (TODO_update_ssa_no_phi
);
7804 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7805 using expand_omp_atomic_fetch_op. If it failed, we try to
7806 call expand_omp_atomic_pipeline, and if it fails too, the
7807 ultimate fallback is wrapping the operation in a mutex
7808 (expand_omp_atomic_mutex). REGION is the atomic region built
7809 by build_omp_regions_1(). */
7812 expand_omp_atomic (struct omp_region
*region
)
7814 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7815 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7816 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7817 tree addr
= gimple_omp_atomic_load_rhs (load
);
7818 tree stored_val
= gimple_omp_atomic_store_val (store
);
7819 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7820 HOST_WIDE_INT index
;
7822 /* Make sure the type is one of the supported sizes. */
7823 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7824 index
= exact_log2 (index
);
7825 if (index
>= 0 && index
<= 4)
7827 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7829 /* __sync builtins require strict data alignment. */
7830 if (exact_log2 (align
) >= index
)
7833 if (loaded_val
== stored_val
7834 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7835 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7836 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7837 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7841 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7842 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7843 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7844 && store_bb
== single_succ (load_bb
)
7845 && first_stmt (store_bb
) == store
7846 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7850 /* When possible, use specialized atomic update functions. */
7851 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7852 && store_bb
== single_succ (load_bb
)
7853 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7854 loaded_val
, stored_val
, index
))
7857 /* If we don't have specialized __sync builtins, try and implement
7858 as a compare and swap loop. */
7859 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7860 loaded_val
, stored_val
, index
))
7865 /* The ultimate fallback is wrapping the operation in a mutex. */
7866 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7870 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7873 expand_omp_target (struct omp_region
*region
)
7875 basic_block entry_bb
, exit_bb
, new_bb
;
7876 struct function
*child_cfun
= NULL
;
7877 tree child_fn
= NULL_TREE
, block
, t
;
7878 gimple_stmt_iterator gsi
;
7879 gimple entry_stmt
, stmt
;
7882 entry_stmt
= last_stmt (region
->entry
);
7883 new_bb
= region
->entry
;
7884 int kind
= gimple_omp_target_kind (entry_stmt
);
7885 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7887 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7888 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7891 entry_bb
= region
->entry
;
7892 exit_bb
= region
->exit
;
7894 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7896 unsigned srcidx
, dstidx
, num
;
7898 /* If the target region needs data sent from the parent
7899 function, then the very first statement (except possible
7900 tree profile counter updates) of the parallel body
7901 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7902 &.OMP_DATA_O is passed as an argument to the child function,
7903 we need to replace it with the argument as seen by the child
7906 In most cases, this will end up being the identity assignment
7907 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7908 a function call that has been inlined, the original PARM_DECL
7909 .OMP_DATA_I may have been converted into a different local
7910 variable. In which case, we need to keep the assignment. */
7911 if (gimple_omp_target_data_arg (entry_stmt
))
7913 basic_block entry_succ_bb
= single_succ (entry_bb
);
7914 gimple_stmt_iterator gsi
;
7916 gimple tgtcopy_stmt
= NULL
;
7918 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7920 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7922 gcc_assert (!gsi_end_p (gsi
));
7923 stmt
= gsi_stmt (gsi
);
7924 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7927 if (gimple_num_ops (stmt
) == 2)
7929 tree arg
= gimple_assign_rhs1 (stmt
);
7931 /* We're ignoring the subcode because we're
7932 effectively doing a STRIP_NOPS. */
7934 if (TREE_CODE (arg
) == ADDR_EXPR
7935 && TREE_OPERAND (arg
, 0) == sender
)
7937 tgtcopy_stmt
= stmt
;
7943 gcc_assert (tgtcopy_stmt
!= NULL
);
7944 arg
= DECL_ARGUMENTS (child_fn
);
7946 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7947 gsi_remove (&gsi
, true);
7950 /* Declare local variables needed in CHILD_CFUN. */
7951 block
= DECL_INITIAL (child_fn
);
7952 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7953 /* The gimplifier could record temporaries in target block
7954 rather than in containing function's local_decls chain,
7955 which would mean cgraph missed finalizing them. Do it now. */
7956 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7957 if (TREE_CODE (t
) == VAR_DECL
7959 && !DECL_EXTERNAL (t
))
7960 varpool_node::finalize_decl (t
);
7961 DECL_SAVED_TREE (child_fn
) = NULL
;
7962 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7963 gimple_set_body (child_fn
, NULL
);
7964 TREE_USED (block
) = 1;
7966 /* Reset DECL_CONTEXT on function arguments. */
7967 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7968 DECL_CONTEXT (t
) = child_fn
;
7970 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7971 so that it can be moved to the child function. */
7972 gsi
= gsi_last_bb (entry_bb
);
7973 stmt
= gsi_stmt (gsi
);
7974 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7975 && gimple_omp_target_kind (stmt
)
7976 == GF_OMP_TARGET_KIND_REGION
);
7977 gsi_remove (&gsi
, true);
7978 e
= split_block (entry_bb
, stmt
);
7980 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7982 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7985 gsi
= gsi_last_bb (exit_bb
);
7986 gcc_assert (!gsi_end_p (gsi
)
7987 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7988 stmt
= gimple_build_return (NULL
);
7989 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7990 gsi_remove (&gsi
, true);
7993 /* Move the target region into CHILD_CFUN. */
7995 block
= gimple_block (entry_stmt
);
7997 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7999 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8000 /* When the OMP expansion process cannot guarantee an up-to-date
8001 loop tree arrange for the child function to fixup loops. */
8002 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8003 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8005 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8006 num
= vec_safe_length (child_cfun
->local_decls
);
8007 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8009 t
= (*child_cfun
->local_decls
)[srcidx
];
8010 if (DECL_CONTEXT (t
) == cfun
->decl
)
8012 if (srcidx
!= dstidx
)
8013 (*child_cfun
->local_decls
)[dstidx
] = t
;
8017 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8019 /* Inform the callgraph about the new function. */
8020 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8021 cgraph_node::add_new_function (child_fn
, true);
8023 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8024 fixed in a following pass. */
8025 push_cfun (child_cfun
);
8026 rebuild_cgraph_edges ();
8028 /* Some EH regions might become dead, see PR34608. If
8029 pass_cleanup_cfg isn't the first pass to happen with the
8030 new child, these dead EH edges might cause problems.
8031 Clean them up now. */
8032 if (flag_exceptions
)
8035 bool changed
= false;
8037 FOR_EACH_BB_FN (bb
, cfun
)
8038 changed
|= gimple_purge_dead_eh_edges (bb
);
8040 cleanup_tree_cfg ();
8045 /* Emit a library call to launch the target region, or do data
8047 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8048 enum built_in_function start_ix
;
8049 location_t clause_loc
;
8051 clauses
= gimple_omp_target_clauses (entry_stmt
);
8053 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8054 start_ix
= BUILT_IN_GOMP_TARGET
;
8055 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8056 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8058 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8060 /* By default, the value of DEVICE is -1 (let runtime library choose)
8061 and there is no conditional. */
8063 device
= build_int_cst (integer_type_node
, -1);
8065 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8067 cond
= OMP_CLAUSE_IF_EXPR (c
);
8069 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8072 device
= OMP_CLAUSE_DEVICE_ID (c
);
8073 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8076 clause_loc
= gimple_location (entry_stmt
);
8078 /* Ensure 'device' is of the correct type. */
8079 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8081 /* If we found the clause 'if (cond)', build
8082 (cond ? device : -2). */
8085 cond
= gimple_boolify (cond
);
8087 basic_block cond_bb
, then_bb
, else_bb
;
8091 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8092 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8094 gsi
= gsi_last_bb (new_bb
);
8096 e
= split_block (new_bb
, gsi_stmt (gsi
));
8099 e
= split_block (new_bb
, NULL
);
8104 then_bb
= create_empty_bb (cond_bb
);
8105 else_bb
= create_empty_bb (then_bb
);
8106 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8107 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8109 stmt
= gimple_build_cond_empty (cond
);
8110 gsi
= gsi_last_bb (cond_bb
);
8111 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8113 gsi
= gsi_start_bb (then_bb
);
8114 stmt
= gimple_build_assign (tmp_var
, device
);
8115 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8117 gsi
= gsi_start_bb (else_bb
);
8118 stmt
= gimple_build_assign (tmp_var
,
8119 build_int_cst (integer_type_node
, -2));
8120 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8122 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8123 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8124 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8125 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8126 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8127 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8132 gsi
= gsi_last_bb (new_bb
);
8133 t
= gimple_omp_target_data_arg (entry_stmt
);
8136 t1
= size_zero_node
;
8137 t2
= build_zero_cst (ptr_type_node
);
8143 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8144 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8145 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8146 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8147 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8151 /* FIXME: This will be address of
8152 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8153 symbol, as soon as the linker plugin is able to create it for us. */
8154 tree openmp_target
= build_zero_cst (ptr_type_node
);
8155 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8157 tree fnaddr
= build_fold_addr_expr (child_fn
);
8158 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8159 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8162 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8163 device
, openmp_target
, t1
, t2
, t3
, t4
);
8164 gimple_set_location (g
, gimple_location (entry_stmt
));
8165 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8166 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8169 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8170 gsi_remove (&gsi
, true);
8172 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8174 gsi
= gsi_last_bb (region
->exit
);
8176 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8177 gsi_remove (&gsi
, true);
8182 /* Expand the parallel region tree rooted at REGION. Expansion
8183 proceeds in depth-first order. Innermost regions are expanded
8184 first. This way, parallel regions that require a new function to
8185 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8186 internal dependencies in their body. */
8189 expand_omp (struct omp_region
*region
)
8193 location_t saved_location
;
8194 gimple inner_stmt
= NULL
;
8196 /* First, determine whether this is a combined parallel+workshare
8198 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8199 determine_parallel_type (region
);
8201 if (region
->type
== GIMPLE_OMP_FOR
8202 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8203 inner_stmt
= last_stmt (region
->inner
->entry
);
8206 expand_omp (region
->inner
);
8208 saved_location
= input_location
;
8209 if (gimple_has_location (last_stmt (region
->entry
)))
8210 input_location
= gimple_location (last_stmt (region
->entry
));
8212 switch (region
->type
)
8214 case GIMPLE_OMP_PARALLEL
:
8215 case GIMPLE_OMP_TASK
:
8216 expand_omp_taskreg (region
);
8219 case GIMPLE_OMP_FOR
:
8220 expand_omp_for (region
, inner_stmt
);
8223 case GIMPLE_OMP_SECTIONS
:
8224 expand_omp_sections (region
);
8227 case GIMPLE_OMP_SECTION
:
8228 /* Individual omp sections are handled together with their
8229 parent GIMPLE_OMP_SECTIONS region. */
8232 case GIMPLE_OMP_SINGLE
:
8233 expand_omp_single (region
);
8236 case GIMPLE_OMP_MASTER
:
8237 case GIMPLE_OMP_TASKGROUP
:
8238 case GIMPLE_OMP_ORDERED
:
8239 case GIMPLE_OMP_CRITICAL
:
8240 case GIMPLE_OMP_TEAMS
:
8241 expand_omp_synch (region
);
8244 case GIMPLE_OMP_ATOMIC_LOAD
:
8245 expand_omp_atomic (region
);
8248 case GIMPLE_OMP_TARGET
:
8249 expand_omp_target (region
);
8256 input_location
= saved_location
;
8257 region
= region
->next
;
8262 /* Helper for build_omp_regions. Scan the dominator tree starting at
8263 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8264 true, the function ends once a single tree is built (otherwise, whole
8265 forest of OMP constructs may be built). */
8268 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8271 gimple_stmt_iterator gsi
;
8275 gsi
= gsi_last_bb (bb
);
8276 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8278 struct omp_region
*region
;
8279 enum gimple_code code
;
8281 stmt
= gsi_stmt (gsi
);
8282 code
= gimple_code (stmt
);
8283 if (code
== GIMPLE_OMP_RETURN
)
8285 /* STMT is the return point out of region PARENT. Mark it
8286 as the exit point and make PARENT the immediately
8287 enclosing region. */
8288 gcc_assert (parent
);
8291 parent
= parent
->outer
;
8293 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8295 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8296 GIMPLE_OMP_RETURN, but matches with
8297 GIMPLE_OMP_ATOMIC_LOAD. */
8298 gcc_assert (parent
);
8299 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8302 parent
= parent
->outer
;
8305 else if (code
== GIMPLE_OMP_CONTINUE
)
8307 gcc_assert (parent
);
8310 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8312 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8313 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8316 else if (code
== GIMPLE_OMP_TARGET
8317 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8318 new_omp_region (bb
, code
, parent
);
8321 /* Otherwise, this directive becomes the parent for a new
8323 region
= new_omp_region (bb
, code
, parent
);
8328 if (single_tree
&& !parent
)
8331 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8333 son
= next_dom_son (CDI_DOMINATORS
, son
))
8334 build_omp_regions_1 (son
, parent
, single_tree
);
8337 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8341 build_omp_regions_root (basic_block root
)
8343 gcc_assert (root_omp_region
== NULL
);
8344 build_omp_regions_1 (root
, NULL
, true);
8345 gcc_assert (root_omp_region
!= NULL
);
8348 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8351 omp_expand_local (basic_block head
)
8353 build_omp_regions_root (head
);
8354 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8356 fprintf (dump_file
, "\nOMP region tree\n\n");
8357 dump_omp_region (dump_file
, root_omp_region
, 0);
8358 fprintf (dump_file
, "\n");
8361 remove_exit_barriers (root_omp_region
);
8362 expand_omp (root_omp_region
);
8364 free_omp_regions ();
8367 /* Scan the CFG and build a tree of OMP regions. Return the root of
8368 the OMP region tree. */
8371 build_omp_regions (void)
8373 gcc_assert (root_omp_region
== NULL
);
8374 calculate_dominance_info (CDI_DOMINATORS
);
8375 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8378 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8381 execute_expand_omp (void)
8383 build_omp_regions ();
8385 if (!root_omp_region
)
8390 fprintf (dump_file
, "\nOMP region tree\n\n");
8391 dump_omp_region (dump_file
, root_omp_region
, 0);
8392 fprintf (dump_file
, "\n");
8395 remove_exit_barriers (root_omp_region
);
8397 expand_omp (root_omp_region
);
8399 cleanup_tree_cfg ();
8401 free_omp_regions ();
8406 /* OMP expansion -- the default pass, run before creation of SSA form. */
8410 const pass_data pass_data_expand_omp
=
8412 GIMPLE_PASS
, /* type */
8413 "ompexp", /* name */
8414 OPTGROUP_NONE
, /* optinfo_flags */
8415 TV_NONE
, /* tv_id */
8416 PROP_gimple_any
, /* properties_required */
8417 0, /* properties_provided */
8418 0, /* properties_destroyed */
8419 0, /* todo_flags_start */
8420 0, /* todo_flags_finish */
8423 class pass_expand_omp
: public gimple_opt_pass
8426 pass_expand_omp (gcc::context
*ctxt
)
8427 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8430 /* opt_pass methods: */
8431 virtual bool gate (function
*)
8433 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8434 || flag_cilkplus
!= 0) && !seen_error ());
8437 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8439 }; // class pass_expand_omp
8444 make_pass_expand_omp (gcc::context
*ctxt
)
8446 return new pass_expand_omp (ctxt
);
8449 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8451 /* If ctx is a worksharing context inside of a cancellable parallel
8452 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8453 and conditional branch to parallel's cancel_label to handle
8454 cancellation in the implicit barrier. */
8457 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8459 gimple omp_return
= gimple_seq_last_stmt (*body
);
8460 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8461 if (gimple_omp_return_nowait_p (omp_return
))
8464 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8465 && ctx
->outer
->cancellable
)
8467 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8468 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8469 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8470 gimple_omp_return_set_lhs (omp_return
, lhs
);
8471 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8472 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8473 fold_convert (c_bool_type
,
8474 boolean_false_node
),
8475 ctx
->outer
->cancel_label
, fallthru_label
);
8476 gimple_seq_add_stmt (body
, g
);
8477 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8481 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8482 CTX is the enclosing OMP context for the current statement. */
8485 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8487 tree block
, control
;
8488 gimple_stmt_iterator tgsi
;
8489 gimple stmt
, new_stmt
, bind
, t
;
8490 gimple_seq ilist
, dlist
, olist
, new_body
;
8492 stmt
= gsi_stmt (*gsi_p
);
8494 push_gimplify_context ();
8498 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8499 &ilist
, &dlist
, ctx
, NULL
);
8501 new_body
= gimple_omp_body (stmt
);
8502 gimple_omp_set_body (stmt
, NULL
);
8503 tgsi
= gsi_start (new_body
);
8504 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8509 sec_start
= gsi_stmt (tgsi
);
8510 sctx
= maybe_lookup_ctx (sec_start
);
8513 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8514 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8515 GSI_CONTINUE_LINKING
);
8516 gimple_omp_set_body (sec_start
, NULL
);
8518 if (gsi_one_before_end_p (tgsi
))
8520 gimple_seq l
= NULL
;
8521 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8523 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8524 gimple_omp_section_set_last (sec_start
);
8527 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8528 GSI_CONTINUE_LINKING
);
8531 block
= make_node (BLOCK
);
8532 bind
= gimple_build_bind (NULL
, new_body
, block
);
8535 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8537 block
= make_node (BLOCK
);
8538 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8539 gsi_replace (gsi_p
, new_stmt
, true);
8541 pop_gimplify_context (new_stmt
);
8542 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8543 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8544 if (BLOCK_VARS (block
))
8545 TREE_USED (block
) = 1;
8548 gimple_seq_add_seq (&new_body
, ilist
);
8549 gimple_seq_add_stmt (&new_body
, stmt
);
8550 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8551 gimple_seq_add_stmt (&new_body
, bind
);
8553 control
= create_tmp_var (unsigned_type_node
, ".section");
8554 t
= gimple_build_omp_continue (control
, control
);
8555 gimple_omp_sections_set_control (stmt
, control
);
8556 gimple_seq_add_stmt (&new_body
, t
);
8558 gimple_seq_add_seq (&new_body
, olist
);
8559 if (ctx
->cancellable
)
8560 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8561 gimple_seq_add_seq (&new_body
, dlist
);
8563 new_body
= maybe_catch_exception (new_body
);
8565 t
= gimple_build_omp_return
8566 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8567 OMP_CLAUSE_NOWAIT
));
8568 gimple_seq_add_stmt (&new_body
, t
);
8569 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8571 gimple_bind_set_body (new_stmt
, new_body
);
8575 /* A subroutine of lower_omp_single. Expand the simple form of
8576 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8578 if (GOMP_single_start ())
8580 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8582 FIXME. It may be better to delay expanding the logic of this until
8583 pass_expand_omp. The expanded logic may make the job more difficult
8584 to a synchronization analysis pass. */
8587 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8589 location_t loc
= gimple_location (single_stmt
);
8590 tree tlabel
= create_artificial_label (loc
);
8591 tree flabel
= create_artificial_label (loc
);
8595 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8596 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8597 call
= gimple_build_call (decl
, 0);
8598 gimple_call_set_lhs (call
, lhs
);
8599 gimple_seq_add_stmt (pre_p
, call
);
8601 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8602 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8605 gimple_seq_add_stmt (pre_p
, cond
);
8606 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8607 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8608 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8612 /* A subroutine of lower_omp_single. Expand the simple form of
8613 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8615 #pragma omp single copyprivate (a, b, c)
8617 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8620 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8626 GOMP_single_copy_end (©out);
8637 FIXME. It may be better to delay expanding the logic of this until
8638 pass_expand_omp. The expanded logic may make the job more difficult
8639 to a synchronization analysis pass. */
8642 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8644 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8645 gimple_seq copyin_seq
;
8646 location_t loc
= gimple_location (single_stmt
);
8648 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8650 ptr_type
= build_pointer_type (ctx
->record_type
);
8651 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8653 l0
= create_artificial_label (loc
);
8654 l1
= create_artificial_label (loc
);
8655 l2
= create_artificial_label (loc
);
8657 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8658 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8659 t
= fold_convert_loc (loc
, ptr_type
, t
);
8660 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8662 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8663 build_int_cst (ptr_type
, 0));
8664 t
= build3 (COND_EXPR
, void_type_node
, t
,
8665 build_and_jump (&l0
), build_and_jump (&l1
));
8666 gimplify_and_add (t
, pre_p
);
8668 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8670 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8673 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8676 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8677 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8678 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8679 gimplify_and_add (t
, pre_p
);
8681 t
= build_and_jump (&l2
);
8682 gimplify_and_add (t
, pre_p
);
8684 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8686 gimple_seq_add_seq (pre_p
, copyin_seq
);
8688 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8692 /* Expand code for an OpenMP single directive. */
8695 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8698 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8699 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8701 push_gimplify_context ();
8703 block
= make_node (BLOCK
);
8704 bind
= gimple_build_bind (NULL
, NULL
, block
);
8705 gsi_replace (gsi_p
, bind
, true);
8708 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8709 &bind_body
, &dlist
, ctx
, NULL
);
8710 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8712 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8714 if (ctx
->record_type
)
8715 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8717 lower_omp_single_simple (single_stmt
, &bind_body
);
8719 gimple_omp_set_body (single_stmt
, NULL
);
8721 gimple_seq_add_seq (&bind_body
, dlist
);
8723 bind_body
= maybe_catch_exception (bind_body
);
8725 t
= gimple_build_omp_return
8726 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8727 OMP_CLAUSE_NOWAIT
));
8728 gimple_seq_add_stmt (&bind_body_tail
, t
);
8729 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8730 if (ctx
->record_type
)
8732 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8733 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8734 TREE_THIS_VOLATILE (clobber
) = 1;
8735 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8736 clobber
), GSI_SAME_STMT
);
8738 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8739 gimple_bind_set_body (bind
, bind_body
);
8741 pop_gimplify_context (bind
);
8743 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8744 BLOCK_VARS (block
) = ctx
->block_vars
;
8745 if (BLOCK_VARS (block
))
8746 TREE_USED (block
) = 1;
8750 /* Expand code for an OpenMP master directive. */
8753 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8755 tree block
, lab
= NULL
, x
, bfn_decl
;
8756 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8757 location_t loc
= gimple_location (stmt
);
8760 push_gimplify_context ();
8762 block
= make_node (BLOCK
);
8763 bind
= gimple_build_bind (NULL
, NULL
, block
);
8764 gsi_replace (gsi_p
, bind
, true);
8765 gimple_bind_add_stmt (bind
, stmt
);
8767 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8768 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8769 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8770 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8772 gimplify_and_add (x
, &tseq
);
8773 gimple_bind_add_seq (bind
, tseq
);
8775 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8776 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8777 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8778 gimple_omp_set_body (stmt
, NULL
);
8780 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8782 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8784 pop_gimplify_context (bind
);
8786 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8787 BLOCK_VARS (block
) = ctx
->block_vars
;
8791 /* Expand code for an OpenMP taskgroup directive. */
8794 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8796 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8797 tree block
= make_node (BLOCK
);
8799 bind
= gimple_build_bind (NULL
, NULL
, block
);
8800 gsi_replace (gsi_p
, bind
, true);
8801 gimple_bind_add_stmt (bind
, stmt
);
8803 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8805 gimple_bind_add_stmt (bind
, x
);
8807 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8808 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8809 gimple_omp_set_body (stmt
, NULL
);
8811 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8813 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8814 BLOCK_VARS (block
) = ctx
->block_vars
;
8818 /* Expand code for an OpenMP ordered directive. */
8821 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8824 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8826 push_gimplify_context ();
8828 block
= make_node (BLOCK
);
8829 bind
= gimple_build_bind (NULL
, NULL
, block
);
8830 gsi_replace (gsi_p
, bind
, true);
8831 gimple_bind_add_stmt (bind
, stmt
);
8833 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8835 gimple_bind_add_stmt (bind
, x
);
8837 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8838 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8839 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8840 gimple_omp_set_body (stmt
, NULL
);
8842 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8843 gimple_bind_add_stmt (bind
, x
);
8845 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8847 pop_gimplify_context (bind
);
8849 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8850 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8854 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8855 substitution of a couple of function calls. But in the NAMED case,
8856 requires that languages coordinate a symbol name. It is therefore
8857 best put here in common code. */
8859 static GTY((param1_is (tree
), param2_is (tree
)))
8860 splay_tree critical_name_mutexes
;
8863 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8866 tree name
, lock
, unlock
;
8867 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8868 location_t loc
= gimple_location (stmt
);
8871 name
= gimple_omp_critical_name (stmt
);
8877 if (!critical_name_mutexes
)
8878 critical_name_mutexes
8879 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8880 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8881 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8883 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8888 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8890 new_str
= ACONCAT ((".gomp_critical_user_",
8891 IDENTIFIER_POINTER (name
), NULL
));
8892 DECL_NAME (decl
) = get_identifier (new_str
);
8893 TREE_PUBLIC (decl
) = 1;
8894 TREE_STATIC (decl
) = 1;
8895 DECL_COMMON (decl
) = 1;
8896 DECL_ARTIFICIAL (decl
) = 1;
8897 DECL_IGNORED_P (decl
) = 1;
8898 varpool_node::finalize_decl (decl
);
8900 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8901 (splay_tree_value
) decl
);
8904 decl
= (tree
) n
->value
;
8906 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8907 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8909 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8910 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8911 build_fold_addr_expr_loc (loc
, decl
));
8915 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8916 lock
= build_call_expr_loc (loc
, lock
, 0);
8918 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8919 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8922 push_gimplify_context ();
8924 block
= make_node (BLOCK
);
8925 bind
= gimple_build_bind (NULL
, NULL
, block
);
8926 gsi_replace (gsi_p
, bind
, true);
8927 gimple_bind_add_stmt (bind
, stmt
);
8929 tbody
= gimple_bind_body (bind
);
8930 gimplify_and_add (lock
, &tbody
);
8931 gimple_bind_set_body (bind
, tbody
);
8933 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8934 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8935 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8936 gimple_omp_set_body (stmt
, NULL
);
8938 tbody
= gimple_bind_body (bind
);
8939 gimplify_and_add (unlock
, &tbody
);
8940 gimple_bind_set_body (bind
, tbody
);
8942 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8944 pop_gimplify_context (bind
);
8945 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8946 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8950 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8951 for a lastprivate clause. Given a loop control predicate of (V
8952 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8953 is appended to *DLIST, iterator initialization is appended to
8957 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8958 gimple_seq
*dlist
, struct omp_context
*ctx
)
8960 tree clauses
, cond
, vinit
;
8961 enum tree_code cond_code
;
8964 cond_code
= fd
->loop
.cond_code
;
8965 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8967 /* When possible, use a strict equality expression. This can let VRP
8968 type optimizations deduce the value and remove a copy. */
8969 if (tree_fits_shwi_p (fd
->loop
.step
))
8971 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
8972 if (step
== 1 || step
== -1)
8973 cond_code
= EQ_EXPR
;
8976 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8978 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8980 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8981 if (!gimple_seq_empty_p (stmts
))
8983 gimple_seq_add_seq (&stmts
, *dlist
);
8986 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8987 vinit
= fd
->loop
.n1
;
8988 if (cond_code
== EQ_EXPR
8989 && tree_fits_shwi_p (fd
->loop
.n2
)
8990 && ! integer_zerop (fd
->loop
.n2
))
8991 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8993 vinit
= unshare_expr (vinit
);
8995 /* Initialize the iterator variable, so that threads that don't execute
8996 any iterations don't execute the lastprivate clauses by accident. */
8997 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9002 /* Lower code for an OpenMP loop directive. */
9005 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9008 struct omp_for_data fd
, *fdp
= NULL
;
9009 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9010 gimple_seq omp_for_body
, body
, dlist
;
9013 push_gimplify_context ();
9015 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9017 block
= make_node (BLOCK
);
9018 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9019 /* Replace at gsi right away, so that 'stmt' is no member
9020 of a sequence anymore as we're going to add to to a different
9022 gsi_replace (gsi_p
, new_stmt
, true);
9024 /* Move declaration of temporaries in the loop body before we make
9026 omp_for_body
= gimple_omp_body (stmt
);
9027 if (!gimple_seq_empty_p (omp_for_body
)
9028 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9030 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9031 tree vars
= gimple_bind_vars (inner_bind
);
9032 gimple_bind_append_vars (new_stmt
, vars
);
9033 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9034 keep them on the inner_bind and it's block. */
9035 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9036 if (gimple_bind_block (inner_bind
))
9037 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9040 if (gimple_omp_for_combined_into_p (stmt
))
9042 extract_omp_for_data (stmt
, &fd
, NULL
);
9045 /* We need two temporaries with fd.loop.v type (istart/iend)
9046 and then (fd.collapse - 1) temporaries with the same
9047 type for count2 ... countN-1 vars if not constant. */
9049 tree type
= fd
.iter_type
;
9051 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9052 count
+= fd
.collapse
- 1;
9053 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9054 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9058 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9059 OMP_CLAUSE__LOOPTEMP_
);
9060 for (i
= 0; i
< count
; i
++)
9065 gcc_assert (outerc
);
9066 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9067 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9068 OMP_CLAUSE__LOOPTEMP_
);
9072 temp
= create_tmp_var (type
, NULL
);
9073 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9075 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9076 OMP_CLAUSE_DECL (*pc
) = temp
;
9077 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9082 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9085 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9087 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9089 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9091 /* Lower the header expressions. At this point, we can assume that
9092 the header is of the form:
9094 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9096 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9097 using the .omp_data_s mapping, if needed. */
9098 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9100 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9101 if (!is_gimple_min_invariant (*rhs_p
))
9102 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9104 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9105 if (!is_gimple_min_invariant (*rhs_p
))
9106 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9108 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9109 if (!is_gimple_min_invariant (*rhs_p
))
9110 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9113 /* Once lowered, extract the bounds and clauses. */
9114 extract_omp_for_data (stmt
, &fd
, NULL
);
9116 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9118 gimple_seq_add_stmt (&body
, stmt
);
9119 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9121 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9124 /* After the loop, add exit clauses. */
9125 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9127 if (ctx
->cancellable
)
9128 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9130 gimple_seq_add_seq (&body
, dlist
);
9132 body
= maybe_catch_exception (body
);
9134 /* Region exit marker goes at the end of the loop body. */
9135 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9136 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9137 pop_gimplify_context (new_stmt
);
9139 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9140 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9141 if (BLOCK_VARS (block
))
9142 TREE_USED (block
) = 1;
9144 gimple_bind_set_body (new_stmt
, body
);
9145 gimple_omp_set_body (stmt
, NULL
);
9146 gimple_omp_for_set_pre_body (stmt
, NULL
);
9149 /* Callback for walk_stmts. Check if the current statement only contains
9150 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9153 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9154 bool *handled_ops_p
,
9155 struct walk_stmt_info
*wi
)
9157 int *info
= (int *) wi
->info
;
9158 gimple stmt
= gsi_stmt (*gsi_p
);
9160 *handled_ops_p
= true;
9161 switch (gimple_code (stmt
))
9165 case GIMPLE_OMP_FOR
:
9166 case GIMPLE_OMP_SECTIONS
:
9167 *info
= *info
== 0 ? 1 : -1;
9176 struct omp_taskcopy_context
9178 /* This field must be at the beginning, as we do "inheritance": Some
9179 callback functions for tree-inline.c (e.g., omp_copy_decl)
9180 receive a copy_body_data pointer that is up-casted to an
9181 omp_context pointer. */
9187 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9189 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9191 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9192 return create_tmp_var (TREE_TYPE (var
), NULL
);
9198 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9200 tree name
, new_fields
= NULL
, type
, f
;
9202 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9203 name
= DECL_NAME (TYPE_NAME (orig_type
));
9204 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9205 TYPE_DECL
, name
, type
);
9206 TYPE_NAME (type
) = name
;
9208 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9210 tree new_f
= copy_node (f
);
9211 DECL_CONTEXT (new_f
) = type
;
9212 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9213 TREE_CHAIN (new_f
) = new_fields
;
9214 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9215 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9216 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9219 tcctx
->cb
.decl_map
->put (f
, new_f
);
9221 TYPE_FIELDS (type
) = nreverse (new_fields
);
9226 /* Create task copyfn. */
9229 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9231 struct function
*child_cfun
;
9232 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9233 tree record_type
, srecord_type
, bind
, list
;
9234 bool record_needs_remap
= false, srecord_needs_remap
= false;
9236 struct omp_taskcopy_context tcctx
;
9237 location_t loc
= gimple_location (task_stmt
);
9239 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9240 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9241 gcc_assert (child_cfun
->cfg
== NULL
);
9242 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9244 /* Reset DECL_CONTEXT on function arguments. */
9245 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9246 DECL_CONTEXT (t
) = child_fn
;
9248 /* Populate the function. */
9249 push_gimplify_context ();
9250 push_cfun (child_cfun
);
9252 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9253 TREE_SIDE_EFFECTS (bind
) = 1;
9255 DECL_SAVED_TREE (child_fn
) = bind
;
9256 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9258 /* Remap src and dst argument types if needed. */
9259 record_type
= ctx
->record_type
;
9260 srecord_type
= ctx
->srecord_type
;
9261 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9262 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9264 record_needs_remap
= true;
9267 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9268 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9270 srecord_needs_remap
= true;
9274 if (record_needs_remap
|| srecord_needs_remap
)
9276 memset (&tcctx
, '\0', sizeof (tcctx
));
9277 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9278 tcctx
.cb
.dst_fn
= child_fn
;
9279 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
9280 gcc_checking_assert (tcctx
.cb
.src_node
);
9281 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9282 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9283 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9284 tcctx
.cb
.eh_lp_nr
= 0;
9285 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9286 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
9289 if (record_needs_remap
)
9290 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9291 if (srecord_needs_remap
)
9292 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9295 tcctx
.cb
.decl_map
= NULL
;
9297 arg
= DECL_ARGUMENTS (child_fn
);
9298 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9299 sarg
= DECL_CHAIN (arg
);
9300 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9302 /* First pass: initialize temporaries used in record_type and srecord_type
9303 sizes and field offsets. */
9304 if (tcctx
.cb
.decl_map
)
9305 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9306 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9310 decl
= OMP_CLAUSE_DECL (c
);
9311 p
= tcctx
.cb
.decl_map
->get (decl
);
9314 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9315 sf
= (tree
) n
->value
;
9316 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9317 src
= build_simple_mem_ref_loc (loc
, sarg
);
9318 src
= omp_build_component_ref (src
, sf
);
9319 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9320 append_to_statement_list (t
, &list
);
9323 /* Second pass: copy shared var pointers and copy construct non-VLA
9324 firstprivate vars. */
9325 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9326 switch (OMP_CLAUSE_CODE (c
))
9328 case OMP_CLAUSE_SHARED
:
9329 decl
= OMP_CLAUSE_DECL (c
);
9330 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9333 f
= (tree
) n
->value
;
9334 if (tcctx
.cb
.decl_map
)
9335 f
= *tcctx
.cb
.decl_map
->get (f
);
9336 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9337 sf
= (tree
) n
->value
;
9338 if (tcctx
.cb
.decl_map
)
9339 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9340 src
= build_simple_mem_ref_loc (loc
, sarg
);
9341 src
= omp_build_component_ref (src
, sf
);
9342 dst
= build_simple_mem_ref_loc (loc
, arg
);
9343 dst
= omp_build_component_ref (dst
, f
);
9344 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9345 append_to_statement_list (t
, &list
);
9347 case OMP_CLAUSE_FIRSTPRIVATE
:
9348 decl
= OMP_CLAUSE_DECL (c
);
9349 if (is_variable_sized (decl
))
9351 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9354 f
= (tree
) n
->value
;
9355 if (tcctx
.cb
.decl_map
)
9356 f
= *tcctx
.cb
.decl_map
->get (f
);
9357 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9360 sf
= (tree
) n
->value
;
9361 if (tcctx
.cb
.decl_map
)
9362 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9363 src
= build_simple_mem_ref_loc (loc
, sarg
);
9364 src
= omp_build_component_ref (src
, sf
);
9365 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9366 src
= build_simple_mem_ref_loc (loc
, src
);
9370 dst
= build_simple_mem_ref_loc (loc
, arg
);
9371 dst
= omp_build_component_ref (dst
, f
);
9372 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9373 append_to_statement_list (t
, &list
);
9375 case OMP_CLAUSE_PRIVATE
:
9376 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9378 decl
= OMP_CLAUSE_DECL (c
);
9379 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9380 f
= (tree
) n
->value
;
9381 if (tcctx
.cb
.decl_map
)
9382 f
= *tcctx
.cb
.decl_map
->get (f
);
9383 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9386 sf
= (tree
) n
->value
;
9387 if (tcctx
.cb
.decl_map
)
9388 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9389 src
= build_simple_mem_ref_loc (loc
, sarg
);
9390 src
= omp_build_component_ref (src
, sf
);
9391 if (use_pointer_for_field (decl
, NULL
))
9392 src
= build_simple_mem_ref_loc (loc
, src
);
9396 dst
= build_simple_mem_ref_loc (loc
, arg
);
9397 dst
= omp_build_component_ref (dst
, f
);
9398 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9399 append_to_statement_list (t
, &list
);
9405 /* Last pass: handle VLA firstprivates. */
9406 if (tcctx
.cb
.decl_map
)
9407 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9408 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9412 decl
= OMP_CLAUSE_DECL (c
);
9413 if (!is_variable_sized (decl
))
9415 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9418 f
= (tree
) n
->value
;
9419 f
= *tcctx
.cb
.decl_map
->get (f
);
9420 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9421 ind
= DECL_VALUE_EXPR (decl
);
9422 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9423 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9424 n
= splay_tree_lookup (ctx
->sfield_map
,
9425 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9426 sf
= (tree
) n
->value
;
9427 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9428 src
= build_simple_mem_ref_loc (loc
, sarg
);
9429 src
= omp_build_component_ref (src
, sf
);
9430 src
= build_simple_mem_ref_loc (loc
, src
);
9431 dst
= build_simple_mem_ref_loc (loc
, arg
);
9432 dst
= omp_build_component_ref (dst
, f
);
9433 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9434 append_to_statement_list (t
, &list
);
9435 n
= splay_tree_lookup (ctx
->field_map
,
9436 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9437 df
= (tree
) n
->value
;
9438 df
= *tcctx
.cb
.decl_map
->get (df
);
9439 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9440 ptr
= omp_build_component_ref (ptr
, df
);
9441 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9442 build_fold_addr_expr_loc (loc
, dst
));
9443 append_to_statement_list (t
, &list
);
9446 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9447 append_to_statement_list (t
, &list
);
9449 if (tcctx
.cb
.decl_map
)
9450 delete tcctx
.cb
.decl_map
;
9451 pop_gimplify_context (NULL
);
9452 BIND_EXPR_BODY (bind
) = list
;
9457 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9461 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9463 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9465 gcc_assert (clauses
);
9466 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9467 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9468 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9470 case OMP_CLAUSE_DEPEND_IN
:
9473 case OMP_CLAUSE_DEPEND_OUT
:
9474 case OMP_CLAUSE_DEPEND_INOUT
:
9480 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9481 tree array
= create_tmp_var (type
, NULL
);
9482 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9484 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9485 gimple_seq_add_stmt (iseq
, g
);
9486 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9488 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9489 gimple_seq_add_stmt (iseq
, g
);
9490 for (i
= 0; i
< 2; i
++)
9492 if ((i
? n_in
: n_out
) == 0)
9494 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9495 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9496 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9498 tree t
= OMP_CLAUSE_DECL (c
);
9499 t
= fold_convert (ptr_type_node
, t
);
9500 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9501 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9502 NULL_TREE
, NULL_TREE
);
9503 g
= gimple_build_assign (r
, t
);
9504 gimple_seq_add_stmt (iseq
, g
);
9507 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9508 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9509 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9510 OMP_CLAUSE_CHAIN (c
) = *p
;
9512 tree clobber
= build_constructor (type
, NULL
);
9513 TREE_THIS_VOLATILE (clobber
) = 1;
9514 g
= gimple_build_assign (array
, clobber
);
9515 gimple_seq_add_stmt (oseq
, g
);
9518 /* Lower the OpenMP parallel or task directive in the current statement
9519 in GSI_P. CTX holds context information for the directive. */
9522 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9526 gimple stmt
= gsi_stmt (*gsi_p
);
9527 gimple par_bind
, bind
, dep_bind
= NULL
;
9528 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9529 location_t loc
= gimple_location (stmt
);
9531 clauses
= gimple_omp_taskreg_clauses (stmt
);
9532 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9533 par_body
= gimple_bind_body (par_bind
);
9534 child_fn
= ctx
->cb
.dst_fn
;
9535 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9536 && !gimple_omp_parallel_combined_p (stmt
))
9538 struct walk_stmt_info wi
;
9541 memset (&wi
, 0, sizeof (wi
));
9544 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9546 gimple_omp_parallel_set_combined_p (stmt
, true);
9548 gimple_seq dep_ilist
= NULL
;
9549 gimple_seq dep_olist
= NULL
;
9550 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9551 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9553 push_gimplify_context ();
9554 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9555 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9558 if (ctx
->srecord_type
)
9559 create_task_copyfn (stmt
, ctx
);
9561 push_gimplify_context ();
9566 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9567 lower_omp (&par_body
, ctx
);
9568 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9569 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9571 /* Declare all the variables created by mapping and the variables
9572 declared in the scope of the parallel body. */
9573 record_vars_into (ctx
->block_vars
, child_fn
);
9574 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9576 if (ctx
->record_type
)
9579 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9580 : ctx
->record_type
, ".omp_data_o");
9581 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9582 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9583 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9588 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9589 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9591 if (ctx
->record_type
)
9593 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9594 TREE_THIS_VOLATILE (clobber
) = 1;
9595 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9599 /* Once all the expansions are done, sequence all the different
9600 fragments inside gimple_omp_body. */
9604 if (ctx
->record_type
)
9606 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9607 /* fixup_child_record_type might have changed receiver_decl's type. */
9608 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9609 gimple_seq_add_stmt (&new_body
,
9610 gimple_build_assign (ctx
->receiver_decl
, t
));
9613 gimple_seq_add_seq (&new_body
, par_ilist
);
9614 gimple_seq_add_seq (&new_body
, par_body
);
9615 gimple_seq_add_seq (&new_body
, par_rlist
);
9616 if (ctx
->cancellable
)
9617 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9618 gimple_seq_add_seq (&new_body
, par_olist
);
9619 new_body
= maybe_catch_exception (new_body
);
9620 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9621 gimple_omp_set_body (stmt
, new_body
);
9623 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9624 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9625 gimple_bind_add_seq (bind
, ilist
);
9626 gimple_bind_add_stmt (bind
, stmt
);
9627 gimple_bind_add_seq (bind
, olist
);
9629 pop_gimplify_context (NULL
);
9633 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9634 gimple_bind_add_stmt (dep_bind
, bind
);
9635 gimple_bind_add_seq (dep_bind
, dep_olist
);
9636 pop_gimplify_context (dep_bind
);
9640 /* Lower the OpenMP target directive in the current statement
9641 in GSI_P. CTX holds context information for the directive. */
9644 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9647 tree child_fn
, t
, c
;
9648 gimple stmt
= gsi_stmt (*gsi_p
);
9649 gimple tgt_bind
= NULL
, bind
;
9650 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9651 location_t loc
= gimple_location (stmt
);
9652 int kind
= gimple_omp_target_kind (stmt
);
9653 unsigned int map_cnt
= 0;
9655 clauses
= gimple_omp_target_clauses (stmt
);
9656 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9658 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9659 tgt_body
= gimple_bind_body (tgt_bind
);
9661 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9662 tgt_body
= gimple_omp_body (stmt
);
9663 child_fn
= ctx
->cb
.dst_fn
;
9665 push_gimplify_context ();
9667 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9668 switch (OMP_CLAUSE_CODE (c
))
9674 case OMP_CLAUSE_MAP
:
9676 case OMP_CLAUSE_FROM
:
9677 var
= OMP_CLAUSE_DECL (c
);
9680 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9681 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9687 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9689 tree var2
= DECL_VALUE_EXPR (var
);
9690 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9691 var2
= TREE_OPERAND (var2
, 0);
9692 gcc_assert (DECL_P (var2
));
9696 if (!maybe_lookup_field (var
, ctx
))
9699 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9701 x
= build_receiver_ref (var
, true, ctx
);
9702 tree new_var
= lookup_decl (var
, ctx
);
9703 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9704 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9705 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9706 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9707 x
= build_simple_mem_ref (x
);
9708 SET_DECL_VALUE_EXPR (new_var
, x
);
9709 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9714 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9716 target_nesting_level
++;
9717 lower_omp (&tgt_body
, ctx
);
9718 target_nesting_level
--;
9720 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9721 lower_omp (&tgt_body
, ctx
);
9723 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9725 /* Declare all the variables created by mapping and the variables
9726 declared in the scope of the target body. */
9727 record_vars_into (ctx
->block_vars
, child_fn
);
9728 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9733 if (ctx
->record_type
)
9736 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9737 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9738 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9739 t
= make_tree_vec (3);
9740 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9742 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9744 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9745 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9746 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9748 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9751 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9752 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9753 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9754 gimple_omp_target_set_data_arg (stmt
, t
);
9756 vec
<constructor_elt
, va_gc
> *vsize
;
9757 vec
<constructor_elt
, va_gc
> *vkind
;
9758 vec_alloc (vsize
, map_cnt
);
9759 vec_alloc (vkind
, map_cnt
);
9760 unsigned int map_idx
= 0;
9762 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9763 switch (OMP_CLAUSE_CODE (c
))
9769 case OMP_CLAUSE_MAP
:
9771 case OMP_CLAUSE_FROM
:
9773 ovar
= OMP_CLAUSE_DECL (c
);
9776 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9777 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9779 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9780 == get_base_address (ovar
));
9781 nc
= OMP_CLAUSE_CHAIN (c
);
9782 ovar
= OMP_CLAUSE_DECL (nc
);
9786 tree x
= build_sender_ref (ovar
, ctx
);
9788 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9789 gimplify_assign (x
, v
, &ilist
);
9795 if (DECL_SIZE (ovar
)
9796 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9798 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9799 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9800 ovar2
= TREE_OPERAND (ovar2
, 0);
9801 gcc_assert (DECL_P (ovar2
));
9804 if (!maybe_lookup_field (ovar
, ctx
))
9810 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9811 tree x
= build_sender_ref (ovar
, ctx
);
9812 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9813 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9814 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9815 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9817 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9819 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9820 mark_addressable (avar
);
9821 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9822 avar
= build_fold_addr_expr (avar
);
9823 gimplify_assign (x
, avar
, &ilist
);
9825 else if (is_gimple_reg (var
))
9827 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9828 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9829 mark_addressable (avar
);
9830 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9831 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9832 gimplify_assign (avar
, var
, &ilist
);
9833 avar
= build_fold_addr_expr (avar
);
9834 gimplify_assign (x
, avar
, &ilist
);
9835 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9836 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9837 && !TYPE_READONLY (TREE_TYPE (var
)))
9839 x
= build_sender_ref (ovar
, ctx
);
9840 x
= build_simple_mem_ref (x
);
9841 gimplify_assign (var
, x
, &olist
);
9846 var
= build_fold_addr_expr (var
);
9847 gimplify_assign (x
, var
, &ilist
);
9850 tree s
= OMP_CLAUSE_SIZE (c
);
9852 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9853 s
= fold_convert (size_type_node
, s
);
9854 tree purpose
= size_int (map_idx
++);
9855 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9856 if (TREE_CODE (s
) != INTEGER_CST
)
9857 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9859 unsigned char tkind
= 0;
9860 switch (OMP_CLAUSE_CODE (c
))
9862 case OMP_CLAUSE_MAP
:
9863 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9866 tkind
= OMP_CLAUSE_MAP_TO
;
9868 case OMP_CLAUSE_FROM
:
9869 tkind
= OMP_CLAUSE_MAP_FROM
;
9874 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9875 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9876 talign
= DECL_ALIGN_UNIT (ovar
);
9877 talign
= ceil_log2 (talign
);
9878 tkind
|= talign
<< 3;
9879 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9880 build_int_cst (unsigned_char_type_node
,
9886 gcc_assert (map_idx
== map_cnt
);
9888 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9889 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9890 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9891 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9892 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9894 gimple_seq initlist
= NULL
;
9895 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9896 TREE_VEC_ELT (t
, 1)),
9897 &initlist
, true, NULL_TREE
);
9898 gimple_seq_add_seq (&ilist
, initlist
);
9900 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
9902 TREE_THIS_VOLATILE (clobber
) = 1;
9903 gimple_seq_add_stmt (&olist
,
9904 gimple_build_assign (TREE_VEC_ELT (t
, 1),
9908 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9909 TREE_THIS_VOLATILE (clobber
) = 1;
9910 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9914 /* Once all the expansions are done, sequence all the different
9915 fragments inside gimple_omp_body. */
9919 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9921 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9922 /* fixup_child_record_type might have changed receiver_decl's type. */
9923 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9924 gimple_seq_add_stmt (&new_body
,
9925 gimple_build_assign (ctx
->receiver_decl
, t
));
9928 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9930 gimple_seq_add_seq (&new_body
, tgt_body
);
9931 new_body
= maybe_catch_exception (new_body
);
9933 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9934 new_body
= tgt_body
;
9935 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9937 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9938 gimple_omp_set_body (stmt
, new_body
);
9941 bind
= gimple_build_bind (NULL
, NULL
,
9942 tgt_bind
? gimple_bind_block (tgt_bind
)
9944 gsi_replace (gsi_p
, bind
, true);
9945 gimple_bind_add_seq (bind
, ilist
);
9946 gimple_bind_add_stmt (bind
, stmt
);
9947 gimple_bind_add_seq (bind
, olist
);
9949 pop_gimplify_context (NULL
);
9952 /* Expand code for an OpenMP teams directive. */
9955 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9957 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9958 push_gimplify_context ();
9960 tree block
= make_node (BLOCK
);
9961 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9962 gsi_replace (gsi_p
, bind
, true);
9963 gimple_seq bind_body
= NULL
;
9964 gimple_seq dlist
= NULL
;
9965 gimple_seq olist
= NULL
;
9967 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9968 OMP_CLAUSE_NUM_TEAMS
);
9969 if (num_teams
== NULL_TREE
)
9970 num_teams
= build_int_cst (unsigned_type_node
, 0);
9973 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9974 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9975 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9977 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9978 OMP_CLAUSE_THREAD_LIMIT
);
9979 if (thread_limit
== NULL_TREE
)
9980 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9983 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9984 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9985 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9989 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9990 &bind_body
, &dlist
, ctx
, NULL
);
9991 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9992 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9993 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9995 location_t loc
= gimple_location (teams_stmt
);
9996 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9997 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9998 gimple_set_location (call
, loc
);
9999 gimple_seq_add_stmt (&bind_body
, call
);
10001 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10002 gimple_omp_set_body (teams_stmt
, NULL
);
10003 gimple_seq_add_seq (&bind_body
, olist
);
10004 gimple_seq_add_seq (&bind_body
, dlist
);
10005 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10006 gimple_bind_set_body (bind
, bind_body
);
10008 pop_gimplify_context (bind
);
10010 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10011 BLOCK_VARS (block
) = ctx
->block_vars
;
10012 if (BLOCK_VARS (block
))
10013 TREE_USED (block
) = 1;
10017 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10018 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10019 of OpenMP context, but with task_shared_vars set. */
10022 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10027 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10028 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10031 if (task_shared_vars
10033 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10036 /* If a global variable has been privatized, TREE_CONSTANT on
10037 ADDR_EXPR might be wrong. */
10038 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10039 recompute_tree_invariant_for_addr_expr (t
);
10041 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10046 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10048 gimple stmt
= gsi_stmt (*gsi_p
);
10049 struct walk_stmt_info wi
;
10051 if (gimple_has_location (stmt
))
10052 input_location
= gimple_location (stmt
);
10054 if (task_shared_vars
)
10055 memset (&wi
, '\0', sizeof (wi
));
10057 /* If we have issued syntax errors, avoid doing any heavy lifting.
10058 Just replace the OpenMP directives with a NOP to avoid
10059 confusing RTL expansion. */
10060 if (seen_error () && is_gimple_omp (stmt
))
10062 gsi_replace (gsi_p
, gimple_build_nop (), true);
10066 switch (gimple_code (stmt
))
10069 if ((ctx
|| task_shared_vars
)
10070 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10071 ctx
? NULL
: &wi
, NULL
)
10072 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10073 ctx
? NULL
: &wi
, NULL
)))
10074 gimple_regimplify_operands (stmt
, gsi_p
);
10077 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10079 case GIMPLE_EH_FILTER
:
10080 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10083 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10084 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10086 case GIMPLE_TRANSACTION
:
10087 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10090 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10092 case GIMPLE_OMP_PARALLEL
:
10093 case GIMPLE_OMP_TASK
:
10094 ctx
= maybe_lookup_ctx (stmt
);
10096 if (ctx
->cancellable
)
10097 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10098 lower_omp_taskreg (gsi_p
, ctx
);
10100 case GIMPLE_OMP_FOR
:
10101 ctx
= maybe_lookup_ctx (stmt
);
10103 if (ctx
->cancellable
)
10104 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10105 lower_omp_for (gsi_p
, ctx
);
10107 case GIMPLE_OMP_SECTIONS
:
10108 ctx
= maybe_lookup_ctx (stmt
);
10110 if (ctx
->cancellable
)
10111 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10112 lower_omp_sections (gsi_p
, ctx
);
10114 case GIMPLE_OMP_SINGLE
:
10115 ctx
= maybe_lookup_ctx (stmt
);
10117 lower_omp_single (gsi_p
, ctx
);
10119 case GIMPLE_OMP_MASTER
:
10120 ctx
= maybe_lookup_ctx (stmt
);
10122 lower_omp_master (gsi_p
, ctx
);
10124 case GIMPLE_OMP_TASKGROUP
:
10125 ctx
= maybe_lookup_ctx (stmt
);
10127 lower_omp_taskgroup (gsi_p
, ctx
);
10129 case GIMPLE_OMP_ORDERED
:
10130 ctx
= maybe_lookup_ctx (stmt
);
10132 lower_omp_ordered (gsi_p
, ctx
);
10134 case GIMPLE_OMP_CRITICAL
:
10135 ctx
= maybe_lookup_ctx (stmt
);
10137 lower_omp_critical (gsi_p
, ctx
);
10139 case GIMPLE_OMP_ATOMIC_LOAD
:
10140 if ((ctx
|| task_shared_vars
)
10141 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10142 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10143 gimple_regimplify_operands (stmt
, gsi_p
);
10145 case GIMPLE_OMP_TARGET
:
10146 ctx
= maybe_lookup_ctx (stmt
);
10148 lower_omp_target (gsi_p
, ctx
);
10150 case GIMPLE_OMP_TEAMS
:
10151 ctx
= maybe_lookup_ctx (stmt
);
10153 lower_omp_teams (gsi_p
, ctx
);
10157 fndecl
= gimple_call_fndecl (stmt
);
10159 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10160 switch (DECL_FUNCTION_CODE (fndecl
))
10162 case BUILT_IN_GOMP_BARRIER
:
10166 case BUILT_IN_GOMP_CANCEL
:
10167 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10170 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10171 cctx
= cctx
->outer
;
10172 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10173 if (!cctx
->cancellable
)
10175 if (DECL_FUNCTION_CODE (fndecl
)
10176 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10178 stmt
= gimple_build_nop ();
10179 gsi_replace (gsi_p
, stmt
, false);
10183 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10185 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10186 gimple_call_set_fndecl (stmt
, fndecl
);
10187 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10190 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10191 gimple_call_set_lhs (stmt
, lhs
);
10192 tree fallthru_label
;
10193 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10195 g
= gimple_build_label (fallthru_label
);
10196 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10197 g
= gimple_build_cond (NE_EXPR
, lhs
,
10198 fold_convert (TREE_TYPE (lhs
),
10199 boolean_false_node
),
10200 cctx
->cancel_label
, fallthru_label
);
10201 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10208 if ((ctx
|| task_shared_vars
)
10209 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10212 /* Just remove clobbers, this should happen only if we have
10213 "privatized" local addressable variables in SIMD regions,
10214 the clobber isn't needed in that case and gimplifying address
10215 of the ARRAY_REF into a pointer and creating MEM_REF based
10216 clobber would create worse code than we get with the clobber
10218 if (gimple_clobber_p (stmt
))
10220 gsi_replace (gsi_p
, gimple_build_nop (), true);
10223 gimple_regimplify_operands (stmt
, gsi_p
);
10230 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10232 location_t saved_location
= input_location
;
10233 gimple_stmt_iterator gsi
;
10234 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10235 lower_omp_1 (&gsi
, ctx
);
10236 /* During gimplification, we have not always invoked fold_stmt
10237 (gimplify.c:maybe_fold_stmt); call it now. */
10238 if (target_nesting_level
)
10239 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10241 input_location
= saved_location
;
10244 /* Main entry point. */
10246 static unsigned int
10247 execute_lower_omp (void)
10251 /* This pass always runs, to provide PROP_gimple_lomp.
10252 But there is nothing to do unless -fopenmp is given. */
10253 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10256 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10257 delete_omp_context
);
10259 body
= gimple_body (current_function_decl
);
10260 scan_omp (&body
, NULL
);
10261 gcc_assert (taskreg_nesting_level
== 0);
10263 if (all_contexts
->root
)
10265 if (task_shared_vars
)
10266 push_gimplify_context ();
10267 lower_omp (&body
, NULL
);
10268 if (task_shared_vars
)
10269 pop_gimplify_context (NULL
);
10274 splay_tree_delete (all_contexts
);
10275 all_contexts
= NULL
;
10277 BITMAP_FREE (task_shared_vars
);
10283 const pass_data pass_data_lower_omp
=
10285 GIMPLE_PASS
, /* type */
10286 "omplower", /* name */
10287 OPTGROUP_NONE
, /* optinfo_flags */
10288 TV_NONE
, /* tv_id */
10289 PROP_gimple_any
, /* properties_required */
10290 PROP_gimple_lomp
, /* properties_provided */
10291 0, /* properties_destroyed */
10292 0, /* todo_flags_start */
10293 0, /* todo_flags_finish */
10296 class pass_lower_omp
: public gimple_opt_pass
10299 pass_lower_omp (gcc::context
*ctxt
)
10300 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10303 /* opt_pass methods: */
10304 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10306 }; // class pass_lower_omp
10308 } // anon namespace
10311 make_pass_lower_omp (gcc::context
*ctxt
)
10313 return new pass_lower_omp (ctxt
);
10316 /* The following is a utility to diagnose OpenMP structured block violations.
10317 It is not part of the "omplower" pass, as that's invoked too late. It
10318 should be invoked by the respective front ends after gimplification. */
10320 static splay_tree all_labels
;
10322 /* Check for mismatched contexts and generate an error if needed. Return
10323 true if an error is detected. */
10326 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10327 gimple branch_ctx
, gimple label_ctx
)
10329 if (label_ctx
== branch_ctx
)
10334 Previously we kept track of the label's entire context in diagnose_sb_[12]
10335 so we could traverse it and issue a correct "exit" or "enter" error
10336 message upon a structured block violation.
10338 We built the context by building a list with tree_cons'ing, but there is
10339 no easy counterpart in gimple tuples. It seems like far too much work
10340 for issuing exit/enter error messages. If someone really misses the
10341 distinct error message... patches welcome.
10345 /* Try to avoid confusing the user by producing and error message
10346 with correct "exit" or "enter" verbiage. We prefer "exit"
10347 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10348 if (branch_ctx
== NULL
)
10354 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10359 label_ctx
= TREE_CHAIN (label_ctx
);
10364 error ("invalid exit from OpenMP structured block");
10366 error ("invalid entry to OpenMP structured block");
10369 bool cilkplus_block
= false;
10373 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10374 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10376 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10377 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10378 cilkplus_block
= true;
10381 /* If it's obvious we have an invalid entry, be specific about the error. */
10382 if (branch_ctx
== NULL
)
10384 if (cilkplus_block
)
10385 error ("invalid entry to Cilk Plus structured block");
10387 error ("invalid entry to OpenMP structured block");
10391 /* Otherwise, be vague and lazy, but efficient. */
10392 if (cilkplus_block
)
10393 error ("invalid branch to/from a Cilk Plus structured block");
10395 error ("invalid branch to/from an OpenMP structured block");
10398 gsi_replace (gsi_p
, gimple_build_nop (), false);
10402 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10403 where each label is found. */
10406 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10407 struct walk_stmt_info
*wi
)
10409 gimple context
= (gimple
) wi
->info
;
10410 gimple inner_context
;
10411 gimple stmt
= gsi_stmt (*gsi_p
);
10413 *handled_ops_p
= true;
10415 switch (gimple_code (stmt
))
10419 case GIMPLE_OMP_PARALLEL
:
10420 case GIMPLE_OMP_TASK
:
10421 case GIMPLE_OMP_SECTIONS
:
10422 case GIMPLE_OMP_SINGLE
:
10423 case GIMPLE_OMP_SECTION
:
10424 case GIMPLE_OMP_MASTER
:
10425 case GIMPLE_OMP_ORDERED
:
10426 case GIMPLE_OMP_CRITICAL
:
10427 case GIMPLE_OMP_TARGET
:
10428 case GIMPLE_OMP_TEAMS
:
10429 case GIMPLE_OMP_TASKGROUP
:
10430 /* The minimal context here is just the current OMP construct. */
10431 inner_context
= stmt
;
10432 wi
->info
= inner_context
;
10433 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10434 wi
->info
= context
;
10437 case GIMPLE_OMP_FOR
:
10438 inner_context
= stmt
;
10439 wi
->info
= inner_context
;
10440 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10442 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10443 diagnose_sb_1
, NULL
, wi
);
10444 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10445 wi
->info
= context
;
10449 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10450 (splay_tree_value
) context
);
10460 /* Pass 2: Check each branch and see if its context differs from that of
10461 the destination label's context. */
10464 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10465 struct walk_stmt_info
*wi
)
10467 gimple context
= (gimple
) wi
->info
;
10469 gimple stmt
= gsi_stmt (*gsi_p
);
10471 *handled_ops_p
= true;
10473 switch (gimple_code (stmt
))
10477 case GIMPLE_OMP_PARALLEL
:
10478 case GIMPLE_OMP_TASK
:
10479 case GIMPLE_OMP_SECTIONS
:
10480 case GIMPLE_OMP_SINGLE
:
10481 case GIMPLE_OMP_SECTION
:
10482 case GIMPLE_OMP_MASTER
:
10483 case GIMPLE_OMP_ORDERED
:
10484 case GIMPLE_OMP_CRITICAL
:
10485 case GIMPLE_OMP_TARGET
:
10486 case GIMPLE_OMP_TEAMS
:
10487 case GIMPLE_OMP_TASKGROUP
:
10489 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10490 wi
->info
= context
;
10493 case GIMPLE_OMP_FOR
:
10495 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10497 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10498 diagnose_sb_2
, NULL
, wi
);
10499 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10500 wi
->info
= context
;
10505 tree lab
= gimple_cond_true_label (stmt
);
10508 n
= splay_tree_lookup (all_labels
,
10509 (splay_tree_key
) lab
);
10510 diagnose_sb_0 (gsi_p
, context
,
10511 n
? (gimple
) n
->value
: NULL
);
10513 lab
= gimple_cond_false_label (stmt
);
10516 n
= splay_tree_lookup (all_labels
,
10517 (splay_tree_key
) lab
);
10518 diagnose_sb_0 (gsi_p
, context
,
10519 n
? (gimple
) n
->value
: NULL
);
10526 tree lab
= gimple_goto_dest (stmt
);
10527 if (TREE_CODE (lab
) != LABEL_DECL
)
10530 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10531 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10535 case GIMPLE_SWITCH
:
10538 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10540 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10541 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10542 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10548 case GIMPLE_RETURN
:
10549 diagnose_sb_0 (gsi_p
, context
, NULL
);
10559 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10562 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10565 gimple last
= last_stmt (bb
);
10566 enum gimple_code code
= gimple_code (last
);
10567 struct omp_region
*cur_region
= *region
;
10568 bool fallthru
= false;
10572 case GIMPLE_OMP_PARALLEL
:
10573 case GIMPLE_OMP_TASK
:
10574 case GIMPLE_OMP_FOR
:
10575 case GIMPLE_OMP_SINGLE
:
10576 case GIMPLE_OMP_TEAMS
:
10577 case GIMPLE_OMP_MASTER
:
10578 case GIMPLE_OMP_TASKGROUP
:
10579 case GIMPLE_OMP_ORDERED
:
10580 case GIMPLE_OMP_CRITICAL
:
10581 case GIMPLE_OMP_SECTION
:
10582 cur_region
= new_omp_region (bb
, code
, cur_region
);
10586 case GIMPLE_OMP_TARGET
:
10587 cur_region
= new_omp_region (bb
, code
, cur_region
);
10589 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10590 cur_region
= cur_region
->outer
;
10593 case GIMPLE_OMP_SECTIONS
:
10594 cur_region
= new_omp_region (bb
, code
, cur_region
);
10598 case GIMPLE_OMP_SECTIONS_SWITCH
:
10602 case GIMPLE_OMP_ATOMIC_LOAD
:
10603 case GIMPLE_OMP_ATOMIC_STORE
:
10607 case GIMPLE_OMP_RETURN
:
10608 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10609 somewhere other than the next block. This will be
10611 cur_region
->exit
= bb
;
10612 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10613 cur_region
= cur_region
->outer
;
10616 case GIMPLE_OMP_CONTINUE
:
10617 cur_region
->cont
= bb
;
10618 switch (cur_region
->type
)
10620 case GIMPLE_OMP_FOR
:
10621 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10622 succs edges as abnormal to prevent splitting
10624 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10625 /* Make the loopback edge. */
10626 make_edge (bb
, single_succ (cur_region
->entry
),
10629 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10630 corresponds to the case that the body of the loop
10631 is not executed at all. */
10632 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10633 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10637 case GIMPLE_OMP_SECTIONS
:
10638 /* Wire up the edges into and out of the nested sections. */
10640 basic_block switch_bb
= single_succ (cur_region
->entry
);
10642 struct omp_region
*i
;
10643 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10645 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10646 make_edge (switch_bb
, i
->entry
, 0);
10647 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10650 /* Make the loopback edge to the block with
10651 GIMPLE_OMP_SECTIONS_SWITCH. */
10652 make_edge (bb
, switch_bb
, 0);
10654 /* Make the edge from the switch to exit. */
10655 make_edge (switch_bb
, bb
->next_bb
, 0);
10661 gcc_unreachable ();
10666 gcc_unreachable ();
10669 if (*region
!= cur_region
)
10671 *region
= cur_region
;
10673 *region_idx
= cur_region
->entry
->index
;
10681 static unsigned int
10682 diagnose_omp_structured_block_errors (void)
10684 struct walk_stmt_info wi
;
10685 gimple_seq body
= gimple_body (current_function_decl
);
10687 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10689 memset (&wi
, 0, sizeof (wi
));
10690 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10692 memset (&wi
, 0, sizeof (wi
));
10693 wi
.want_locations
= true;
10694 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10696 gimple_set_body (current_function_decl
, body
);
10698 splay_tree_delete (all_labels
);
10706 const pass_data pass_data_diagnose_omp_blocks
=
10708 GIMPLE_PASS
, /* type */
10709 "*diagnose_omp_blocks", /* name */
10710 OPTGROUP_NONE
, /* optinfo_flags */
10711 TV_NONE
, /* tv_id */
10712 PROP_gimple_any
, /* properties_required */
10713 0, /* properties_provided */
10714 0, /* properties_destroyed */
10715 0, /* todo_flags_start */
10716 0, /* todo_flags_finish */
10719 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10722 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10723 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10726 /* opt_pass methods: */
10727 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
10728 virtual unsigned int execute (function
*)
10730 return diagnose_omp_structured_block_errors ();
10733 }; // class pass_diagnose_omp_blocks
10735 } // anon namespace
10738 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10740 return new pass_diagnose_omp_blocks (ctxt
);
10743 /* SIMD clone supporting code. */
10745 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10746 of arguments to reserve space for. */
10748 static struct cgraph_simd_clone
*
10749 simd_clone_struct_alloc (int nargs
)
10751 struct cgraph_simd_clone
*clone_info
;
10752 size_t len
= (sizeof (struct cgraph_simd_clone
)
10753 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
10754 clone_info
= (struct cgraph_simd_clone
*)
10755 ggc_internal_cleared_alloc (len
);
10759 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10762 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
10763 struct cgraph_simd_clone
*from
)
10765 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
10766 + ((from
->nargs
- from
->inbranch
)
10767 * sizeof (struct cgraph_simd_clone_arg
))));
10770 /* Return vector of parameter types of function FNDECL. This uses
10771 TYPE_ARG_TYPES if available, otherwise falls back to types of
10772 DECL_ARGUMENTS types. */
10775 simd_clone_vector_of_formal_parm_types (tree fndecl
)
10777 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
10778 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
10779 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
10782 FOR_EACH_VEC_ELT (args
, i
, arg
)
10783 args
[i
] = TREE_TYPE (args
[i
]);
10787 /* Given a simd function in NODE, extract the simd specific
10788 information from the OMP clauses passed in CLAUSES, and return
10789 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10790 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10791 otherwise set to FALSE. */
10793 static struct cgraph_simd_clone
*
10794 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
10795 bool *inbranch_specified
)
10797 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
10800 *inbranch_specified
= false;
10802 n
= args
.length ();
10803 if (n
> 0 && args
.last () == void_type_node
)
10806 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10807 be cloned have a distinctive artificial label in addition to "omp
10811 && lookup_attribute ("cilk simd function",
10812 DECL_ATTRIBUTES (node
->decl
)));
10814 /* Allocate one more than needed just in case this is an in-branch
10815 clone which will require a mask argument. */
10816 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
10817 clone_info
->nargs
= n
;
10818 clone_info
->cilk_elemental
= cilk_clone
;
10825 clauses
= TREE_VALUE (clauses
);
10826 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
10829 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
10831 switch (OMP_CLAUSE_CODE (t
))
10833 case OMP_CLAUSE_INBRANCH
:
10834 clone_info
->inbranch
= 1;
10835 *inbranch_specified
= true;
10837 case OMP_CLAUSE_NOTINBRANCH
:
10838 clone_info
->inbranch
= 0;
10839 *inbranch_specified
= true;
10841 case OMP_CLAUSE_SIMDLEN
:
10842 clone_info
->simdlen
10843 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
10845 case OMP_CLAUSE_LINEAR
:
10847 tree decl
= OMP_CLAUSE_DECL (t
);
10848 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
10849 int argno
= TREE_INT_CST_LOW (decl
);
10850 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
10852 clone_info
->args
[argno
].arg_type
10853 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
10854 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10855 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
10856 && clone_info
->args
[argno
].linear_step
< n
);
10860 if (POINTER_TYPE_P (args
[argno
]))
10861 step
= fold_convert (ssizetype
, step
);
10862 if (!tree_fits_shwi_p (step
))
10864 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10865 "ignoring large linear step");
10869 else if (integer_zerop (step
))
10871 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10872 "ignoring zero linear step");
10878 clone_info
->args
[argno
].arg_type
10879 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
10880 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10885 case OMP_CLAUSE_UNIFORM
:
10887 tree decl
= OMP_CLAUSE_DECL (t
);
10888 int argno
= tree_to_uhwi (decl
);
10889 clone_info
->args
[argno
].arg_type
10890 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
10893 case OMP_CLAUSE_ALIGNED
:
10895 tree decl
= OMP_CLAUSE_DECL (t
);
10896 int argno
= tree_to_uhwi (decl
);
10897 clone_info
->args
[argno
].alignment
10898 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
10909 /* Given a SIMD clone in NODE, calculate the characteristic data
10910 type and return the coresponding type. The characteristic data
10911 type is computed as described in the Intel Vector ABI. */
10914 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
10915 struct cgraph_simd_clone
*clone_info
)
10917 tree type
= integer_type_node
;
10918 tree fndecl
= node
->decl
;
10920 /* a) For non-void function, the characteristic data type is the
10922 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
10923 type
= TREE_TYPE (TREE_TYPE (fndecl
));
10925 /* b) If the function has any non-uniform, non-linear parameters,
10926 then the characteristic data type is the type of the first
10930 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
10931 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
10932 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
10940 /* c) If the characteristic data type determined by a) or b) above
10941 is struct, union, or class type which is pass-by-value (except
10942 for the type that maps to the built-in complex data type), the
10943 characteristic data type is int. */
10944 if (RECORD_OR_UNION_TYPE_P (type
)
10945 && !aggregate_value_p (type
, NULL
)
10946 && TREE_CODE (type
) != COMPLEX_TYPE
)
10947 return integer_type_node
;
10949 /* d) If none of the above three classes is applicable, the
10950 characteristic data type is int. */
10954 /* e) For Intel Xeon Phi native and offload compilation, if the
10955 resulting characteristic data type is 8-bit or 16-bit integer
10956 data type, the characteristic data type is int. */
10957 /* Well, we don't handle Xeon Phi yet. */
10961 simd_clone_mangle (struct cgraph_node
*node
,
10962 struct cgraph_simd_clone
*clone_info
)
10964 char vecsize_mangle
= clone_info
->vecsize_mangle
;
10965 char mask
= clone_info
->inbranch
? 'M' : 'N';
10966 unsigned int simdlen
= clone_info
->simdlen
;
10970 gcc_assert (vecsize_mangle
&& simdlen
);
10972 pp_string (&pp
, "_ZGV");
10973 pp_character (&pp
, vecsize_mangle
);
10974 pp_character (&pp
, mask
);
10975 pp_decimal_int (&pp
, simdlen
);
10977 for (n
= 0; n
< clone_info
->nargs
; ++n
)
10979 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
10981 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
10982 pp_character (&pp
, 'u');
10983 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
10985 gcc_assert (arg
.linear_step
!= 0);
10986 pp_character (&pp
, 'l');
10987 if (arg
.linear_step
> 1)
10988 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
10989 else if (arg
.linear_step
< 0)
10991 pp_character (&pp
, 'n');
10992 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
10996 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
10998 pp_character (&pp
, 's');
10999 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11002 pp_character (&pp
, 'v');
11005 pp_character (&pp
, 'a');
11006 pp_decimal_int (&pp
, arg
.alignment
);
11010 pp_underscore (&pp
);
11012 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11013 const char *str
= pp_formatted_text (&pp
);
11015 /* If there already is a SIMD clone with the same mangled name, don't
11016 add another one. This can happen e.g. for
11017 #pragma omp declare simd
11018 #pragma omp declare simd simdlen(8)
11019 int foo (int, int);
11020 if the simdlen is assumed to be 8 for the first one, etc. */
11021 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11022 clone
= clone
->simdclone
->next_clone
)
11023 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11027 return get_identifier (str
);
11030 /* Create a simd clone of OLD_NODE and return it. */
11032 static struct cgraph_node
*
11033 simd_clone_create (struct cgraph_node
*old_node
)
11035 struct cgraph_node
*new_node
;
11036 if (old_node
->definition
)
11038 if (!old_node
->has_gimple_body_p ())
11040 old_node
->get_body ();
11041 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
11047 tree old_decl
= old_node
->decl
;
11048 tree new_decl
= copy_node (old_node
->decl
);
11049 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11050 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11051 SET_DECL_RTL (new_decl
, NULL
);
11052 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11053 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11054 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
11055 new_node
->call_function_insertion_hooks ();
11057 if (new_node
== NULL
)
11060 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11062 /* The function cgraph_function_versioning () will force the new
11063 symbol local. Undo this, and inherit external visability from
11065 new_node
->local
.local
= old_node
->local
.local
;
11066 new_node
->externally_visible
= old_node
->externally_visible
;
11071 /* Adjust the return type of the given function to its appropriate
11072 vector counterpart. Returns a simd array to be used throughout the
11073 function as a return value. */
11076 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11078 tree fndecl
= node
->decl
;
11079 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11080 unsigned int veclen
;
11083 /* Adjust the function return type. */
11084 if (orig_rettype
== void_type_node
)
11086 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11087 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
11088 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
11089 veclen
= node
->simdclone
->vecsize_int
;
11091 veclen
= node
->simdclone
->vecsize_float
;
11092 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
11093 if (veclen
> node
->simdclone
->simdlen
)
11094 veclen
= node
->simdclone
->simdlen
;
11095 if (veclen
== node
->simdclone
->simdlen
)
11096 TREE_TYPE (TREE_TYPE (fndecl
))
11097 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
11098 node
->simdclone
->simdlen
);
11101 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
11102 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11103 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11105 if (!node
->definition
)
11108 t
= DECL_RESULT (fndecl
);
11109 /* Adjust the DECL_RESULT. */
11110 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11111 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11114 tree atype
= build_array_type_nelts (orig_rettype
,
11115 node
->simdclone
->simdlen
);
11116 if (veclen
!= node
->simdclone
->simdlen
)
11117 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11119 /* Set up a SIMD array to use as the return value. */
11120 tree retval
= create_tmp_var_raw (atype
, "retval");
11121 gimple_add_tmp_var (retval
);
11125 /* Each vector argument has a corresponding array to be used locally
11126 as part of the eventual loop. Create such temporary array and
11129 PREFIX is the prefix to be used for the temporary.
11131 TYPE is the inner element type.
11133 SIMDLEN is the number of elements. */
11136 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11138 tree atype
= build_array_type_nelts (type
, simdlen
);
11139 tree avar
= create_tmp_var_raw (atype
, prefix
);
11140 gimple_add_tmp_var (avar
);
11144 /* Modify the function argument types to their corresponding vector
11145 counterparts if appropriate. Also, create one array for each simd
11146 argument to be used locally when using the function arguments as
11149 NODE is the function whose arguments are to be adjusted.
11151 Returns an adjustment vector that will be filled describing how the
11152 argument types will be adjusted. */
11154 static ipa_parm_adjustment_vec
11155 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11158 ipa_parm_adjustment_vec adjustments
;
11160 if (node
->definition
)
11161 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11163 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11164 adjustments
.create (args
.length ());
11165 unsigned i
, j
, veclen
;
11166 struct ipa_parm_adjustment adj
;
11167 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11169 memset (&adj
, 0, sizeof (adj
));
11170 tree parm
= args
[i
];
11171 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11172 adj
.base_index
= i
;
11175 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11176 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11178 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11180 /* No adjustment necessary for scalar arguments. */
11181 adj
.op
= IPA_PARM_OP_COPY
;
11185 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11186 veclen
= node
->simdclone
->vecsize_int
;
11188 veclen
= node
->simdclone
->vecsize_float
;
11189 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11190 if (veclen
> node
->simdclone
->simdlen
)
11191 veclen
= node
->simdclone
->simdlen
;
11192 adj
.arg_prefix
= "simd";
11193 adj
.type
= build_vector_type (parm_type
, veclen
);
11194 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11195 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11197 adjustments
.safe_push (adj
);
11200 memset (&adj
, 0, sizeof (adj
));
11201 adj
.op
= IPA_PARM_OP_NEW
;
11202 adj
.arg_prefix
= "simd";
11203 adj
.base_index
= i
;
11204 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11208 if (node
->definition
)
11209 node
->simdclone
->args
[i
].simd_array
11210 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11211 parm_type
, node
->simdclone
->simdlen
);
11213 adjustments
.safe_push (adj
);
11216 if (node
->simdclone
->inbranch
)
11219 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11222 memset (&adj
, 0, sizeof (adj
));
11223 adj
.op
= IPA_PARM_OP_NEW
;
11224 adj
.arg_prefix
= "mask";
11226 adj
.base_index
= i
;
11227 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11228 veclen
= node
->simdclone
->vecsize_int
;
11230 veclen
= node
->simdclone
->vecsize_float
;
11231 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11232 if (veclen
> node
->simdclone
->simdlen
)
11233 veclen
= node
->simdclone
->simdlen
;
11234 adj
.type
= build_vector_type (base_type
, veclen
);
11235 adjustments
.safe_push (adj
);
11237 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11238 adjustments
.safe_push (adj
);
11240 /* We have previously allocated one extra entry for the mask. Use
11242 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11244 if (node
->definition
)
11246 sc
->args
[i
].orig_arg
11247 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11248 sc
->args
[i
].simd_array
11249 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11251 sc
->args
[i
].orig_type
= base_type
;
11252 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11255 if (node
->definition
)
11256 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11259 tree new_arg_types
= NULL_TREE
, new_reversed
;
11260 bool last_parm_void
= false;
11261 if (args
.length () > 0 && args
.last () == void_type_node
)
11262 last_parm_void
= true;
11264 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11265 j
= adjustments
.length ();
11266 for (i
= 0; i
< j
; i
++)
11268 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11270 if (adj
->op
== IPA_PARM_OP_COPY
)
11271 ptype
= args
[adj
->base_index
];
11274 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11276 new_reversed
= nreverse (new_arg_types
);
11277 if (last_parm_void
)
11280 TREE_CHAIN (new_arg_types
) = void_list_node
;
11282 new_reversed
= void_list_node
;
11285 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11286 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11287 TREE_TYPE (node
->decl
) = new_type
;
11289 adjustments
.release ();
11292 return adjustments
;
11295 /* Initialize and copy the function arguments in NODE to their
11296 corresponding local simd arrays. Returns a fresh gimple_seq with
11297 the instruction sequence generated. */
11300 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11301 ipa_parm_adjustment_vec adjustments
)
11303 gimple_seq seq
= NULL
;
11304 unsigned i
= 0, j
= 0, k
;
11306 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11308 arg
= DECL_CHAIN (arg
), i
++, j
++)
11310 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11313 node
->simdclone
->args
[i
].vector_arg
= arg
;
11315 tree array
= node
->simdclone
->args
[i
].simd_array
;
11316 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11318 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11319 tree ptr
= build_fold_addr_expr (array
);
11320 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11321 build_int_cst (ptype
, 0));
11322 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11323 gimplify_and_add (t
, &seq
);
11327 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11328 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11329 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11331 tree ptr
= build_fold_addr_expr (array
);
11335 arg
= DECL_CHAIN (arg
);
11339 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11340 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11341 build_int_cst (ptype
, k
* elemsize
));
11342 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11343 gimplify_and_add (t
, &seq
);
11350 /* Callback info for ipa_simd_modify_stmt_ops below. */
11352 struct modify_stmt_info
{
11353 ipa_parm_adjustment_vec adjustments
;
11355 /* True if the parent statement was modified by
11356 ipa_simd_modify_stmt_ops. */
11360 /* Callback for walk_gimple_op.
11362 Adjust operands from a given statement as specified in the
11363 adjustments vector in the callback data. */
11366 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11368 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11369 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11370 tree
*orig_tp
= tp
;
11371 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11372 tp
= &TREE_OPERAND (*tp
, 0);
11373 struct ipa_parm_adjustment
*cand
= NULL
;
11374 if (TREE_CODE (*tp
) == PARM_DECL
)
11375 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11379 *walk_subtrees
= 0;
11382 tree repl
= NULL_TREE
;
11384 repl
= unshare_expr (cand
->new_decl
);
11389 *walk_subtrees
= 0;
11390 bool modified
= info
->modified
;
11391 info
->modified
= false;
11392 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11393 if (!info
->modified
)
11395 info
->modified
= modified
;
11398 info
->modified
= modified
;
11407 repl
= build_fold_addr_expr (repl
);
11409 = gimple_build_assign (make_ssa_name (TREE_TYPE (repl
), NULL
), repl
);
11410 repl
= gimple_assign_lhs (stmt
);
11411 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11412 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11415 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11417 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11423 info
->modified
= true;
11427 /* Traverse the function body and perform all modifications as
11428 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11429 modified such that the replacement/reduction value will now be an
11430 offset into the corresponding simd_array.
11432 This function will replace all function argument uses with their
11433 corresponding simd array elements, and ajust the return values
11437 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11438 ipa_parm_adjustment_vec adjustments
,
11439 tree retval_array
, tree iter
)
11442 unsigned int i
, j
, l
;
11444 /* Re-use the adjustments array, but this time use it to replace
11445 every function argument use to an offset into the corresponding
11447 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11449 if (!node
->simdclone
->args
[i
].vector_arg
)
11452 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11453 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11454 adjustments
[j
].new_decl
11455 = build4 (ARRAY_REF
,
11457 node
->simdclone
->args
[i
].simd_array
,
11459 NULL_TREE
, NULL_TREE
);
11460 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11461 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11462 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11465 l
= adjustments
.length ();
11466 for (i
= 1; i
< num_ssa_names
; i
++)
11468 tree name
= ssa_name (i
);
11470 && SSA_NAME_VAR (name
)
11471 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11473 for (j
= 0; j
< l
; j
++)
11474 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11475 && adjustments
[j
].new_decl
)
11478 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11481 = copy_var_decl (adjustments
[j
].base
,
11482 DECL_NAME (adjustments
[j
].base
),
11483 TREE_TYPE (adjustments
[j
].base
));
11484 adjustments
[j
].new_ssa_base
= base_var
;
11487 base_var
= adjustments
[j
].new_ssa_base
;
11488 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11490 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11491 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11492 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11493 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11494 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11495 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11496 gimple stmt
= gimple_build_assign (name
, new_decl
);
11497 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11500 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11505 struct modify_stmt_info info
;
11506 info
.adjustments
= adjustments
;
11508 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11510 gimple_stmt_iterator gsi
;
11512 gsi
= gsi_start_bb (bb
);
11513 while (!gsi_end_p (gsi
))
11515 gimple stmt
= gsi_stmt (gsi
);
11517 struct walk_stmt_info wi
;
11519 memset (&wi
, 0, sizeof (wi
));
11520 info
.modified
= false;
11522 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11524 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11526 tree retval
= gimple_return_retval (stmt
);
11529 gsi_remove (&gsi
, true);
11533 /* Replace `return foo' with `retval_array[iter] = foo'. */
11534 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11535 retval_array
, iter
, NULL
, NULL
);
11536 stmt
= gimple_build_assign (ref
, retval
);
11537 gsi_replace (&gsi
, stmt
, true);
11538 info
.modified
= true;
11543 update_stmt (stmt
);
11544 if (maybe_clean_eh_stmt (stmt
))
11545 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11552 /* Adjust the argument types in NODE to their appropriate vector
11556 simd_clone_adjust (struct cgraph_node
*node
)
11558 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11560 targetm
.simd_clone
.adjust (node
);
11562 tree retval
= simd_clone_adjust_return_type (node
);
11563 ipa_parm_adjustment_vec adjustments
11564 = simd_clone_adjust_argument_types (node
);
11566 push_gimplify_context ();
11568 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11570 /* Adjust all uses of vector arguments accordingly. Adjust all
11571 return values accordingly. */
11572 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11573 tree iter1
= make_ssa_name (iter
, NULL
);
11574 tree iter2
= make_ssa_name (iter
, NULL
);
11575 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11577 /* Initialize the iteration variable. */
11578 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11579 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11580 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11581 /* Insert the SIMD array and iv initialization at function
11583 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11585 pop_gimplify_context (NULL
);
11587 /* Create a new BB right before the original exit BB, to hold the
11588 iteration increment and the condition/branch. */
11589 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11590 basic_block incr_bb
= create_empty_bb (orig_exit
);
11591 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11592 flag. Set it now to be a FALLTHRU_EDGE. */
11593 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11594 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11595 for (unsigned i
= 0;
11596 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
11598 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
11599 redirect_edge_succ (e
, incr_bb
);
11601 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
11602 e
->probability
= REG_BR_PROB_BASE
;
11603 gsi
= gsi_last_bb (incr_bb
);
11604 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
11605 build_int_cst (unsigned_type_node
,
11607 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11609 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11610 struct loop
*loop
= alloc_loop ();
11611 cfun
->has_force_vectorize_loops
= true;
11612 loop
->safelen
= node
->simdclone
->simdlen
;
11613 loop
->force_vectorize
= true;
11614 loop
->header
= body_bb
;
11615 add_bb_to_loop (incr_bb
, loop
);
11617 /* Branch around the body if the mask applies. */
11618 if (node
->simdclone
->inbranch
)
11620 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
11622 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
11623 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
11624 tree aref
= build4 (ARRAY_REF
,
11625 TREE_TYPE (TREE_TYPE (mask_array
)),
11628 g
= gimple_build_assign (mask
, aref
);
11629 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11630 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
11631 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
11633 aref
= build1 (VIEW_CONVERT_EXPR
,
11634 build_nonstandard_integer_type (bitsize
, 0), mask
);
11635 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
11636 g
= gimple_build_assign (mask
, aref
);
11637 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11640 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
11642 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11643 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
11644 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
11647 /* Generate the condition. */
11648 g
= gimple_build_cond (LT_EXPR
,
11650 build_int_cst (unsigned_type_node
,
11651 node
->simdclone
->simdlen
),
11653 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11654 e
= split_block (incr_bb
, gsi_stmt (gsi
));
11655 basic_block latch_bb
= e
->dest
;
11656 basic_block new_exit_bb
= e
->dest
;
11657 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
11658 loop
->latch
= latch_bb
;
11660 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
11662 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
11663 /* The successor of incr_bb is already pointing to latch_bb; just
11665 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11666 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
11668 gimple phi
= create_phi_node (iter1
, body_bb
);
11669 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
11670 edge latch_edge
= single_succ_edge (latch_bb
);
11671 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
11673 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11675 /* Generate the new return. */
11676 gsi
= gsi_last_bb (new_exit_bb
);
11678 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
11679 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
11680 retval
= TREE_OPERAND (retval
, 0);
11683 retval
= build1 (VIEW_CONVERT_EXPR
,
11684 TREE_TYPE (TREE_TYPE (node
->decl
)),
11686 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
11687 false, GSI_CONTINUE_LINKING
);
11689 g
= gimple_build_return (retval
);
11690 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11692 /* Handle aligned clauses by replacing default defs of the aligned
11693 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11694 lhs. Handle linear by adding PHIs. */
11695 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
11696 if (node
->simdclone
->args
[i
].alignment
11697 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
11698 && (node
->simdclone
->args
[i
].alignment
11699 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
11700 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
11703 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
11704 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11705 tree def
= ssa_default_def (cfun
, orig_arg
);
11706 if (def
&& !has_zero_uses (def
))
11708 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
11709 gimple_seq seq
= NULL
;
11710 bool need_cvt
= false;
11712 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
11714 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
11717 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
11718 gimple_call_set_lhs (g
, t
);
11719 gimple_seq_add_stmt_without_update (&seq
, g
);
11722 t
= make_ssa_name (orig_arg
, NULL
);
11723 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
11724 gimple_call_lhs (g
),
11726 gimple_seq_add_stmt_without_update (&seq
, g
);
11728 gsi_insert_seq_on_edge_immediate
11729 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
11731 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11732 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
11734 node
->create_edge (cgraph_node::get_create (fn
),
11735 call
, entry_bb
->count
, freq
);
11737 imm_use_iterator iter
;
11738 use_operand_p use_p
;
11740 tree repl
= gimple_get_lhs (g
);
11741 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11742 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
11745 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11746 SET_USE (use_p
, repl
);
11749 else if (node
->simdclone
->args
[i
].arg_type
11750 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11752 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11753 tree def
= ssa_default_def (cfun
, orig_arg
);
11754 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11755 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
11756 if (def
&& !has_zero_uses (def
))
11758 iter1
= make_ssa_name (orig_arg
, NULL
);
11759 iter2
= make_ssa_name (orig_arg
, NULL
);
11760 phi
= create_phi_node (iter1
, body_bb
);
11761 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
11762 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11763 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11764 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
11765 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11766 ? TREE_TYPE (orig_arg
) : sizetype
;
11768 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
11769 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
11770 gsi
= gsi_last_bb (incr_bb
);
11771 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11773 imm_use_iterator iter
;
11774 use_operand_p use_p
;
11776 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11777 if (use_stmt
== phi
)
11780 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11781 SET_USE (use_p
, iter1
);
11785 calculate_dominance_info (CDI_DOMINATORS
);
11786 add_loop (loop
, loop
->header
->loop_father
);
11787 update_ssa (TODO_update_ssa
);
11792 /* If the function in NODE is tagged as an elemental SIMD function,
11793 create the appropriate SIMD clones. */
11796 expand_simd_clones (struct cgraph_node
*node
)
11798 tree attr
= lookup_attribute ("omp declare simd",
11799 DECL_ATTRIBUTES (node
->decl
));
11800 if (attr
== NULL_TREE
11801 || node
->global
.inlined_to
11802 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
11806 #pragma omp declare simd
11808 in C, there we don't know the argument types at all. */
11809 if (!node
->definition
11810 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
11815 /* Start with parsing the "omp declare simd" attribute(s). */
11816 bool inbranch_clause_specified
;
11817 struct cgraph_simd_clone
*clone_info
11818 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
11819 &inbranch_clause_specified
);
11820 if (clone_info
== NULL
)
11823 int orig_simdlen
= clone_info
->simdlen
;
11824 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
11825 /* The target can return 0 (no simd clones should be created),
11826 1 (just one ISA of simd clones should be created) or higher
11827 count of ISA variants. In that case, clone_info is initialized
11828 for the first ISA variant. */
11830 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
11835 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11836 also create one inbranch and one !inbranch clone of it. */
11837 for (int i
= 0; i
< count
* 2; i
++)
11839 struct cgraph_simd_clone
*clone
= clone_info
;
11840 if (inbranch_clause_specified
&& (i
& 1) != 0)
11845 clone
= simd_clone_struct_alloc (clone_info
->nargs
11847 simd_clone_struct_copy (clone
, clone_info
);
11848 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11849 and simd_clone_adjust_argument_types did to the first
11851 clone
->nargs
-= clone_info
->inbranch
;
11852 clone
->simdlen
= orig_simdlen
;
11853 /* And call the target hook again to get the right ISA. */
11854 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
11858 clone
->inbranch
= 1;
11861 /* simd_clone_mangle might fail if such a clone has been created
11863 tree id
= simd_clone_mangle (node
, clone
);
11864 if (id
== NULL_TREE
)
11867 /* Only when we are sure we want to create the clone actually
11868 clone the function (or definitions) or create another
11869 extern FUNCTION_DECL (for prototypes without definitions). */
11870 struct cgraph_node
*n
= simd_clone_create (node
);
11874 n
->simdclone
= clone
;
11875 clone
->origin
= node
;
11876 clone
->next_clone
= NULL
;
11877 if (node
->simd_clones
== NULL
)
11879 clone
->prev_clone
= n
;
11880 node
->simd_clones
= n
;
11884 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
11885 clone
->prev_clone
->simdclone
->next_clone
= n
;
11886 node
->simd_clones
->simdclone
->prev_clone
= n
;
11888 change_decl_assembler_name (n
->decl
, id
);
11889 /* And finally adjust the return type, parameters and for
11890 definitions also function body. */
11891 if (node
->definition
)
11892 simd_clone_adjust (n
);
11895 simd_clone_adjust_return_type (n
);
11896 simd_clone_adjust_argument_types (n
);
11900 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
11903 /* Entry point for IPA simd clone creation pass. */
11905 static unsigned int
11906 ipa_omp_simd_clone (void)
11908 struct cgraph_node
*node
;
11909 FOR_EACH_FUNCTION (node
)
11910 expand_simd_clones (node
);
11916 const pass_data pass_data_omp_simd_clone
=
11918 SIMPLE_IPA_PASS
, /* type */
11919 "simdclone", /* name */
11920 OPTGROUP_NONE
, /* optinfo_flags */
11921 TV_NONE
, /* tv_id */
11922 ( PROP_ssa
| PROP_cfg
), /* properties_required */
11923 0, /* properties_provided */
11924 0, /* properties_destroyed */
11925 0, /* todo_flags_start */
11926 0, /* todo_flags_finish */
11929 class pass_omp_simd_clone
: public simple_ipa_opt_pass
11932 pass_omp_simd_clone(gcc::context
*ctxt
)
11933 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
11936 /* opt_pass methods: */
11937 virtual bool gate (function
*);
11938 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
11942 pass_omp_simd_clone::gate (function
*)
11944 return ((flag_openmp
|| flag_openmp_simd
11946 || (in_lto_p
&& !flag_wpa
))
11947 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
11950 } // anon namespace
11952 simple_ipa_opt_pass
*
11953 make_pass_omp_simd_clone (gcc::context
*ctxt
)
11955 return new pass_omp_simd_clone (ctxt
);
11958 #include "gt-omp-low.h"