1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
31 #include "tree-iterator.h"
32 #include "tree-inline.h"
33 #include "langhooks.h"
34 #include "diagnostic-core.h"
39 #include "tree-pass.h"
42 #include "splay-tree.h"
47 #include "gimple-low.h"
48 #include "tree-cfgcleanup.h"
51 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
52 phases. The first phase scans the function looking for OMP statements
53 and then for variables that must be replaced to satisfy data sharing
54 clauses. The second phase expands code for the constructs, as well as
55 re-gimplifying things when variables have been replaced with complex
58 Final code generation is done by pass_expand_omp. The flowgraph is
59 scanned for parallel regions which are then moved to a new
60 function, to be invoked by the thread library. */
62 /* Parallel region information. Every parallel and workshare
63 directive is enclosed between two markers, the OMP_* directive
64 and a corresponding OMP_RETURN statement. */
68 /* The enclosing region. */
69 struct omp_region
*outer
;
71 /* First child region. */
72 struct omp_region
*inner
;
74 /* Next peer region. */
75 struct omp_region
*next
;
77 /* Block containing the omp directive as its last stmt. */
80 /* Block containing the OMP_RETURN as its last stmt. */
83 /* Block containing the OMP_CONTINUE as its last stmt. */
86 /* If this is a combined parallel+workshare region, this is a list
87 of additional arguments needed by the combined parallel+workshare
89 vec
<tree
, va_gc
> *ws_args
;
91 /* The code for the omp directive of this region. */
92 enum gimple_code type
;
94 /* Schedule kind, only used for OMP_FOR type regions. */
95 enum omp_clause_schedule_kind sched_kind
;
97 /* True if this is a combined parallel+workshare region. */
98 bool is_combined_parallel
;
101 /* Context structure. Used to store information about each parallel
102 directive in the code. */
104 typedef struct omp_context
106 /* This field must be at the beginning, as we do "inheritance": Some
107 callback functions for tree-inline.c (e.g., omp_copy_decl)
108 receive a copy_body_data pointer that is up-casted to an
109 omp_context pointer. */
112 /* The tree of contexts corresponding to the encountered constructs. */
113 struct omp_context
*outer
;
116 /* Map variables to fields in a structure that allows communication
117 between sending and receiving threads. */
118 splay_tree field_map
;
123 /* These are used just by task contexts, if task firstprivate fn is
124 needed. srecord_type is used to communicate from the thread
125 that encountered the task construct to task firstprivate fn,
126 record_type is allocated by GOMP_task, initialized by task firstprivate
127 fn and passed to the task body fn. */
128 splay_tree sfield_map
;
131 /* A chain of variables to add to the top-level block surrounding the
132 construct. In the case of a parallel, this is in the child function. */
135 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
136 barriers should jump to during omplower pass. */
139 /* What to do with variables with implicitly determined sharing
141 enum omp_clause_default_kind default_kind
;
143 /* Nesting depth of this context. Used to beautify error messages re
144 invalid gotos. The outermost ctx is depth 1, with depth 0 being
145 reserved for the main body of the function. */
148 /* True if this parallel directive is nested within another. */
151 /* True if this construct can be cancelled. */
156 struct omp_for_data_loop
158 tree v
, n1
, n2
, step
;
159 enum tree_code cond_code
;
162 /* A structure describing the main elements of a parallel loop. */
166 struct omp_for_data_loop loop
;
171 bool have_nowait
, have_ordered
;
172 enum omp_clause_schedule_kind sched_kind
;
173 struct omp_for_data_loop
*loops
;
177 static splay_tree all_contexts
;
178 static int taskreg_nesting_level
;
179 static int target_nesting_level
;
180 static struct omp_region
*root_omp_region
;
181 static bitmap task_shared_vars
;
183 static void scan_omp (gimple_seq
*, omp_context
*);
184 static tree
scan_omp_1_op (tree
*, int *, void *);
186 #define WALK_SUBSTMTS \
190 case GIMPLE_EH_FILTER: \
191 case GIMPLE_TRANSACTION: \
192 /* The sub-statements for these should be walked. */ \
193 *handled_ops_p = false; \
196 /* Convenience function for calling scan_omp_1_op on tree operands. */
199 scan_omp_op (tree
*tp
, omp_context
*ctx
)
201 struct walk_stmt_info wi
;
203 memset (&wi
, 0, sizeof (wi
));
205 wi
.want_locations
= true;
207 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
210 static void lower_omp (gimple_seq
*, omp_context
*);
211 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
212 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
214 /* Find an OpenMP clause of type KIND within CLAUSES. */
217 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
219 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
220 if (OMP_CLAUSE_CODE (clauses
) == kind
)
226 /* Return true if CTX is for an omp parallel. */
229 is_parallel_ctx (omp_context
*ctx
)
231 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
235 /* Return true if CTX is for an omp task. */
238 is_task_ctx (omp_context
*ctx
)
240 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
244 /* Return true if CTX is for an omp parallel or omp task. */
247 is_taskreg_ctx (omp_context
*ctx
)
249 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
250 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
254 /* Return true if REGION is a combined parallel+workshare region. */
257 is_combined_parallel (struct omp_region
*region
)
259 return region
->is_combined_parallel
;
263 /* Extract the header elements of parallel loop FOR_STMT and store
267 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
268 struct omp_for_data_loop
*loops
)
270 tree t
, var
, *collapse_iter
, *collapse_count
;
271 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
272 struct omp_for_data_loop
*loop
;
274 struct omp_for_data_loop dummy_loop
;
275 location_t loc
= gimple_location (for_stmt
);
276 bool simd
= gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_SIMD
;
277 bool distribute
= gimple_omp_for_kind (for_stmt
)
278 == GF_OMP_FOR_KIND_DISTRIBUTE
;
280 fd
->for_stmt
= for_stmt
;
282 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
283 if (fd
->collapse
> 1)
286 fd
->loops
= &fd
->loop
;
288 fd
->have_nowait
= distribute
|| simd
;
289 fd
->have_ordered
= false;
290 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
291 fd
->chunk_size
= NULL_TREE
;
292 collapse_iter
= NULL
;
293 collapse_count
= NULL
;
295 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
296 switch (OMP_CLAUSE_CODE (t
))
298 case OMP_CLAUSE_NOWAIT
:
299 fd
->have_nowait
= true;
301 case OMP_CLAUSE_ORDERED
:
302 fd
->have_ordered
= true;
304 case OMP_CLAUSE_SCHEDULE
:
305 gcc_assert (!distribute
);
306 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
307 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
309 case OMP_CLAUSE_DIST_SCHEDULE
:
310 gcc_assert (distribute
);
311 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
313 case OMP_CLAUSE_COLLAPSE
:
314 if (fd
->collapse
> 1)
316 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
317 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
323 /* FIXME: for now map schedule(auto) to schedule(static).
324 There should be analysis to determine whether all iterations
325 are approximately the same amount of work (then schedule(static)
326 is best) or if it varies (then schedule(dynamic,N) is better). */
327 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
329 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
330 gcc_assert (fd
->chunk_size
== NULL
);
332 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
333 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
334 gcc_assert (fd
->chunk_size
== NULL
);
335 else if (fd
->chunk_size
== NULL
)
337 /* We only need to compute a default chunk size for ordered
338 static loops and dynamic loops. */
339 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
341 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
342 ? integer_zero_node
: integer_one_node
;
345 for (i
= 0; i
< fd
->collapse
; i
++)
347 if (fd
->collapse
== 1)
349 else if (loops
!= NULL
)
354 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
355 gcc_assert (SSA_VAR_P (loop
->v
));
356 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
357 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
358 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
359 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
361 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
362 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
363 switch (loop
->cond_code
)
369 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
370 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
372 loop
->n2
= fold_build2_loc (loc
,
373 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
374 build_int_cst (TREE_TYPE (loop
->n2
), 1));
375 loop
->cond_code
= LT_EXPR
;
378 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
379 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
381 loop
->n2
= fold_build2_loc (loc
,
382 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
383 build_int_cst (TREE_TYPE (loop
->n2
), 1));
384 loop
->cond_code
= GT_EXPR
;
390 t
= gimple_omp_for_incr (for_stmt
, i
);
391 gcc_assert (TREE_OPERAND (t
, 0) == var
);
392 switch (TREE_CODE (t
))
395 loop
->step
= TREE_OPERAND (t
, 1);
397 case POINTER_PLUS_EXPR
:
398 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
401 loop
->step
= TREE_OPERAND (t
, 1);
402 loop
->step
= fold_build1_loc (loc
,
403 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
411 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
412 && !fd
->have_ordered
))
414 if (fd
->collapse
== 1)
415 iter_type
= TREE_TYPE (loop
->v
);
417 || TYPE_PRECISION (iter_type
)
418 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
420 = build_nonstandard_integer_type
421 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
423 else if (iter_type
!= long_long_unsigned_type_node
)
425 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
426 iter_type
= long_long_unsigned_type_node
;
427 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
428 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
429 >= TYPE_PRECISION (iter_type
))
433 if (loop
->cond_code
== LT_EXPR
)
434 n
= fold_build2_loc (loc
,
435 PLUS_EXPR
, TREE_TYPE (loop
->v
),
436 loop
->n2
, loop
->step
);
439 if (TREE_CODE (n
) != INTEGER_CST
440 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
441 iter_type
= long_long_unsigned_type_node
;
443 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
444 > TYPE_PRECISION (iter_type
))
448 if (loop
->cond_code
== LT_EXPR
)
451 n2
= fold_build2_loc (loc
,
452 PLUS_EXPR
, TREE_TYPE (loop
->v
),
453 loop
->n2
, loop
->step
);
457 n1
= fold_build2_loc (loc
,
458 MINUS_EXPR
, TREE_TYPE (loop
->v
),
459 loop
->n2
, loop
->step
);
462 if (TREE_CODE (n1
) != INTEGER_CST
463 || TREE_CODE (n2
) != INTEGER_CST
464 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
465 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
466 iter_type
= long_long_unsigned_type_node
;
470 if (collapse_count
&& *collapse_count
== NULL
)
472 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
473 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
474 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
475 if (t
&& integer_zerop (t
))
476 count
= build_zero_cst (long_long_unsigned_type_node
);
477 else if ((i
== 0 || count
!= NULL_TREE
)
478 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
479 && TREE_CONSTANT (loop
->n1
)
480 && TREE_CONSTANT (loop
->n2
)
481 && TREE_CODE (loop
->step
) == INTEGER_CST
)
483 tree itype
= TREE_TYPE (loop
->v
);
485 if (POINTER_TYPE_P (itype
))
486 itype
= signed_type_for (itype
);
487 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
488 t
= fold_build2_loc (loc
,
490 fold_convert_loc (loc
, itype
, loop
->step
), t
);
491 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
492 fold_convert_loc (loc
, itype
, loop
->n2
));
493 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
494 fold_convert_loc (loc
, itype
, loop
->n1
));
495 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
496 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
497 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
498 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
499 fold_convert_loc (loc
, itype
,
502 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
503 fold_convert_loc (loc
, itype
, loop
->step
));
504 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
505 if (count
!= NULL_TREE
)
506 count
= fold_build2_loc (loc
,
507 MULT_EXPR
, long_long_unsigned_type_node
,
511 if (TREE_CODE (count
) != INTEGER_CST
)
514 else if (count
&& !integer_zerop (count
))
521 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
522 || fd
->have_ordered
))
524 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
525 iter_type
= long_long_unsigned_type_node
;
527 iter_type
= long_integer_type_node
;
529 else if (collapse_iter
&& *collapse_iter
!= NULL
)
530 iter_type
= TREE_TYPE (*collapse_iter
);
531 fd
->iter_type
= iter_type
;
532 if (collapse_iter
&& *collapse_iter
== NULL
)
533 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
534 if (collapse_count
&& *collapse_count
== NULL
)
537 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
539 *collapse_count
= create_tmp_var (iter_type
, ".count");
542 if (fd
->collapse
> 1)
544 fd
->loop
.v
= *collapse_iter
;
545 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
546 fd
->loop
.n2
= *collapse_count
;
547 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
548 fd
->loop
.cond_code
= LT_EXPR
;
553 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
554 is the immediate dominator of PAR_ENTRY_BB, return true if there
555 are no data dependencies that would prevent expanding the parallel
556 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
558 When expanding a combined parallel+workshare region, the call to
559 the child function may need additional arguments in the case of
560 GIMPLE_OMP_FOR regions. In some cases, these arguments are
561 computed out of variables passed in from the parent to the child
562 via 'struct .omp_data_s'. For instance:
564 #pragma omp parallel for schedule (guided, i * 4)
569 # BLOCK 2 (PAR_ENTRY_BB)
571 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
573 # BLOCK 3 (WS_ENTRY_BB)
574 .omp_data_i = &.omp_data_o;
575 D.1667 = .omp_data_i->i;
577 #pragma omp for schedule (guided, D.1598)
579 When we outline the parallel region, the call to the child function
580 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
581 that value is computed *after* the call site. So, in principle we
582 cannot do the transformation.
584 To see whether the code in WS_ENTRY_BB blocks the combined
585 parallel+workshare call, we collect all the variables used in the
586 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
587 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
590 FIXME. If we had the SSA form built at this point, we could merely
591 hoist the code in block 3 into block 2 and be done with it. But at
592 this point we don't have dataflow information and though we could
593 hack something up here, it is really not worth the aggravation. */
596 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
598 struct omp_for_data fd
;
599 gimple ws_stmt
= last_stmt (ws_entry_bb
);
601 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
604 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
606 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
608 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
610 if (fd
.iter_type
!= long_integer_type_node
)
613 /* FIXME. We give up too easily here. If any of these arguments
614 are not constants, they will likely involve variables that have
615 been mapped into fields of .omp_data_s for sharing with the child
616 function. With appropriate data flow, it would be possible to
618 if (!is_gimple_min_invariant (fd
.loop
.n1
)
619 || !is_gimple_min_invariant (fd
.loop
.n2
)
620 || !is_gimple_min_invariant (fd
.loop
.step
)
621 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
628 /* Collect additional arguments needed to emit a combined
629 parallel+workshare call. WS_STMT is the workshare directive being
632 static vec
<tree
, va_gc
> *
633 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
636 location_t loc
= gimple_location (ws_stmt
);
637 vec
<tree
, va_gc
> *ws_args
;
639 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
641 struct omp_for_data fd
;
644 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
648 if (gimple_omp_for_combined_into_p (ws_stmt
))
651 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
652 OMP_CLAUSE__LOOPTEMP_
);
654 n1
= OMP_CLAUSE_DECL (innerc
);
655 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
656 OMP_CLAUSE__LOOPTEMP_
);
658 n2
= OMP_CLAUSE_DECL (innerc
);
661 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
663 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
664 ws_args
->quick_push (t
);
666 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
667 ws_args
->quick_push (t
);
669 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
670 ws_args
->quick_push (t
);
674 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
675 ws_args
->quick_push (t
);
680 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
682 /* Number of sections is equal to the number of edges from the
683 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
684 the exit of the sections region. */
685 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
686 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
687 vec_alloc (ws_args
, 1);
688 ws_args
->quick_push (t
);
696 /* Discover whether REGION is a combined parallel+workshare region. */
699 determine_parallel_type (struct omp_region
*region
)
701 basic_block par_entry_bb
, par_exit_bb
;
702 basic_block ws_entry_bb
, ws_exit_bb
;
704 if (region
== NULL
|| region
->inner
== NULL
705 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
706 || region
->inner
->cont
== NULL
)
709 /* We only support parallel+for and parallel+sections. */
710 if (region
->type
!= GIMPLE_OMP_PARALLEL
711 || (region
->inner
->type
!= GIMPLE_OMP_FOR
712 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
715 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
716 WS_EXIT_BB -> PAR_EXIT_BB. */
717 par_entry_bb
= region
->entry
;
718 par_exit_bb
= region
->exit
;
719 ws_entry_bb
= region
->inner
->entry
;
720 ws_exit_bb
= region
->inner
->exit
;
722 if (single_succ (par_entry_bb
) == ws_entry_bb
723 && single_succ (ws_exit_bb
) == par_exit_bb
724 && workshare_safe_to_combine_p (ws_entry_bb
)
725 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
726 || (last_and_only_stmt (ws_entry_bb
)
727 && last_and_only_stmt (par_exit_bb
))))
729 gimple par_stmt
= last_stmt (par_entry_bb
);
730 gimple ws_stmt
= last_stmt (ws_entry_bb
);
732 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
734 /* If this is a combined parallel loop, we need to determine
735 whether or not to use the combined library calls. There
736 are two cases where we do not apply the transformation:
737 static loops and any kind of ordered loop. In the first
738 case, we already open code the loop so there is no need
739 to do anything else. In the latter case, the combined
740 parallel loop call would still need extra synchronization
741 to implement ordered semantics, so there would not be any
742 gain in using the combined call. */
743 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
744 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
746 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
747 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
749 region
->is_combined_parallel
= false;
750 region
->inner
->is_combined_parallel
= false;
755 region
->is_combined_parallel
= true;
756 region
->inner
->is_combined_parallel
= true;
757 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
762 /* Return true if EXPR is variable sized. */
765 is_variable_sized (const_tree expr
)
767 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
770 /* Return true if DECL is a reference type. */
773 is_reference (tree decl
)
775 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
778 /* Lookup variables in the decl or field splay trees. The "maybe" form
779 allows for the variable form to not have been entered, otherwise we
780 assert that the variable must have been entered. */
783 lookup_decl (tree var
, omp_context
*ctx
)
786 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
791 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
794 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
795 return n
? *n
: NULL_TREE
;
799 lookup_field (tree var
, omp_context
*ctx
)
802 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
803 return (tree
) n
->value
;
807 lookup_sfield (tree var
, omp_context
*ctx
)
810 n
= splay_tree_lookup (ctx
->sfield_map
811 ? ctx
->sfield_map
: ctx
->field_map
,
812 (splay_tree_key
) var
);
813 return (tree
) n
->value
;
817 maybe_lookup_field (tree var
, omp_context
*ctx
)
820 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
821 return n
? (tree
) n
->value
: NULL_TREE
;
824 /* Return true if DECL should be copied by pointer. SHARED_CTX is
825 the parallel context if DECL is to be shared. */
828 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
830 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
833 /* We can only use copy-in/copy-out semantics for shared variables
834 when we know the value is not accessible from an outer scope. */
837 /* ??? Trivially accessible from anywhere. But why would we even
838 be passing an address in this case? Should we simply assert
839 this to be false, or should we have a cleanup pass that removes
840 these from the list of mappings? */
841 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
844 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
845 without analyzing the expression whether or not its location
846 is accessible to anyone else. In the case of nested parallel
847 regions it certainly may be. */
848 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
851 /* Do not use copy-in/copy-out for variables that have their
853 if (TREE_ADDRESSABLE (decl
))
856 /* lower_send_shared_vars only uses copy-in, but not copy-out
858 if (TREE_READONLY (decl
)
859 || ((TREE_CODE (decl
) == RESULT_DECL
860 || TREE_CODE (decl
) == PARM_DECL
)
861 && DECL_BY_REFERENCE (decl
)))
864 /* Disallow copy-in/out in nested parallel if
865 decl is shared in outer parallel, otherwise
866 each thread could store the shared variable
867 in its own copy-in location, making the
868 variable no longer really shared. */
869 if (shared_ctx
->is_nested
)
873 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
874 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
881 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
882 c
; c
= OMP_CLAUSE_CHAIN (c
))
883 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
884 && OMP_CLAUSE_DECL (c
) == decl
)
888 goto maybe_mark_addressable_and_ret
;
892 /* For tasks avoid using copy-in/out. As tasks can be
893 deferred or executed in different thread, when GOMP_task
894 returns, the task hasn't necessarily terminated. */
895 if (is_task_ctx (shared_ctx
))
898 maybe_mark_addressable_and_ret
:
899 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
900 if (is_gimple_reg (outer
))
902 /* Taking address of OUTER in lower_send_shared_vars
903 might need regimplification of everything that uses the
905 if (!task_shared_vars
)
906 task_shared_vars
= BITMAP_ALLOC (NULL
);
907 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
908 TREE_ADDRESSABLE (outer
) = 1;
917 /* Construct a new automatic decl similar to VAR. */
920 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
922 tree copy
= copy_var_decl (var
, name
, type
);
924 DECL_CONTEXT (copy
) = current_function_decl
;
925 DECL_CHAIN (copy
) = ctx
->block_vars
;
926 ctx
->block_vars
= copy
;
932 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
934 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
937 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
940 omp_build_component_ref (tree obj
, tree field
)
942 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
943 if (TREE_THIS_VOLATILE (field
))
944 TREE_THIS_VOLATILE (ret
) |= 1;
945 if (TREE_READONLY (field
))
946 TREE_READONLY (ret
) |= 1;
950 /* Build tree nodes to access the field for VAR on the receiver side. */
953 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
955 tree x
, field
= lookup_field (var
, ctx
);
957 /* If the receiver record type was remapped in the child function,
958 remap the field into the new record type. */
959 x
= maybe_lookup_field (field
, ctx
);
963 x
= build_simple_mem_ref (ctx
->receiver_decl
);
964 x
= omp_build_component_ref (x
, field
);
966 x
= build_simple_mem_ref (x
);
971 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
972 of a parallel, this is a component reference; for workshare constructs
973 this is some variable. */
976 build_outer_var_ref (tree var
, omp_context
*ctx
)
980 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
982 else if (is_variable_sized (var
))
984 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
985 x
= build_outer_var_ref (x
, ctx
);
986 x
= build_simple_mem_ref (x
);
988 else if (is_taskreg_ctx (ctx
))
990 bool by_ref
= use_pointer_for_field (var
, NULL
);
991 x
= build_receiver_ref (var
, by_ref
, ctx
);
993 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
994 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
996 /* #pragma omp simd isn't a worksharing construct, and can reference even
997 private vars in its linear etc. clauses. */
999 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1000 x
= lookup_decl (var
, ctx
->outer
);
1001 else if (ctx
->outer
)
1002 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1006 else if (ctx
->outer
)
1007 x
= lookup_decl (var
, ctx
->outer
);
1008 else if (is_reference (var
))
1009 /* This can happen with orphaned constructs. If var is reference, it is
1010 possible it is shared and as such valid. */
1015 if (is_reference (var
))
1016 x
= build_simple_mem_ref (x
);
1021 /* Build tree nodes to access the field for VAR on the sender side. */
1024 build_sender_ref (tree var
, omp_context
*ctx
)
1026 tree field
= lookup_sfield (var
, ctx
);
1027 return omp_build_component_ref (ctx
->sender_decl
, field
);
1030 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1033 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1035 tree field
, type
, sfield
= NULL_TREE
;
1037 gcc_assert ((mask
& 1) == 0
1038 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1039 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1040 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1042 type
= TREE_TYPE (var
);
1045 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1046 type
= build_pointer_type (build_pointer_type (type
));
1049 type
= build_pointer_type (type
);
1050 else if ((mask
& 3) == 1 && is_reference (var
))
1051 type
= TREE_TYPE (type
);
1053 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1054 FIELD_DECL
, DECL_NAME (var
), type
);
1056 /* Remember what variable this field was created for. This does have a
1057 side effect of making dwarf2out ignore this member, so for helpful
1058 debugging we clear it later in delete_omp_context. */
1059 DECL_ABSTRACT_ORIGIN (field
) = var
;
1060 if (type
== TREE_TYPE (var
))
1062 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1063 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1064 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1067 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1069 if ((mask
& 3) == 3)
1071 insert_field_into_struct (ctx
->record_type
, field
);
1072 if (ctx
->srecord_type
)
1074 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1075 FIELD_DECL
, DECL_NAME (var
), type
);
1076 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1077 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1078 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1079 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1080 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1085 if (ctx
->srecord_type
== NULL_TREE
)
1089 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1090 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1091 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1093 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1094 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1095 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1096 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1097 splay_tree_insert (ctx
->sfield_map
,
1098 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1099 (splay_tree_value
) sfield
);
1103 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1104 : ctx
->srecord_type
, field
);
1108 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1109 (splay_tree_value
) field
);
1110 if ((mask
& 2) && ctx
->sfield_map
)
1111 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1112 (splay_tree_value
) sfield
);
1116 install_var_local (tree var
, omp_context
*ctx
)
1118 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1119 insert_decl_map (&ctx
->cb
, var
, new_var
);
1123 /* Adjust the replacement for DECL in CTX for the new context. This means
1124 copying the DECL_VALUE_EXPR, and fixing up the type. */
1127 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1129 tree new_decl
, size
;
1131 new_decl
= lookup_decl (decl
, ctx
);
1133 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1135 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1136 && DECL_HAS_VALUE_EXPR_P (decl
))
1138 tree ve
= DECL_VALUE_EXPR (decl
);
1139 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1140 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1141 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1144 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1146 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1147 if (size
== error_mark_node
)
1148 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1149 DECL_SIZE (new_decl
) = size
;
1151 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1152 if (size
== error_mark_node
)
1153 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1154 DECL_SIZE_UNIT (new_decl
) = size
;
1158 /* The callback for remap_decl. Search all containing contexts for a
1159 mapping of the variable; this avoids having to duplicate the splay
1160 tree ahead of time. We know a mapping doesn't already exist in the
1161 given context. Create new mappings to implement default semantics. */
1164 omp_copy_decl (tree var
, copy_body_data
*cb
)
1166 omp_context
*ctx
= (omp_context
*) cb
;
1169 if (TREE_CODE (var
) == LABEL_DECL
)
1171 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1172 DECL_CONTEXT (new_var
) = current_function_decl
;
1173 insert_decl_map (&ctx
->cb
, var
, new_var
);
1177 while (!is_taskreg_ctx (ctx
))
1182 new_var
= maybe_lookup_decl (var
, ctx
);
1187 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1190 return error_mark_node
;
1194 /* Return the parallel region associated with STMT. */
1196 /* Debugging dumps for parallel regions. */
1197 void dump_omp_region (FILE *, struct omp_region
*, int);
1198 void debug_omp_region (struct omp_region
*);
1199 void debug_all_omp_regions (void);
1201 /* Dump the parallel region tree rooted at REGION. */
1204 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1206 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1207 gimple_code_name
[region
->type
]);
1210 dump_omp_region (file
, region
->inner
, indent
+ 4);
1214 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1215 region
->cont
->index
);
1219 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1220 region
->exit
->index
);
1222 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1225 dump_omp_region (file
, region
->next
, indent
);
1229 debug_omp_region (struct omp_region
*region
)
1231 dump_omp_region (stderr
, region
, 0);
1235 debug_all_omp_regions (void)
1237 dump_omp_region (stderr
, root_omp_region
, 0);
1241 /* Create a new parallel region starting at STMT inside region PARENT. */
1243 static struct omp_region
*
1244 new_omp_region (basic_block bb
, enum gimple_code type
,
1245 struct omp_region
*parent
)
1247 struct omp_region
*region
= XCNEW (struct omp_region
);
1249 region
->outer
= parent
;
1251 region
->type
= type
;
1255 /* This is a nested region. Add it to the list of inner
1256 regions in PARENT. */
1257 region
->next
= parent
->inner
;
1258 parent
->inner
= region
;
1262 /* This is a toplevel region. Add it to the list of toplevel
1263 regions in ROOT_OMP_REGION. */
1264 region
->next
= root_omp_region
;
1265 root_omp_region
= region
;
1271 /* Release the memory associated with the region tree rooted at REGION. */
1274 free_omp_region_1 (struct omp_region
*region
)
1276 struct omp_region
*i
, *n
;
1278 for (i
= region
->inner
; i
; i
= n
)
1281 free_omp_region_1 (i
);
1287 /* Release the memory for the entire omp region tree. */
1290 free_omp_regions (void)
1292 struct omp_region
*r
, *n
;
1293 for (r
= root_omp_region
; r
; r
= n
)
1296 free_omp_region_1 (r
);
1298 root_omp_region
= NULL
;
1302 /* Create a new context, with OUTER_CTX being the surrounding context. */
1304 static omp_context
*
1305 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1307 omp_context
*ctx
= XCNEW (omp_context
);
1309 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1310 (splay_tree_value
) ctx
);
1315 ctx
->outer
= outer_ctx
;
1316 ctx
->cb
= outer_ctx
->cb
;
1317 ctx
->cb
.block
= NULL
;
1318 ctx
->depth
= outer_ctx
->depth
+ 1;
1322 ctx
->cb
.src_fn
= current_function_decl
;
1323 ctx
->cb
.dst_fn
= current_function_decl
;
1324 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1325 gcc_checking_assert (ctx
->cb
.src_node
);
1326 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1327 ctx
->cb
.src_cfun
= cfun
;
1328 ctx
->cb
.copy_decl
= omp_copy_decl
;
1329 ctx
->cb
.eh_lp_nr
= 0;
1330 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1334 ctx
->cb
.decl_map
= pointer_map_create ();
1339 static gimple_seq
maybe_catch_exception (gimple_seq
);
1341 /* Finalize task copyfn. */
1344 finalize_task_copyfn (gimple task_stmt
)
1346 struct function
*child_cfun
;
1348 gimple_seq seq
= NULL
, new_seq
;
1351 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1352 if (child_fn
== NULL_TREE
)
1355 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1356 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1358 push_cfun (child_cfun
);
1359 bind
= gimplify_body (child_fn
, false);
1360 gimple_seq_add_stmt (&seq
, bind
);
1361 new_seq
= maybe_catch_exception (seq
);
1364 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1366 gimple_seq_add_stmt (&seq
, bind
);
1368 gimple_set_body (child_fn
, seq
);
1371 /* Inform the callgraph about the new function. */
1372 cgraph_add_new_function (child_fn
, false);
1375 /* Destroy a omp_context data structures. Called through the splay tree
1376 value delete callback. */
1379 delete_omp_context (splay_tree_value value
)
1381 omp_context
*ctx
= (omp_context
*) value
;
1383 pointer_map_destroy (ctx
->cb
.decl_map
);
1386 splay_tree_delete (ctx
->field_map
);
1387 if (ctx
->sfield_map
)
1388 splay_tree_delete (ctx
->sfield_map
);
1390 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1391 it produces corrupt debug information. */
1392 if (ctx
->record_type
)
1395 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1396 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1398 if (ctx
->srecord_type
)
1401 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1402 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1405 if (is_task_ctx (ctx
))
1406 finalize_task_copyfn (ctx
->stmt
);
1411 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1415 fixup_child_record_type (omp_context
*ctx
)
1417 tree f
, type
= ctx
->record_type
;
1419 /* ??? It isn't sufficient to just call remap_type here, because
1420 variably_modified_type_p doesn't work the way we expect for
1421 record types. Testing each field for whether it needs remapping
1422 and creating a new record by hand works, however. */
1423 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1424 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1428 tree name
, new_fields
= NULL
;
1430 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1431 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1432 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1433 TYPE_DECL
, name
, type
);
1434 TYPE_NAME (type
) = name
;
1436 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1438 tree new_f
= copy_node (f
);
1439 DECL_CONTEXT (new_f
) = type
;
1440 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1441 DECL_CHAIN (new_f
) = new_fields
;
1442 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1443 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1445 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1449 /* Arrange to be able to look up the receiver field
1450 given the sender field. */
1451 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1452 (splay_tree_value
) new_f
);
1454 TYPE_FIELDS (type
) = nreverse (new_fields
);
1458 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1461 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1462 specified by CLAUSES. */
1465 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1468 bool scan_array_reductions
= false;
1470 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1474 switch (OMP_CLAUSE_CODE (c
))
1476 case OMP_CLAUSE_PRIVATE
:
1477 decl
= OMP_CLAUSE_DECL (c
);
1478 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1480 else if (!is_variable_sized (decl
))
1481 install_var_local (decl
, ctx
);
1484 case OMP_CLAUSE_SHARED
:
1485 /* Ignore shared directives in teams construct. */
1486 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1488 gcc_assert (is_taskreg_ctx (ctx
));
1489 decl
= OMP_CLAUSE_DECL (c
);
1490 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1491 || !is_variable_sized (decl
));
1492 /* Global variables don't need to be copied,
1493 the receiver side will use them directly. */
1494 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1496 by_ref
= use_pointer_for_field (decl
, ctx
);
1497 if (! TREE_READONLY (decl
)
1498 || TREE_ADDRESSABLE (decl
)
1500 || is_reference (decl
))
1502 install_var_field (decl
, by_ref
, 3, ctx
);
1503 install_var_local (decl
, ctx
);
1506 /* We don't need to copy const scalar vars back. */
1507 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1510 case OMP_CLAUSE_LASTPRIVATE
:
1511 /* Let the corresponding firstprivate clause create
1513 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1517 case OMP_CLAUSE_FIRSTPRIVATE
:
1518 case OMP_CLAUSE_REDUCTION
:
1519 case OMP_CLAUSE_LINEAR
:
1520 decl
= OMP_CLAUSE_DECL (c
);
1522 if (is_variable_sized (decl
))
1524 if (is_task_ctx (ctx
))
1525 install_var_field (decl
, false, 1, ctx
);
1528 else if (is_taskreg_ctx (ctx
))
1531 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1532 by_ref
= use_pointer_for_field (decl
, NULL
);
1534 if (is_task_ctx (ctx
)
1535 && (global
|| by_ref
|| is_reference (decl
)))
1537 install_var_field (decl
, false, 1, ctx
);
1539 install_var_field (decl
, by_ref
, 2, ctx
);
1542 install_var_field (decl
, by_ref
, 3, ctx
);
1544 install_var_local (decl
, ctx
);
1547 case OMP_CLAUSE__LOOPTEMP_
:
1548 gcc_assert (is_parallel_ctx (ctx
));
1549 decl
= OMP_CLAUSE_DECL (c
);
1550 install_var_field (decl
, false, 3, ctx
);
1551 install_var_local (decl
, ctx
);
1554 case OMP_CLAUSE_COPYPRIVATE
:
1555 case OMP_CLAUSE_COPYIN
:
1556 decl
= OMP_CLAUSE_DECL (c
);
1557 by_ref
= use_pointer_for_field (decl
, NULL
);
1558 install_var_field (decl
, by_ref
, 3, ctx
);
1561 case OMP_CLAUSE_DEFAULT
:
1562 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1565 case OMP_CLAUSE_FINAL
:
1567 case OMP_CLAUSE_NUM_THREADS
:
1568 case OMP_CLAUSE_NUM_TEAMS
:
1569 case OMP_CLAUSE_THREAD_LIMIT
:
1570 case OMP_CLAUSE_DEVICE
:
1571 case OMP_CLAUSE_SCHEDULE
:
1572 case OMP_CLAUSE_DIST_SCHEDULE
:
1573 case OMP_CLAUSE_DEPEND
:
1575 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1579 case OMP_CLAUSE_FROM
:
1580 case OMP_CLAUSE_MAP
:
1582 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1583 decl
= OMP_CLAUSE_DECL (c
);
1584 /* Global variables with "omp declare target" attribute
1585 don't need to be copied, the receiver side will use them
1587 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1589 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1590 && lookup_attribute ("omp declare target",
1591 DECL_ATTRIBUTES (decl
)))
1593 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1594 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1596 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1597 #pragma omp target data, there is nothing to map for
1599 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1600 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1605 if (DECL_SIZE (decl
)
1606 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1608 tree decl2
= DECL_VALUE_EXPR (decl
);
1609 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1610 decl2
= TREE_OPERAND (decl2
, 0);
1611 gcc_assert (DECL_P (decl2
));
1612 install_var_field (decl2
, true, 3, ctx
);
1613 install_var_local (decl2
, ctx
);
1614 install_var_local (decl
, ctx
);
1618 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1619 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1620 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1621 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1622 install_var_field (decl
, true, 7, ctx
);
1624 install_var_field (decl
, true, 3, ctx
);
1625 if (gimple_omp_target_kind (ctx
->stmt
)
1626 == GF_OMP_TARGET_KIND_REGION
)
1627 install_var_local (decl
, ctx
);
1632 tree base
= get_base_address (decl
);
1633 tree nc
= OMP_CLAUSE_CHAIN (c
);
1636 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1637 && OMP_CLAUSE_DECL (nc
) == base
1638 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1639 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1641 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1642 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1646 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1647 (splay_tree_key
) decl
));
1649 = build_decl (OMP_CLAUSE_LOCATION (c
),
1650 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1651 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1652 insert_field_into_struct (ctx
->record_type
, field
);
1653 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1654 (splay_tree_value
) field
);
1659 case OMP_CLAUSE_NOWAIT
:
1660 case OMP_CLAUSE_ORDERED
:
1661 case OMP_CLAUSE_COLLAPSE
:
1662 case OMP_CLAUSE_UNTIED
:
1663 case OMP_CLAUSE_MERGEABLE
:
1664 case OMP_CLAUSE_PROC_BIND
:
1665 case OMP_CLAUSE_SAFELEN
:
1668 case OMP_CLAUSE_ALIGNED
:
1669 decl
= OMP_CLAUSE_DECL (c
);
1670 if (is_global_var (decl
)
1671 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1672 install_var_local (decl
, ctx
);
1680 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1682 switch (OMP_CLAUSE_CODE (c
))
1684 case OMP_CLAUSE_LASTPRIVATE
:
1685 /* Let the corresponding firstprivate clause create
1687 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1688 scan_array_reductions
= true;
1689 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1693 case OMP_CLAUSE_PRIVATE
:
1694 case OMP_CLAUSE_FIRSTPRIVATE
:
1695 case OMP_CLAUSE_REDUCTION
:
1696 case OMP_CLAUSE_LINEAR
:
1697 decl
= OMP_CLAUSE_DECL (c
);
1698 if (is_variable_sized (decl
))
1699 install_var_local (decl
, ctx
);
1700 fixup_remapped_decl (decl
, ctx
,
1701 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1702 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1703 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1704 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1705 scan_array_reductions
= true;
1708 case OMP_CLAUSE_SHARED
:
1709 /* Ignore shared directives in teams construct. */
1710 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1712 decl
= OMP_CLAUSE_DECL (c
);
1713 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1714 fixup_remapped_decl (decl
, ctx
, false);
1717 case OMP_CLAUSE_MAP
:
1718 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1720 decl
= OMP_CLAUSE_DECL (c
);
1722 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1723 && lookup_attribute ("omp declare target",
1724 DECL_ATTRIBUTES (decl
)))
1728 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1729 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1730 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1732 tree new_decl
= lookup_decl (decl
, ctx
);
1733 TREE_TYPE (new_decl
)
1734 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1736 else if (DECL_SIZE (decl
)
1737 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1739 tree decl2
= DECL_VALUE_EXPR (decl
);
1740 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1741 decl2
= TREE_OPERAND (decl2
, 0);
1742 gcc_assert (DECL_P (decl2
));
1743 fixup_remapped_decl (decl2
, ctx
, false);
1744 fixup_remapped_decl (decl
, ctx
, true);
1747 fixup_remapped_decl (decl
, ctx
, false);
1751 case OMP_CLAUSE_COPYPRIVATE
:
1752 case OMP_CLAUSE_COPYIN
:
1753 case OMP_CLAUSE_DEFAULT
:
1755 case OMP_CLAUSE_NUM_THREADS
:
1756 case OMP_CLAUSE_NUM_TEAMS
:
1757 case OMP_CLAUSE_THREAD_LIMIT
:
1758 case OMP_CLAUSE_DEVICE
:
1759 case OMP_CLAUSE_SCHEDULE
:
1760 case OMP_CLAUSE_DIST_SCHEDULE
:
1761 case OMP_CLAUSE_NOWAIT
:
1762 case OMP_CLAUSE_ORDERED
:
1763 case OMP_CLAUSE_COLLAPSE
:
1764 case OMP_CLAUSE_UNTIED
:
1765 case OMP_CLAUSE_FINAL
:
1766 case OMP_CLAUSE_MERGEABLE
:
1767 case OMP_CLAUSE_PROC_BIND
:
1768 case OMP_CLAUSE_SAFELEN
:
1769 case OMP_CLAUSE_ALIGNED
:
1770 case OMP_CLAUSE_DEPEND
:
1771 case OMP_CLAUSE__LOOPTEMP_
:
1773 case OMP_CLAUSE_FROM
:
1781 if (scan_array_reductions
)
1782 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1783 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1784 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1786 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1787 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1789 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1790 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1791 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1794 /* Create a new name for omp child function. Returns an identifier. */
1796 static GTY(()) unsigned int tmp_ompfn_id_num
;
1799 create_omp_child_function_name (bool task_copy
)
1801 return (clone_function_name (current_function_decl
,
1802 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1805 /* Build a decl for the omp child function. It'll not contain a body
1806 yet, just the bare decl. */
1809 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1811 tree decl
, type
, name
, t
;
1813 name
= create_omp_child_function_name (task_copy
);
1815 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1816 ptr_type_node
, NULL_TREE
);
1818 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1820 decl
= build_decl (gimple_location (ctx
->stmt
),
1821 FUNCTION_DECL
, name
, type
);
1824 ctx
->cb
.dst_fn
= decl
;
1826 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1828 TREE_STATIC (decl
) = 1;
1829 TREE_USED (decl
) = 1;
1830 DECL_ARTIFICIAL (decl
) = 1;
1831 DECL_NAMELESS (decl
) = 1;
1832 DECL_IGNORED_P (decl
) = 0;
1833 TREE_PUBLIC (decl
) = 0;
1834 DECL_UNINLINABLE (decl
) = 1;
1835 DECL_EXTERNAL (decl
) = 0;
1836 DECL_CONTEXT (decl
) = NULL_TREE
;
1837 DECL_INITIAL (decl
) = make_node (BLOCK
);
1838 bool target_p
= false;
1839 if (lookup_attribute ("omp declare target",
1840 DECL_ATTRIBUTES (current_function_decl
)))
1845 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1846 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1847 && gimple_omp_target_kind (octx
->stmt
)
1848 == GF_OMP_TARGET_KIND_REGION
)
1855 DECL_ATTRIBUTES (decl
)
1856 = tree_cons (get_identifier ("omp declare target"),
1857 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1859 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1860 RESULT_DECL
, NULL_TREE
, void_type_node
);
1861 DECL_ARTIFICIAL (t
) = 1;
1862 DECL_IGNORED_P (t
) = 1;
1863 DECL_CONTEXT (t
) = decl
;
1864 DECL_RESULT (decl
) = t
;
1866 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1867 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1868 DECL_ARTIFICIAL (t
) = 1;
1869 DECL_NAMELESS (t
) = 1;
1870 DECL_ARG_TYPE (t
) = ptr_type_node
;
1871 DECL_CONTEXT (t
) = current_function_decl
;
1873 DECL_ARGUMENTS (decl
) = t
;
1875 ctx
->receiver_decl
= t
;
1878 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1879 PARM_DECL
, get_identifier (".omp_data_o"),
1881 DECL_ARTIFICIAL (t
) = 1;
1882 DECL_NAMELESS (t
) = 1;
1883 DECL_ARG_TYPE (t
) = ptr_type_node
;
1884 DECL_CONTEXT (t
) = current_function_decl
;
1886 TREE_ADDRESSABLE (t
) = 1;
1887 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1888 DECL_ARGUMENTS (decl
) = t
;
1891 /* Allocate memory for the function structure. The call to
1892 allocate_struct_function clobbers CFUN, so we need to restore
1894 push_struct_function (decl
);
1895 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1899 /* Callback for walk_gimple_seq. Check if combined parallel
1900 contains gimple_omp_for_combined_into_p OMP_FOR. */
1903 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1904 bool *handled_ops_p
,
1905 struct walk_stmt_info
*wi
)
1907 gimple stmt
= gsi_stmt (*gsi_p
);
1909 *handled_ops_p
= true;
1910 switch (gimple_code (stmt
))
1914 case GIMPLE_OMP_FOR
:
1915 if (gimple_omp_for_combined_into_p (stmt
)
1916 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1919 return integer_zero_node
;
1928 /* Scan an OpenMP parallel directive. */
1931 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1935 gimple stmt
= gsi_stmt (*gsi
);
1937 /* Ignore parallel directives with empty bodies, unless there
1938 are copyin clauses. */
1940 && empty_body_p (gimple_omp_body (stmt
))
1941 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1942 OMP_CLAUSE_COPYIN
) == NULL
)
1944 gsi_replace (gsi
, gimple_build_nop (), false);
1948 if (gimple_omp_parallel_combined_p (stmt
))
1951 struct walk_stmt_info wi
;
1953 memset (&wi
, 0, sizeof (wi
));
1955 walk_gimple_seq (gimple_omp_body (stmt
),
1956 find_combined_for
, NULL
, &wi
);
1957 for_stmt
= (gimple
) wi
.info
;
1960 struct omp_for_data fd
;
1961 extract_omp_for_data (for_stmt
, &fd
, NULL
);
1962 /* We need two temporaries with fd.loop.v type (istart/iend)
1963 and then (fd.collapse - 1) temporaries with the same
1964 type for count2 ... countN-1 vars if not constant. */
1965 size_t count
= 2, i
;
1966 tree type
= fd
.iter_type
;
1968 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
1969 count
+= fd
.collapse
- 1;
1970 for (i
= 0; i
< count
; i
++)
1972 tree temp
= create_tmp_var (type
, NULL
);
1973 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
1974 OMP_CLAUSE__LOOPTEMP_
);
1975 OMP_CLAUSE_DECL (c
) = temp
;
1976 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
1977 gimple_omp_parallel_set_clauses (stmt
, c
);
1982 ctx
= new_omp_context (stmt
, outer_ctx
);
1983 if (taskreg_nesting_level
> 1)
1984 ctx
->is_nested
= true;
1985 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1986 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
1987 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1988 name
= create_tmp_var_name (".omp_data_s");
1989 name
= build_decl (gimple_location (stmt
),
1990 TYPE_DECL
, name
, ctx
->record_type
);
1991 DECL_ARTIFICIAL (name
) = 1;
1992 DECL_NAMELESS (name
) = 1;
1993 TYPE_NAME (ctx
->record_type
) = name
;
1994 create_omp_child_function (ctx
, false);
1995 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
1997 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
1998 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2000 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2001 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2004 layout_type (ctx
->record_type
);
2005 fixup_child_record_type (ctx
);
2009 /* Scan an OpenMP task directive. */
2012 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2016 gimple stmt
= gsi_stmt (*gsi
);
2017 location_t loc
= gimple_location (stmt
);
2019 /* Ignore task directives with empty bodies. */
2021 && empty_body_p (gimple_omp_body (stmt
)))
2023 gsi_replace (gsi
, gimple_build_nop (), false);
2027 ctx
= new_omp_context (stmt
, outer_ctx
);
2028 if (taskreg_nesting_level
> 1)
2029 ctx
->is_nested
= true;
2030 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2031 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2032 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2033 name
= create_tmp_var_name (".omp_data_s");
2034 name
= build_decl (gimple_location (stmt
),
2035 TYPE_DECL
, name
, ctx
->record_type
);
2036 DECL_ARTIFICIAL (name
) = 1;
2037 DECL_NAMELESS (name
) = 1;
2038 TYPE_NAME (ctx
->record_type
) = name
;
2039 create_omp_child_function (ctx
, false);
2040 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2042 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2044 if (ctx
->srecord_type
)
2046 name
= create_tmp_var_name (".omp_data_a");
2047 name
= build_decl (gimple_location (stmt
),
2048 TYPE_DECL
, name
, ctx
->srecord_type
);
2049 DECL_ARTIFICIAL (name
) = 1;
2050 DECL_NAMELESS (name
) = 1;
2051 TYPE_NAME (ctx
->srecord_type
) = name
;
2052 create_omp_child_function (ctx
, true);
2055 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2057 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2059 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2060 t
= build_int_cst (long_integer_type_node
, 0);
2061 gimple_omp_task_set_arg_size (stmt
, t
);
2062 t
= build_int_cst (long_integer_type_node
, 1);
2063 gimple_omp_task_set_arg_align (stmt
, t
);
2067 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2068 /* Move VLA fields to the end. */
2069 p
= &TYPE_FIELDS (ctx
->record_type
);
2071 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2072 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2075 *p
= TREE_CHAIN (*p
);
2076 TREE_CHAIN (*q
) = NULL_TREE
;
2077 q
= &TREE_CHAIN (*q
);
2080 p
= &DECL_CHAIN (*p
);
2082 layout_type (ctx
->record_type
);
2083 fixup_child_record_type (ctx
);
2084 if (ctx
->srecord_type
)
2085 layout_type (ctx
->srecord_type
);
2086 t
= fold_convert_loc (loc
, long_integer_type_node
,
2087 TYPE_SIZE_UNIT (ctx
->record_type
));
2088 gimple_omp_task_set_arg_size (stmt
, t
);
2089 t
= build_int_cst (long_integer_type_node
,
2090 TYPE_ALIGN_UNIT (ctx
->record_type
));
2091 gimple_omp_task_set_arg_align (stmt
, t
);
2096 /* Scan an OpenMP loop directive. */
2099 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2104 ctx
= new_omp_context (stmt
, outer_ctx
);
2106 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2108 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2109 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2111 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2112 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2113 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2114 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2116 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2119 /* Scan an OpenMP sections directive. */
2122 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2126 ctx
= new_omp_context (stmt
, outer_ctx
);
2127 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2128 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2131 /* Scan an OpenMP single directive. */
2134 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2139 ctx
= new_omp_context (stmt
, outer_ctx
);
2140 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2141 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2142 name
= create_tmp_var_name (".omp_copy_s");
2143 name
= build_decl (gimple_location (stmt
),
2144 TYPE_DECL
, name
, ctx
->record_type
);
2145 TYPE_NAME (ctx
->record_type
) = name
;
2147 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2148 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2150 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2151 ctx
->record_type
= NULL
;
2153 layout_type (ctx
->record_type
);
2156 /* Scan an OpenMP target{, data, update} directive. */
2159 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2163 int kind
= gimple_omp_target_kind (stmt
);
2165 ctx
= new_omp_context (stmt
, outer_ctx
);
2166 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2167 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2168 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2169 name
= create_tmp_var_name (".omp_data_t");
2170 name
= build_decl (gimple_location (stmt
),
2171 TYPE_DECL
, name
, ctx
->record_type
);
2172 DECL_ARTIFICIAL (name
) = 1;
2173 DECL_NAMELESS (name
) = 1;
2174 TYPE_NAME (ctx
->record_type
) = name
;
2175 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2177 create_omp_child_function (ctx
, false);
2178 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2181 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2182 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2184 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2185 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2188 TYPE_FIELDS (ctx
->record_type
)
2189 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2190 #ifdef ENABLE_CHECKING
2192 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2193 for (field
= TYPE_FIELDS (ctx
->record_type
);
2195 field
= DECL_CHAIN (field
))
2196 gcc_assert (DECL_ALIGN (field
) == align
);
2198 layout_type (ctx
->record_type
);
2199 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2200 fixup_child_record_type (ctx
);
2204 /* Scan an OpenMP teams directive. */
2207 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2209 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2210 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2211 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2214 /* Check OpenMP nesting restrictions. */
2216 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2220 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2221 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
2223 error_at (gimple_location (stmt
),
2224 "OpenMP constructs may not be nested inside simd region");
2227 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2229 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2230 || (gimple_omp_for_kind (stmt
)
2231 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2232 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2234 error_at (gimple_location (stmt
),
2235 "only distribute or parallel constructs are allowed to "
2236 "be closely nested inside teams construct");
2241 switch (gimple_code (stmt
))
2243 case GIMPLE_OMP_FOR
:
2244 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_SIMD
)
2246 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2248 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2250 error_at (gimple_location (stmt
),
2251 "distribute construct must be closely nested inside "
2259 if (is_gimple_call (stmt
)
2260 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2261 == BUILT_IN_GOMP_CANCEL
2262 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2263 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2265 const char *bad
= NULL
;
2266 const char *kind
= NULL
;
2269 error_at (gimple_location (stmt
), "orphaned %qs construct",
2270 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2271 == BUILT_IN_GOMP_CANCEL
2272 ? "#pragma omp cancel"
2273 : "#pragma omp cancellation point");
2276 switch (host_integerp (gimple_call_arg (stmt
, 0), 0)
2277 ? tree_low_cst (gimple_call_arg (stmt
, 0), 0)
2281 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2282 bad
= "#pragma omp parallel";
2283 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2284 == BUILT_IN_GOMP_CANCEL
2285 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2286 ctx
->cancellable
= true;
2290 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2291 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2292 bad
= "#pragma omp for";
2293 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2294 == BUILT_IN_GOMP_CANCEL
2295 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2297 ctx
->cancellable
= true;
2298 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2300 warning_at (gimple_location (stmt
), 0,
2301 "%<#pragma omp cancel for%> inside "
2302 "%<nowait%> for construct");
2303 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2304 OMP_CLAUSE_ORDERED
))
2305 warning_at (gimple_location (stmt
), 0,
2306 "%<#pragma omp cancel for%> inside "
2307 "%<ordered%> for construct");
2312 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2313 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2314 bad
= "#pragma omp sections";
2315 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2316 == BUILT_IN_GOMP_CANCEL
2317 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2319 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2321 ctx
->cancellable
= true;
2322 if (find_omp_clause (gimple_omp_sections_clauses
2325 warning_at (gimple_location (stmt
), 0,
2326 "%<#pragma omp cancel sections%> inside "
2327 "%<nowait%> sections construct");
2331 gcc_assert (ctx
->outer
2332 && gimple_code (ctx
->outer
->stmt
)
2333 == GIMPLE_OMP_SECTIONS
);
2334 ctx
->outer
->cancellable
= true;
2335 if (find_omp_clause (gimple_omp_sections_clauses
2338 warning_at (gimple_location (stmt
), 0,
2339 "%<#pragma omp cancel sections%> inside "
2340 "%<nowait%> sections construct");
2346 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2347 bad
= "#pragma omp task";
2349 ctx
->cancellable
= true;
2353 error_at (gimple_location (stmt
), "invalid arguments");
2358 error_at (gimple_location (stmt
),
2359 "%<%s %s%> construct not closely nested inside of %qs",
2360 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2361 == BUILT_IN_GOMP_CANCEL
2362 ? "#pragma omp cancel"
2363 : "#pragma omp cancellation point", kind
, bad
);
2368 case GIMPLE_OMP_SECTIONS
:
2369 case GIMPLE_OMP_SINGLE
:
2370 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2371 switch (gimple_code (ctx
->stmt
))
2373 case GIMPLE_OMP_FOR
:
2374 case GIMPLE_OMP_SECTIONS
:
2375 case GIMPLE_OMP_SINGLE
:
2376 case GIMPLE_OMP_ORDERED
:
2377 case GIMPLE_OMP_MASTER
:
2378 case GIMPLE_OMP_TASK
:
2379 case GIMPLE_OMP_CRITICAL
:
2380 if (is_gimple_call (stmt
))
2382 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2383 != BUILT_IN_GOMP_BARRIER
)
2385 error_at (gimple_location (stmt
),
2386 "barrier region may not be closely nested inside "
2387 "of work-sharing, critical, ordered, master or "
2388 "explicit task region");
2391 error_at (gimple_location (stmt
),
2392 "work-sharing region may not be closely nested inside "
2393 "of work-sharing, critical, ordered, master or explicit "
2396 case GIMPLE_OMP_PARALLEL
:
2402 case GIMPLE_OMP_MASTER
:
2403 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2404 switch (gimple_code (ctx
->stmt
))
2406 case GIMPLE_OMP_FOR
:
2407 case GIMPLE_OMP_SECTIONS
:
2408 case GIMPLE_OMP_SINGLE
:
2409 case GIMPLE_OMP_TASK
:
2410 error_at (gimple_location (stmt
),
2411 "master region may not be closely nested inside "
2412 "of work-sharing or explicit task region");
2414 case GIMPLE_OMP_PARALLEL
:
2420 case GIMPLE_OMP_ORDERED
:
2421 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2422 switch (gimple_code (ctx
->stmt
))
2424 case GIMPLE_OMP_CRITICAL
:
2425 case GIMPLE_OMP_TASK
:
2426 error_at (gimple_location (stmt
),
2427 "ordered region may not be closely nested inside "
2428 "of critical or explicit task region");
2430 case GIMPLE_OMP_FOR
:
2431 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2432 OMP_CLAUSE_ORDERED
) == NULL
)
2434 error_at (gimple_location (stmt
),
2435 "ordered region must be closely nested inside "
2436 "a loop region with an ordered clause");
2440 case GIMPLE_OMP_PARALLEL
:
2441 error_at (gimple_location (stmt
),
2442 "ordered region must be closely nested inside "
2443 "a loop region with an ordered clause");
2449 case GIMPLE_OMP_CRITICAL
:
2450 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2451 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2452 && (gimple_omp_critical_name (stmt
)
2453 == gimple_omp_critical_name (ctx
->stmt
)))
2455 error_at (gimple_location (stmt
),
2456 "critical region may not be nested inside a critical "
2457 "region with the same name");
2461 case GIMPLE_OMP_TEAMS
:
2463 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2464 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2466 error_at (gimple_location (stmt
),
2467 "teams construct not closely nested inside of target "
2479 /* Helper function scan_omp.
2481 Callback for walk_tree or operators in walk_gimple_stmt used to
2482 scan for OpenMP directives in TP. */
2485 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2487 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2488 omp_context
*ctx
= (omp_context
*) wi
->info
;
2491 switch (TREE_CODE (t
))
2498 *tp
= remap_decl (t
, &ctx
->cb
);
2502 if (ctx
&& TYPE_P (t
))
2503 *tp
= remap_type (t
, &ctx
->cb
);
2504 else if (!DECL_P (t
))
2509 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2510 if (tem
!= TREE_TYPE (t
))
2512 if (TREE_CODE (t
) == INTEGER_CST
)
2513 *tp
= build_int_cst_wide (tem
,
2514 TREE_INT_CST_LOW (t
),
2515 TREE_INT_CST_HIGH (t
));
2517 TREE_TYPE (t
) = tem
;
2528 /* Helper function for scan_omp.
2530 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2531 the current statement in GSI. */
2534 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2535 struct walk_stmt_info
*wi
)
2537 gimple stmt
= gsi_stmt (*gsi
);
2538 omp_context
*ctx
= (omp_context
*) wi
->info
;
2540 if (gimple_has_location (stmt
))
2541 input_location
= gimple_location (stmt
);
2543 /* Check the OpenMP nesting restrictions. */
2544 bool remove
= false;
2545 if (is_gimple_omp (stmt
))
2546 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2547 else if (is_gimple_call (stmt
))
2549 tree fndecl
= gimple_call_fndecl (stmt
);
2551 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2552 switch (DECL_FUNCTION_CODE (fndecl
))
2554 case BUILT_IN_GOMP_BARRIER
:
2555 case BUILT_IN_GOMP_CANCEL
:
2556 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2557 case BUILT_IN_GOMP_TASKYIELD
:
2558 case BUILT_IN_GOMP_TASKWAIT
:
2559 case BUILT_IN_GOMP_TASKGROUP_START
:
2560 case BUILT_IN_GOMP_TASKGROUP_END
:
2561 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2569 stmt
= gimple_build_nop ();
2570 gsi_replace (gsi
, stmt
, false);
2573 *handled_ops_p
= true;
2575 switch (gimple_code (stmt
))
2577 case GIMPLE_OMP_PARALLEL
:
2578 taskreg_nesting_level
++;
2579 scan_omp_parallel (gsi
, ctx
);
2580 taskreg_nesting_level
--;
2583 case GIMPLE_OMP_TASK
:
2584 taskreg_nesting_level
++;
2585 scan_omp_task (gsi
, ctx
);
2586 taskreg_nesting_level
--;
2589 case GIMPLE_OMP_FOR
:
2590 scan_omp_for (stmt
, ctx
);
2593 case GIMPLE_OMP_SECTIONS
:
2594 scan_omp_sections (stmt
, ctx
);
2597 case GIMPLE_OMP_SINGLE
:
2598 scan_omp_single (stmt
, ctx
);
2601 case GIMPLE_OMP_SECTION
:
2602 case GIMPLE_OMP_MASTER
:
2603 case GIMPLE_OMP_TASKGROUP
:
2604 case GIMPLE_OMP_ORDERED
:
2605 case GIMPLE_OMP_CRITICAL
:
2606 ctx
= new_omp_context (stmt
, ctx
);
2607 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2610 case GIMPLE_OMP_TARGET
:
2611 scan_omp_target (stmt
, ctx
);
2614 case GIMPLE_OMP_TEAMS
:
2615 scan_omp_teams (stmt
, ctx
);
2622 *handled_ops_p
= false;
2624 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2625 insert_decl_map (&ctx
->cb
, var
, var
);
2629 *handled_ops_p
= false;
2637 /* Scan all the statements starting at the current statement. CTX
2638 contains context information about the OpenMP directives and
2639 clauses found during the scan. */
2642 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2644 location_t saved_location
;
2645 struct walk_stmt_info wi
;
2647 memset (&wi
, 0, sizeof (wi
));
2649 wi
.want_locations
= true;
2651 saved_location
= input_location
;
2652 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2653 input_location
= saved_location
;
2656 /* Re-gimplification and code generation routines. */
2658 /* Build a call to GOMP_barrier. */
2661 build_omp_barrier (tree lhs
)
2663 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2664 : BUILT_IN_GOMP_BARRIER
);
2665 gimple g
= gimple_build_call (fndecl
, 0);
2667 gimple_call_set_lhs (g
, lhs
);
2671 /* If a context was created for STMT when it was scanned, return it. */
2673 static omp_context
*
2674 maybe_lookup_ctx (gimple stmt
)
2677 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2678 return n
? (omp_context
*) n
->value
: NULL
;
2682 /* Find the mapping for DECL in CTX or the immediately enclosing
2683 context that has a mapping for DECL.
2685 If CTX is a nested parallel directive, we may have to use the decl
2686 mappings created in CTX's parent context. Suppose that we have the
2687 following parallel nesting (variable UIDs showed for clarity):
2690 #omp parallel shared(iD.1562) -> outer parallel
2691 iD.1562 = iD.1562 + 1;
2693 #omp parallel shared (iD.1562) -> inner parallel
2694 iD.1562 = iD.1562 - 1;
2696 Each parallel structure will create a distinct .omp_data_s structure
2697 for copying iD.1562 in/out of the directive:
2699 outer parallel .omp_data_s.1.i -> iD.1562
2700 inner parallel .omp_data_s.2.i -> iD.1562
2702 A shared variable mapping will produce a copy-out operation before
2703 the parallel directive and a copy-in operation after it. So, in
2704 this case we would have:
2707 .omp_data_o.1.i = iD.1562;
2708 #omp parallel shared(iD.1562) -> outer parallel
2709 .omp_data_i.1 = &.omp_data_o.1
2710 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2712 .omp_data_o.2.i = iD.1562; -> **
2713 #omp parallel shared(iD.1562) -> inner parallel
2714 .omp_data_i.2 = &.omp_data_o.2
2715 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2718 ** This is a problem. The symbol iD.1562 cannot be referenced
2719 inside the body of the outer parallel region. But since we are
2720 emitting this copy operation while expanding the inner parallel
2721 directive, we need to access the CTX structure of the outer
2722 parallel directive to get the correct mapping:
2724 .omp_data_o.2.i = .omp_data_i.1->i
2726 Since there may be other workshare or parallel directives enclosing
2727 the parallel directive, it may be necessary to walk up the context
2728 parent chain. This is not a problem in general because nested
2729 parallelism happens only rarely. */
2732 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2737 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2738 t
= maybe_lookup_decl (decl
, up
);
2740 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2742 return t
? t
: decl
;
2746 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2747 in outer contexts. */
2750 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2755 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2756 t
= maybe_lookup_decl (decl
, up
);
2758 return t
? t
: decl
;
2762 /* Construct the initialization value for reduction CLAUSE. */
2765 omp_reduction_init (tree clause
, tree type
)
2767 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2768 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2775 case TRUTH_ORIF_EXPR
:
2776 case TRUTH_XOR_EXPR
:
2778 return build_zero_cst (type
);
2781 case TRUTH_AND_EXPR
:
2782 case TRUTH_ANDIF_EXPR
:
2784 return fold_convert_loc (loc
, type
, integer_one_node
);
2787 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2790 if (SCALAR_FLOAT_TYPE_P (type
))
2792 REAL_VALUE_TYPE max
, min
;
2793 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2796 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2799 real_maxval (&min
, 1, TYPE_MODE (type
));
2800 return build_real (type
, min
);
2804 gcc_assert (INTEGRAL_TYPE_P (type
));
2805 return TYPE_MIN_VALUE (type
);
2809 if (SCALAR_FLOAT_TYPE_P (type
))
2811 REAL_VALUE_TYPE max
;
2812 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2815 real_maxval (&max
, 0, TYPE_MODE (type
));
2816 return build_real (type
, max
);
2820 gcc_assert (INTEGRAL_TYPE_P (type
));
2821 return TYPE_MAX_VALUE (type
);
2829 /* Return alignment to be assumed for var in CLAUSE, which should be
2830 OMP_CLAUSE_ALIGNED. */
2833 omp_clause_aligned_alignment (tree clause
)
2835 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2836 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2838 /* Otherwise return implementation defined alignment. */
2839 unsigned int al
= 1;
2840 enum machine_mode mode
, vmode
;
2841 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2843 vs
= 1 << floor_log2 (vs
);
2844 static enum mode_class classes
[]
2845 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2846 for (int i
= 0; i
< 4; i
+= 2)
2847 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2849 mode
= GET_MODE_WIDER_MODE (mode
))
2851 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2852 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2855 && GET_MODE_SIZE (vmode
) < vs
2856 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2857 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2859 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2860 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2862 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2863 / GET_MODE_SIZE (mode
));
2864 if (TYPE_MODE (type
) != vmode
)
2866 if (TYPE_ALIGN_UNIT (type
) > al
)
2867 al
= TYPE_ALIGN_UNIT (type
);
2869 return build_int_cst (integer_type_node
, al
);
2872 /* Return maximum possible vectorization factor for the target. */
2879 || (!flag_tree_loop_vectorize
2880 && (global_options_set
.x_flag_tree_loop_vectorize
2881 || global_options_set
.x_flag_tree_vectorize
)))
2884 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2887 vs
= 1 << floor_log2 (vs
);
2890 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2891 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2892 return GET_MODE_NUNITS (vqimode
);
2896 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2900 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2901 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2905 max_vf
= omp_max_vf ();
2908 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2909 OMP_CLAUSE_SAFELEN
);
2911 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
), max_vf
) == -1)
2912 max_vf
= tree_low_cst (OMP_CLAUSE_SAFELEN_EXPR (c
), 0);
2916 idx
= create_tmp_var (unsigned_type_node
, NULL
);
2917 lane
= create_tmp_var (unsigned_type_node
, NULL
);
2923 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
2924 tree avar
= create_tmp_var_raw (atype
, NULL
);
2925 if (TREE_ADDRESSABLE (new_var
))
2926 TREE_ADDRESSABLE (avar
) = 1;
2927 DECL_ATTRIBUTES (avar
)
2928 = tree_cons (get_identifier ("omp simd array"), NULL
,
2929 DECL_ATTRIBUTES (avar
));
2930 gimple_add_tmp_var (avar
);
2931 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
2932 NULL_TREE
, NULL_TREE
);
2933 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
2934 NULL_TREE
, NULL_TREE
);
2935 if (DECL_P (new_var
))
2937 SET_DECL_VALUE_EXPR (new_var
, lvar
);
2938 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
2943 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2944 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2945 private variables. Initialization statements go in ILIST, while calls
2946 to destructors go in DLIST. */
2949 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
2950 omp_context
*ctx
, struct omp_for_data
*fd
)
2952 tree c
, dtor
, copyin_seq
, x
, ptr
;
2953 bool copyin_by_ref
= false;
2954 bool lastprivate_firstprivate
= false;
2955 bool reduction_omp_orig_ref
= false;
2957 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2958 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
);
2960 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
2961 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
2962 gimple_seq llist
[2] = { NULL
, NULL
};
2966 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
2967 with data sharing clauses referencing variable sized vars. That
2968 is unnecessarily hard to support and very unlikely to result in
2969 vectorized code anyway. */
2971 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2972 switch (OMP_CLAUSE_CODE (c
))
2974 case OMP_CLAUSE_REDUCTION
:
2975 case OMP_CLAUSE_PRIVATE
:
2976 case OMP_CLAUSE_FIRSTPRIVATE
:
2977 case OMP_CLAUSE_LASTPRIVATE
:
2978 case OMP_CLAUSE_LINEAR
:
2979 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
2986 /* Do all the fixed sized types in the first pass, and the variable sized
2987 types in the second pass. This makes sure that the scalar arguments to
2988 the variable sized types are processed before we use them in the
2989 variable sized operations. */
2990 for (pass
= 0; pass
< 2; ++pass
)
2992 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2994 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
2997 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3001 case OMP_CLAUSE_PRIVATE
:
3002 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3005 case OMP_CLAUSE_SHARED
:
3006 /* Ignore shared directives in teams construct. */
3007 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3009 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3011 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3014 case OMP_CLAUSE_FIRSTPRIVATE
:
3015 case OMP_CLAUSE_COPYIN
:
3016 case OMP_CLAUSE_LINEAR
:
3018 case OMP_CLAUSE_REDUCTION
:
3019 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3020 reduction_omp_orig_ref
= true;
3022 case OMP_CLAUSE__LOOPTEMP_
:
3023 /* Handle _looptemp_ clauses only on parallel. */
3027 case OMP_CLAUSE_LASTPRIVATE
:
3028 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3030 lastprivate_firstprivate
= true;
3035 case OMP_CLAUSE_ALIGNED
:
3038 var
= OMP_CLAUSE_DECL (c
);
3039 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3040 && !is_global_var (var
))
3042 new_var
= maybe_lookup_decl (var
, ctx
);
3043 if (new_var
== NULL_TREE
)
3044 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3045 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3046 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3047 omp_clause_aligned_alignment (c
));
3048 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3049 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3050 gimplify_and_add (x
, ilist
);
3052 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3053 && is_global_var (var
))
3055 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3056 new_var
= lookup_decl (var
, ctx
);
3057 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3058 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3059 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3060 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3061 omp_clause_aligned_alignment (c
));
3062 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3063 x
= create_tmp_var (ptype
, NULL
);
3064 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3065 gimplify_and_add (t
, ilist
);
3066 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3067 SET_DECL_VALUE_EXPR (new_var
, t
);
3068 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3075 new_var
= var
= OMP_CLAUSE_DECL (c
);
3076 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3077 new_var
= lookup_decl (var
, ctx
);
3079 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3084 else if (is_variable_sized (var
))
3086 /* For variable sized types, we need to allocate the
3087 actual storage here. Call alloca and store the
3088 result in the pointer decl that we created elsewhere. */
3092 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3097 ptr
= DECL_VALUE_EXPR (new_var
);
3098 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3099 ptr
= TREE_OPERAND (ptr
, 0);
3100 gcc_assert (DECL_P (ptr
));
3101 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3103 /* void *tmp = __builtin_alloca */
3104 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3105 stmt
= gimple_build_call (atmp
, 1, x
);
3106 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3107 gimple_add_tmp_var (tmp
);
3108 gimple_call_set_lhs (stmt
, tmp
);
3110 gimple_seq_add_stmt (ilist
, stmt
);
3112 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3113 gimplify_assign (ptr
, x
, ilist
);
3116 else if (is_reference (var
))
3118 /* For references that are being privatized for Fortran,
3119 allocate new backing storage for the new pointer
3120 variable. This allows us to avoid changing all the
3121 code that expects a pointer to something that expects
3122 a direct variable. */
3126 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3127 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3129 x
= build_receiver_ref (var
, false, ctx
);
3130 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3132 else if (TREE_CONSTANT (x
))
3134 const char *name
= NULL
;
3135 if (DECL_NAME (var
))
3136 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3138 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3140 gimple_add_tmp_var (x
);
3141 TREE_ADDRESSABLE (x
) = 1;
3142 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3146 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3147 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3150 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3151 gimplify_assign (new_var
, x
, ilist
);
3153 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3155 else if (c_kind
== OMP_CLAUSE_REDUCTION
3156 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3164 switch (OMP_CLAUSE_CODE (c
))
3166 case OMP_CLAUSE_SHARED
:
3167 /* Ignore shared directives in teams construct. */
3168 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3170 /* Shared global vars are just accessed directly. */
3171 if (is_global_var (new_var
))
3173 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3174 needs to be delayed until after fixup_child_record_type so
3175 that we get the correct type during the dereference. */
3176 by_ref
= use_pointer_for_field (var
, ctx
);
3177 x
= build_receiver_ref (var
, by_ref
, ctx
);
3178 SET_DECL_VALUE_EXPR (new_var
, x
);
3179 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3181 /* ??? If VAR is not passed by reference, and the variable
3182 hasn't been initialized yet, then we'll get a warning for
3183 the store into the omp_data_s structure. Ideally, we'd be
3184 able to notice this and not store anything at all, but
3185 we're generating code too early. Suppress the warning. */
3187 TREE_NO_WARNING (var
) = 1;
3190 case OMP_CLAUSE_LASTPRIVATE
:
3191 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3195 case OMP_CLAUSE_PRIVATE
:
3196 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3197 x
= build_outer_var_ref (var
, ctx
);
3198 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3200 if (is_task_ctx (ctx
))
3201 x
= build_receiver_ref (var
, false, ctx
);
3203 x
= build_outer_var_ref (var
, ctx
);
3209 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3212 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3213 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3214 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3215 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3216 idx
, lane
, ivar
, lvar
))
3219 x
= lang_hooks
.decls
.omp_clause_default_ctor
3220 (c
, unshare_expr (ivar
), x
);
3222 gimplify_and_add (x
, &llist
[0]);
3225 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3228 gimple_seq tseq
= NULL
;
3231 gimplify_stmt (&dtor
, &tseq
);
3232 gimple_seq_add_seq (&llist
[1], tseq
);
3239 gimplify_and_add (nx
, ilist
);
3243 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3246 gimple_seq tseq
= NULL
;
3249 gimplify_stmt (&dtor
, &tseq
);
3250 gimple_seq_add_seq (dlist
, tseq
);
3254 case OMP_CLAUSE_LINEAR
:
3255 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3256 goto do_firstprivate
;
3257 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3260 x
= build_outer_var_ref (var
, ctx
);
3263 case OMP_CLAUSE_FIRSTPRIVATE
:
3264 if (is_task_ctx (ctx
))
3266 if (is_reference (var
) || is_variable_sized (var
))
3268 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3270 || use_pointer_for_field (var
, NULL
))
3272 x
= build_receiver_ref (var
, false, ctx
);
3273 SET_DECL_VALUE_EXPR (new_var
, x
);
3274 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3279 x
= build_outer_var_ref (var
, ctx
);
3282 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3283 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3285 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3286 ? sizetype
: TREE_TYPE (x
);
3287 tree t
= fold_convert (stept
,
3288 OMP_CLAUSE_LINEAR_STEP (c
));
3289 tree c
= find_omp_clause (clauses
,
3290 OMP_CLAUSE__LOOPTEMP_
);
3292 tree l
= OMP_CLAUSE_DECL (c
);
3293 if (fd
->collapse
== 1)
3295 tree n1
= fd
->loop
.n1
;
3296 tree step
= fd
->loop
.step
;
3297 tree itype
= TREE_TYPE (l
);
3298 if (POINTER_TYPE_P (itype
))
3299 itype
= signed_type_for (itype
);
3300 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3301 if (TYPE_UNSIGNED (itype
)
3302 && fd
->loop
.cond_code
== GT_EXPR
)
3303 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3304 fold_build1 (NEGATE_EXPR
,
3306 fold_build1 (NEGATE_EXPR
,
3309 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3311 t
= fold_build2 (MULT_EXPR
, stept
,
3312 fold_convert (stept
, l
), t
);
3313 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3314 x
= fold_build2 (POINTER_PLUS_EXPR
,
3315 TREE_TYPE (x
), x
, t
);
3317 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3320 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3321 || TREE_ADDRESSABLE (new_var
))
3322 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3323 idx
, lane
, ivar
, lvar
))
3325 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3327 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3328 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3329 gimplify_and_add (x
, ilist
);
3330 gimple_stmt_iterator gsi
3331 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3333 = gimple_build_assign (unshare_expr (lvar
), iv
);
3334 gsi_insert_before_without_update (&gsi
, g
,
3336 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3337 ? sizetype
: TREE_TYPE (x
);
3338 tree t
= fold_convert (stept
,
3339 OMP_CLAUSE_LINEAR_STEP (c
));
3340 enum tree_code code
= PLUS_EXPR
;
3341 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3342 code
= POINTER_PLUS_EXPR
;
3343 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3344 gsi_insert_before_without_update (&gsi
, g
,
3348 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3349 (c
, unshare_expr (ivar
), x
);
3350 gimplify_and_add (x
, &llist
[0]);
3351 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3354 gimple_seq tseq
= NULL
;
3357 gimplify_stmt (&dtor
, &tseq
);
3358 gimple_seq_add_seq (&llist
[1], tseq
);
3363 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3364 gimplify_and_add (x
, ilist
);
3367 case OMP_CLAUSE__LOOPTEMP_
:
3368 gcc_assert (is_parallel_ctx (ctx
));
3369 x
= build_outer_var_ref (var
, ctx
);
3370 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3371 gimplify_and_add (x
, ilist
);
3374 case OMP_CLAUSE_COPYIN
:
3375 by_ref
= use_pointer_for_field (var
, NULL
);
3376 x
= build_receiver_ref (var
, by_ref
, ctx
);
3377 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3378 append_to_statement_list (x
, ©in_seq
);
3379 copyin_by_ref
|= by_ref
;
3382 case OMP_CLAUSE_REDUCTION
:
3383 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3385 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3387 x
= build_outer_var_ref (var
, ctx
);
3389 if (is_reference (var
)
3390 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3392 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3393 SET_DECL_VALUE_EXPR (placeholder
, x
);
3394 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3395 tree new_vard
= new_var
;
3396 if (is_reference (var
))
3398 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3399 new_vard
= TREE_OPERAND (new_var
, 0);
3400 gcc_assert (DECL_P (new_vard
));
3403 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3404 idx
, lane
, ivar
, lvar
))
3406 if (new_vard
== new_var
)
3408 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3409 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3413 SET_DECL_VALUE_EXPR (new_vard
,
3414 build_fold_addr_expr (ivar
));
3415 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3417 x
= lang_hooks
.decls
.omp_clause_default_ctor
3418 (c
, unshare_expr (ivar
),
3419 build_outer_var_ref (var
, ctx
));
3421 gimplify_and_add (x
, &llist
[0]);
3422 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3424 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3425 lower_omp (&tseq
, ctx
);
3426 gimple_seq_add_seq (&llist
[0], tseq
);
3428 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3429 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3430 lower_omp (&tseq
, ctx
);
3431 gimple_seq_add_seq (&llist
[1], tseq
);
3432 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3433 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3434 if (new_vard
== new_var
)
3435 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3437 SET_DECL_VALUE_EXPR (new_vard
,
3438 build_fold_addr_expr (lvar
));
3439 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3444 gimplify_stmt (&dtor
, &tseq
);
3445 gimple_seq_add_seq (&llist
[1], tseq
);
3449 x
= lang_hooks
.decls
.omp_clause_default_ctor
3450 (c
, new_var
, unshare_expr (x
));
3452 gimplify_and_add (x
, ilist
);
3453 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3455 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3456 lower_omp (&tseq
, ctx
);
3457 gimple_seq_add_seq (ilist
, tseq
);
3459 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3462 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3463 lower_omp (&tseq
, ctx
);
3464 gimple_seq_add_seq (dlist
, tseq
);
3465 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3467 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3472 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3473 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3475 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3476 idx
, lane
, ivar
, lvar
))
3478 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3479 tree ref
= build_outer_var_ref (var
, ctx
);
3481 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3483 /* reduction(-:var) sums up the partial results, so it
3484 acts identically to reduction(+:var). */
3485 if (code
== MINUS_EXPR
)
3488 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3489 ref
= build_outer_var_ref (var
, ctx
);
3490 gimplify_assign (ref
, x
, &llist
[1]);
3494 gimplify_assign (new_var
, x
, ilist
);
3496 gimplify_assign (build_outer_var_ref (var
, ctx
),
3510 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3511 /* Don't want uninit warnings on simduid, it is always uninitialized,
3512 but we use it not for the value, but for the DECL_UID only. */
3513 TREE_NO_WARNING (uid
) = 1;
3515 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3516 gimple_call_set_lhs (g
, lane
);
3517 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3518 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3519 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3520 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3521 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3522 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3523 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3524 build_int_cst (unsigned_type_node
, 0),
3526 gimple_seq_add_stmt (ilist
, g
);
3527 for (int i
= 0; i
< 2; i
++)
3530 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3531 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3532 gimple_call_set_lhs (g
, vf
);
3533 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3534 gimple_seq_add_stmt (seq
, g
);
3535 tree t
= build_int_cst (unsigned_type_node
, 0);
3536 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3537 gimple_seq_add_stmt (seq
, g
);
3538 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3539 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3540 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3541 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3542 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3543 gimple_seq_add_seq (seq
, llist
[i
]);
3544 t
= build_int_cst (unsigned_type_node
, 1);
3545 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3546 gimple_seq_add_stmt (seq
, g
);
3547 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3548 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3549 gimple_seq_add_stmt (seq
, g
);
3550 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3554 /* The copyin sequence is not to be executed by the main thread, since
3555 that would result in self-copies. Perhaps not visible to scalars,
3556 but it certainly is to C++ operator=. */
3559 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3561 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3562 build_int_cst (TREE_TYPE (x
), 0));
3563 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3564 gimplify_and_add (x
, ilist
);
3567 /* If any copyin variable is passed by reference, we must ensure the
3568 master thread doesn't modify it before it is copied over in all
3569 threads. Similarly for variables in both firstprivate and
3570 lastprivate clauses we need to ensure the lastprivate copying
3571 happens after firstprivate copying in all threads. And similarly
3572 for UDRs if initializer expression refers to omp_orig. */
3573 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3575 /* Don't add any barrier for #pragma omp simd or
3576 #pragma omp distribute. */
3577 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3578 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3579 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3582 /* If max_vf is non-zero, then we can use only a vectorization factor
3583 up to the max_vf we chose. So stick it into the safelen clause. */
3586 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3587 OMP_CLAUSE_SAFELEN
);
3589 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3592 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3593 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3595 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3596 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3602 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3603 both parallel and workshare constructs. PREDICATE may be NULL if it's
3607 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3610 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3611 bool par_clauses
= false;
3612 tree simduid
= NULL
, lastlane
= NULL
;
3614 /* Early exit if there are no lastprivate or linear clauses. */
3615 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3616 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3617 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3618 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3620 if (clauses
== NULL
)
3622 /* If this was a workshare clause, see if it had been combined
3623 with its parallel. In that case, look for the clauses on the
3624 parallel statement itself. */
3625 if (is_parallel_ctx (ctx
))
3629 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3632 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3633 OMP_CLAUSE_LASTPRIVATE
);
3634 if (clauses
== NULL
)
3642 tree label_true
, arm1
, arm2
;
3644 label
= create_artificial_label (UNKNOWN_LOCATION
);
3645 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3646 arm1
= TREE_OPERAND (predicate
, 0);
3647 arm2
= TREE_OPERAND (predicate
, 1);
3648 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3649 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3650 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3652 gimple_seq_add_stmt (stmt_list
, stmt
);
3653 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3656 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3657 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
3659 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3661 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3664 for (c
= clauses
; c
;)
3667 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3669 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3670 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3671 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3673 var
= OMP_CLAUSE_DECL (c
);
3674 new_var
= lookup_decl (var
, ctx
);
3676 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3678 tree val
= DECL_VALUE_EXPR (new_var
);
3679 if (TREE_CODE (val
) == ARRAY_REF
3680 && VAR_P (TREE_OPERAND (val
, 0))
3681 && lookup_attribute ("omp simd array",
3682 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3685 if (lastlane
== NULL
)
3687 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3689 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3691 TREE_OPERAND (val
, 1));
3692 gimple_call_set_lhs (g
, lastlane
);
3693 gimple_seq_add_stmt (stmt_list
, g
);
3695 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3696 TREE_OPERAND (val
, 0), lastlane
,
3697 NULL_TREE
, NULL_TREE
);
3701 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3702 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3704 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3705 gimple_seq_add_seq (stmt_list
,
3706 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3707 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3710 x
= build_outer_var_ref (var
, ctx
);
3711 if (is_reference (var
))
3712 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3713 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3714 gimplify_and_add (x
, stmt_list
);
3716 c
= OMP_CLAUSE_CHAIN (c
);
3717 if (c
== NULL
&& !par_clauses
)
3719 /* If this was a workshare clause, see if it had been combined
3720 with its parallel. In that case, continue looking for the
3721 clauses also on the parallel statement itself. */
3722 if (is_parallel_ctx (ctx
))
3726 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3729 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3730 OMP_CLAUSE_LASTPRIVATE
);
3736 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3740 /* Generate code to implement the REDUCTION clauses. */
3743 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3745 gimple_seq sub_seq
= NULL
;
3750 /* SIMD reductions are handled in lower_rec_input_clauses. */
3751 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3752 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
3755 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3756 update in that case, otherwise use a lock. */
3757 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3758 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3760 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3762 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3772 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3774 tree var
, ref
, new_var
;
3775 enum tree_code code
;
3776 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3778 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3781 var
= OMP_CLAUSE_DECL (c
);
3782 new_var
= lookup_decl (var
, ctx
);
3783 if (is_reference (var
))
3784 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3785 ref
= build_outer_var_ref (var
, ctx
);
3786 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3788 /* reduction(-:var) sums up the partial results, so it acts
3789 identically to reduction(+:var). */
3790 if (code
== MINUS_EXPR
)
3795 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3797 addr
= save_expr (addr
);
3798 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3799 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3800 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3801 gimplify_and_add (x
, stmt_seqp
);
3805 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3807 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3809 if (is_reference (var
)
3810 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3812 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3813 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3814 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3815 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3816 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3817 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3818 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3822 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3823 ref
= build_outer_var_ref (var
, ctx
);
3824 gimplify_assign (ref
, x
, &sub_seq
);
3828 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
3830 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3832 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
3834 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
3836 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3840 /* Generate code to implement the COPYPRIVATE clauses. */
3843 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
3848 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3850 tree var
, new_var
, ref
, x
;
3852 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3854 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
3857 var
= OMP_CLAUSE_DECL (c
);
3858 by_ref
= use_pointer_for_field (var
, NULL
);
3860 ref
= build_sender_ref (var
, ctx
);
3861 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
3864 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
3865 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
3867 gimplify_assign (ref
, x
, slist
);
3869 ref
= build_receiver_ref (var
, false, ctx
);
3872 ref
= fold_convert_loc (clause_loc
,
3873 build_pointer_type (TREE_TYPE (new_var
)),
3875 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
3877 if (is_reference (var
))
3879 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
3880 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
3881 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3883 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
3884 gimplify_and_add (x
, rlist
);
3889 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3890 and REDUCTION from the sender (aka parent) side. */
3893 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
3898 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3900 tree val
, ref
, x
, var
;
3901 bool by_ref
, do_in
= false, do_out
= false;
3902 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3904 switch (OMP_CLAUSE_CODE (c
))
3906 case OMP_CLAUSE_PRIVATE
:
3907 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3910 case OMP_CLAUSE_FIRSTPRIVATE
:
3911 case OMP_CLAUSE_COPYIN
:
3912 case OMP_CLAUSE_LASTPRIVATE
:
3913 case OMP_CLAUSE_REDUCTION
:
3914 case OMP_CLAUSE__LOOPTEMP_
:
3920 val
= OMP_CLAUSE_DECL (c
);
3921 var
= lookup_decl_in_outer_ctx (val
, ctx
);
3923 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
3924 && is_global_var (var
))
3926 if (is_variable_sized (val
))
3928 by_ref
= use_pointer_for_field (val
, NULL
);
3930 switch (OMP_CLAUSE_CODE (c
))
3932 case OMP_CLAUSE_PRIVATE
:
3933 case OMP_CLAUSE_FIRSTPRIVATE
:
3934 case OMP_CLAUSE_COPYIN
:
3935 case OMP_CLAUSE__LOOPTEMP_
:
3939 case OMP_CLAUSE_LASTPRIVATE
:
3940 if (by_ref
|| is_reference (val
))
3942 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3949 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
3954 case OMP_CLAUSE_REDUCTION
:
3956 do_out
= !(by_ref
|| is_reference (val
));
3965 ref
= build_sender_ref (val
, ctx
);
3966 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
3967 gimplify_assign (ref
, x
, ilist
);
3968 if (is_task_ctx (ctx
))
3969 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
3974 ref
= build_sender_ref (val
, ctx
);
3975 gimplify_assign (var
, ref
, olist
);
3980 /* Generate code to implement SHARED from the sender (aka parent)
3981 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
3982 list things that got automatically shared. */
3985 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
3987 tree var
, ovar
, nvar
, f
, x
, record_type
;
3989 if (ctx
->record_type
== NULL
)
3992 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
3993 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
3995 ovar
= DECL_ABSTRACT_ORIGIN (f
);
3996 nvar
= maybe_lookup_decl (ovar
, ctx
);
3997 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4000 /* If CTX is a nested parallel directive. Find the immediately
4001 enclosing parallel or workshare construct that contains a
4002 mapping for OVAR. */
4003 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4005 if (use_pointer_for_field (ovar
, ctx
))
4007 x
= build_sender_ref (ovar
, ctx
);
4008 var
= build_fold_addr_expr (var
);
4009 gimplify_assign (x
, var
, ilist
);
4013 x
= build_sender_ref (ovar
, ctx
);
4014 gimplify_assign (x
, var
, ilist
);
4016 if (!TREE_READONLY (var
)
4017 /* We don't need to receive a new reference to a result
4018 or parm decl. In fact we may not store to it as we will
4019 invalidate any pending RSO and generate wrong gimple
4021 && !((TREE_CODE (var
) == RESULT_DECL
4022 || TREE_CODE (var
) == PARM_DECL
)
4023 && DECL_BY_REFERENCE (var
)))
4025 x
= build_sender_ref (ovar
, ctx
);
4026 gimplify_assign (var
, x
, olist
);
4033 /* A convenience function to build an empty GIMPLE_COND with just the
4037 gimple_build_cond_empty (tree cond
)
4039 enum tree_code pred_code
;
4042 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4043 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4047 /* Build the function calls to GOMP_parallel_start etc to actually
4048 generate the parallel operation. REGION is the parallel region
4049 being expanded. BB is the block where to insert the code. WS_ARGS
4050 will be set if this is a call to a combined parallel+workshare
4051 construct, it contains the list of additional arguments needed by
4052 the workshare construct. */
4055 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4056 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4058 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4059 gimple_stmt_iterator gsi
;
4061 enum built_in_function start_ix
;
4063 location_t clause_loc
;
4064 vec
<tree
, va_gc
> *args
;
4066 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4068 /* Determine what flavor of GOMP_parallel we will be
4070 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4071 if (is_combined_parallel (region
))
4073 switch (region
->inner
->type
)
4075 case GIMPLE_OMP_FOR
:
4076 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4077 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4078 + (region
->inner
->sched_kind
4079 == OMP_CLAUSE_SCHEDULE_RUNTIME
4080 ? 3 : region
->inner
->sched_kind
));
4081 start_ix
= (enum built_in_function
)start_ix2
;
4083 case GIMPLE_OMP_SECTIONS
:
4084 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4091 /* By default, the value of NUM_THREADS is zero (selected at run time)
4092 and there is no conditional. */
4094 val
= build_int_cst (unsigned_type_node
, 0);
4095 flags
= build_int_cst (unsigned_type_node
, 0);
4097 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4099 cond
= OMP_CLAUSE_IF_EXPR (c
);
4101 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4104 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4105 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4108 clause_loc
= gimple_location (entry_stmt
);
4110 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4112 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4114 /* Ensure 'val' is of the correct type. */
4115 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4117 /* If we found the clause 'if (cond)', build either
4118 (cond != 0) or (cond ? val : 1u). */
4121 gimple_stmt_iterator gsi
;
4123 cond
= gimple_boolify (cond
);
4125 if (integer_zerop (val
))
4126 val
= fold_build2_loc (clause_loc
,
4127 EQ_EXPR
, unsigned_type_node
, cond
,
4128 build_int_cst (TREE_TYPE (cond
), 0));
4131 basic_block cond_bb
, then_bb
, else_bb
;
4132 edge e
, e_then
, e_else
;
4133 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4135 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4136 if (gimple_in_ssa_p (cfun
))
4138 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4139 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4140 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4149 e
= split_block (bb
, NULL
);
4154 then_bb
= create_empty_bb (cond_bb
);
4155 else_bb
= create_empty_bb (then_bb
);
4156 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4157 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4159 stmt
= gimple_build_cond_empty (cond
);
4160 gsi
= gsi_start_bb (cond_bb
);
4161 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4163 gsi
= gsi_start_bb (then_bb
);
4164 stmt
= gimple_build_assign (tmp_then
, val
);
4165 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4167 gsi
= gsi_start_bb (else_bb
);
4168 stmt
= gimple_build_assign
4169 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4170 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4172 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4173 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4176 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4177 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4179 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4180 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4182 if (gimple_in_ssa_p (cfun
))
4184 gimple phi
= create_phi_node (tmp_join
, bb
);
4185 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4186 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4192 gsi
= gsi_start_bb (bb
);
4193 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4194 false, GSI_CONTINUE_LINKING
);
4197 gsi
= gsi_last_bb (bb
);
4198 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4200 t1
= null_pointer_node
;
4202 t1
= build_fold_addr_expr (t
);
4203 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4205 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4206 args
->quick_push (t2
);
4207 args
->quick_push (t1
);
4208 args
->quick_push (val
);
4210 args
->splice (*ws_args
);
4211 args
->quick_push (flags
);
4213 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4214 builtin_decl_explicit (start_ix
), args
);
4216 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4217 false, GSI_CONTINUE_LINKING
);
4221 /* Build the function call to GOMP_task to actually
4222 generate the task operation. BB is the block where to insert the code. */
4225 expand_task_call (basic_block bb
, gimple entry_stmt
)
4227 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4228 gimple_stmt_iterator gsi
;
4229 location_t loc
= gimple_location (entry_stmt
);
4231 clauses
= gimple_omp_task_clauses (entry_stmt
);
4233 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4235 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4237 cond
= boolean_true_node
;
4239 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4240 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4241 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4242 flags
= build_int_cst (unsigned_type_node
,
4243 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4245 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4248 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4249 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4250 build_int_cst (unsigned_type_node
, 2),
4251 build_int_cst (unsigned_type_node
, 0));
4252 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4255 depend
= OMP_CLAUSE_DECL (depend
);
4257 depend
= build_int_cst (ptr_type_node
, 0);
4259 gsi
= gsi_last_bb (bb
);
4260 t
= gimple_omp_task_data_arg (entry_stmt
);
4262 t2
= null_pointer_node
;
4264 t2
= build_fold_addr_expr_loc (loc
, t
);
4265 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4266 t
= gimple_omp_task_copy_fn (entry_stmt
);
4268 t3
= null_pointer_node
;
4270 t3
= build_fold_addr_expr_loc (loc
, t
);
4272 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4274 gimple_omp_task_arg_size (entry_stmt
),
4275 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4278 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4279 false, GSI_CONTINUE_LINKING
);
4283 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4284 catch handler and return it. This prevents programs from violating the
4285 structured block semantics with throws. */
4288 maybe_catch_exception (gimple_seq body
)
4293 if (!flag_exceptions
)
4296 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4297 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4299 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4301 g
= gimple_build_eh_must_not_throw (decl
);
4302 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4305 return gimple_seq_alloc_with_stmt (g
);
4308 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4311 vec2chain (vec
<tree
, va_gc
> *v
)
4313 tree chain
= NULL_TREE
, t
;
4316 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4318 DECL_CHAIN (t
) = chain
;
4326 /* Remove barriers in REGION->EXIT's block. Note that this is only
4327 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4328 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4329 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4333 remove_exit_barrier (struct omp_region
*region
)
4335 gimple_stmt_iterator gsi
;
4336 basic_block exit_bb
;
4340 int any_addressable_vars
= -1;
4342 exit_bb
= region
->exit
;
4344 /* If the parallel region doesn't return, we don't have REGION->EXIT
4349 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4350 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4351 statements that can appear in between are extremely limited -- no
4352 memory operations at all. Here, we allow nothing at all, so the
4353 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4354 gsi
= gsi_last_bb (exit_bb
);
4355 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4357 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4360 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4362 gsi
= gsi_last_bb (e
->src
);
4363 if (gsi_end_p (gsi
))
4365 stmt
= gsi_stmt (gsi
);
4366 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4367 && !gimple_omp_return_nowait_p (stmt
))
4369 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4370 in many cases. If there could be tasks queued, the barrier
4371 might be needed to let the tasks run before some local
4372 variable of the parallel that the task uses as shared
4373 runs out of scope. The task can be spawned either
4374 from within current function (this would be easy to check)
4375 or from some function it calls and gets passed an address
4376 of such a variable. */
4377 if (any_addressable_vars
< 0)
4379 gimple parallel_stmt
= last_stmt (region
->entry
);
4380 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4381 tree local_decls
, block
, decl
;
4384 any_addressable_vars
= 0;
4385 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4386 if (TREE_ADDRESSABLE (decl
))
4388 any_addressable_vars
= 1;
4391 for (block
= gimple_block (stmt
);
4392 !any_addressable_vars
4394 && TREE_CODE (block
) == BLOCK
;
4395 block
= BLOCK_SUPERCONTEXT (block
))
4397 for (local_decls
= BLOCK_VARS (block
);
4399 local_decls
= DECL_CHAIN (local_decls
))
4400 if (TREE_ADDRESSABLE (local_decls
))
4402 any_addressable_vars
= 1;
4405 if (block
== gimple_block (parallel_stmt
))
4409 if (!any_addressable_vars
)
4410 gimple_omp_return_set_nowait (stmt
);
4416 remove_exit_barriers (struct omp_region
*region
)
4418 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4419 remove_exit_barrier (region
);
4423 region
= region
->inner
;
4424 remove_exit_barriers (region
);
4425 while (region
->next
)
4427 region
= region
->next
;
4428 remove_exit_barriers (region
);
4433 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4434 calls. These can't be declared as const functions, but
4435 within one parallel body they are constant, so they can be
4436 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4437 which are declared const. Similarly for task body, except
4438 that in untied task omp_get_thread_num () can change at any task
4439 scheduling point. */
4442 optimize_omp_library_calls (gimple entry_stmt
)
4445 gimple_stmt_iterator gsi
;
4446 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4447 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4448 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4449 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4450 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4451 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4452 OMP_CLAUSE_UNTIED
) != NULL
);
4455 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4457 gimple call
= gsi_stmt (gsi
);
4460 if (is_gimple_call (call
)
4461 && (decl
= gimple_call_fndecl (call
))
4462 && DECL_EXTERNAL (decl
)
4463 && TREE_PUBLIC (decl
)
4464 && DECL_INITIAL (decl
) == NULL
)
4468 if (DECL_NAME (decl
) == thr_num_id
)
4470 /* In #pragma omp task untied omp_get_thread_num () can change
4471 during the execution of the task region. */
4474 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4476 else if (DECL_NAME (decl
) == num_thr_id
)
4477 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4481 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4482 || gimple_call_num_args (call
) != 0)
4485 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4488 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4489 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4490 TREE_TYPE (TREE_TYPE (built_in
))))
4493 gimple_call_set_fndecl (call
, built_in
);
4498 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4502 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4506 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4507 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4510 if (TREE_CODE (t
) == ADDR_EXPR
)
4511 recompute_tree_invariant_for_addr_expr (t
);
4513 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4517 /* Prepend TO = FROM assignment before *GSI_P. */
4520 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4522 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4523 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4524 true, GSI_SAME_STMT
);
4525 gimple stmt
= gimple_build_assign (to
, from
);
4526 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4527 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4528 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4530 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4531 gimple_regimplify_operands (stmt
, &gsi
);
4535 /* Expand the OpenMP parallel or task directive starting at REGION. */
4538 expand_omp_taskreg (struct omp_region
*region
)
4540 basic_block entry_bb
, exit_bb
, new_bb
;
4541 struct function
*child_cfun
;
4542 tree child_fn
, block
, t
;
4543 gimple_stmt_iterator gsi
;
4544 gimple entry_stmt
, stmt
;
4546 vec
<tree
, va_gc
> *ws_args
;
4548 entry_stmt
= last_stmt (region
->entry
);
4549 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4550 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4552 entry_bb
= region
->entry
;
4553 exit_bb
= region
->exit
;
4555 if (is_combined_parallel (region
))
4556 ws_args
= region
->ws_args
;
4560 if (child_cfun
->cfg
)
4562 /* Due to inlining, it may happen that we have already outlined
4563 the region, in which case all we need to do is make the
4564 sub-graph unreachable and emit the parallel call. */
4565 edge entry_succ_e
, exit_succ_e
;
4566 gimple_stmt_iterator gsi
;
4568 entry_succ_e
= single_succ_edge (entry_bb
);
4570 gsi
= gsi_last_bb (entry_bb
);
4571 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4572 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4573 gsi_remove (&gsi
, true);
4578 exit_succ_e
= single_succ_edge (exit_bb
);
4579 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4581 remove_edge_and_dominated_blocks (entry_succ_e
);
4585 unsigned srcidx
, dstidx
, num
;
4587 /* If the parallel region needs data sent from the parent
4588 function, then the very first statement (except possible
4589 tree profile counter updates) of the parallel body
4590 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4591 &.OMP_DATA_O is passed as an argument to the child function,
4592 we need to replace it with the argument as seen by the child
4595 In most cases, this will end up being the identity assignment
4596 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4597 a function call that has been inlined, the original PARM_DECL
4598 .OMP_DATA_I may have been converted into a different local
4599 variable. In which case, we need to keep the assignment. */
4600 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4602 basic_block entry_succ_bb
= single_succ (entry_bb
);
4603 gimple_stmt_iterator gsi
;
4605 gimple parcopy_stmt
= NULL
;
4607 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4611 gcc_assert (!gsi_end_p (gsi
));
4612 stmt
= gsi_stmt (gsi
);
4613 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4616 if (gimple_num_ops (stmt
) == 2)
4618 tree arg
= gimple_assign_rhs1 (stmt
);
4620 /* We're ignore the subcode because we're
4621 effectively doing a STRIP_NOPS. */
4623 if (TREE_CODE (arg
) == ADDR_EXPR
4624 && TREE_OPERAND (arg
, 0)
4625 == gimple_omp_taskreg_data_arg (entry_stmt
))
4627 parcopy_stmt
= stmt
;
4633 gcc_assert (parcopy_stmt
!= NULL
);
4634 arg
= DECL_ARGUMENTS (child_fn
);
4636 if (!gimple_in_ssa_p (cfun
))
4638 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4639 gsi_remove (&gsi
, true);
4642 /* ?? Is setting the subcode really necessary ?? */
4643 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4644 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4649 /* If we are in ssa form, we must load the value from the default
4650 definition of the argument. That should not be defined now,
4651 since the argument is not used uninitialized. */
4652 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4653 narg
= make_ssa_name (arg
, gimple_build_nop ());
4654 set_ssa_default_def (cfun
, arg
, narg
);
4655 /* ?? Is setting the subcode really necessary ?? */
4656 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4657 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4658 update_stmt (parcopy_stmt
);
4662 /* Declare local variables needed in CHILD_CFUN. */
4663 block
= DECL_INITIAL (child_fn
);
4664 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4665 /* The gimplifier could record temporaries in parallel/task block
4666 rather than in containing function's local_decls chain,
4667 which would mean cgraph missed finalizing them. Do it now. */
4668 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4669 if (TREE_CODE (t
) == VAR_DECL
4671 && !DECL_EXTERNAL (t
))
4672 varpool_finalize_decl (t
);
4673 DECL_SAVED_TREE (child_fn
) = NULL
;
4674 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4675 gimple_set_body (child_fn
, NULL
);
4676 TREE_USED (block
) = 1;
4678 /* Reset DECL_CONTEXT on function arguments. */
4679 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4680 DECL_CONTEXT (t
) = child_fn
;
4682 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4683 so that it can be moved to the child function. */
4684 gsi
= gsi_last_bb (entry_bb
);
4685 stmt
= gsi_stmt (gsi
);
4686 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4687 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4688 gsi_remove (&gsi
, true);
4689 e
= split_block (entry_bb
, stmt
);
4691 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4693 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4696 gsi
= gsi_last_bb (exit_bb
);
4697 gcc_assert (!gsi_end_p (gsi
)
4698 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4699 stmt
= gimple_build_return (NULL
);
4700 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4701 gsi_remove (&gsi
, true);
4704 /* Move the parallel region into CHILD_CFUN. */
4706 if (gimple_in_ssa_p (cfun
))
4708 init_tree_ssa (child_cfun
);
4709 init_ssa_operands (child_cfun
);
4710 child_cfun
->gimple_df
->in_ssa_p
= true;
4714 block
= gimple_block (entry_stmt
);
4716 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4718 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4719 /* When the OMP expansion process cannot guarantee an up-to-date
4720 loop tree arrange for the child function to fixup loops. */
4721 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4722 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4724 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4725 num
= vec_safe_length (child_cfun
->local_decls
);
4726 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4728 t
= (*child_cfun
->local_decls
)[srcidx
];
4729 if (DECL_CONTEXT (t
) == cfun
->decl
)
4731 if (srcidx
!= dstidx
)
4732 (*child_cfun
->local_decls
)[dstidx
] = t
;
4736 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4738 /* Inform the callgraph about the new function. */
4739 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4740 cgraph_add_new_function (child_fn
, true);
4742 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4743 fixed in a following pass. */
4744 push_cfun (child_cfun
);
4746 optimize_omp_library_calls (entry_stmt
);
4747 rebuild_cgraph_edges ();
4749 /* Some EH regions might become dead, see PR34608. If
4750 pass_cleanup_cfg isn't the first pass to happen with the
4751 new child, these dead EH edges might cause problems.
4752 Clean them up now. */
4753 if (flag_exceptions
)
4756 bool changed
= false;
4759 changed
|= gimple_purge_dead_eh_edges (bb
);
4761 cleanup_tree_cfg ();
4763 if (gimple_in_ssa_p (cfun
))
4764 update_ssa (TODO_update_ssa
);
4768 /* Emit a library call to launch the children threads. */
4769 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4770 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4772 expand_task_call (new_bb
, entry_stmt
);
4773 if (gimple_in_ssa_p (cfun
))
4774 update_ssa (TODO_update_ssa_only_virtuals
);
4778 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4779 of the combined collapse > 1 loop constructs, generate code like:
4780 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4785 count3 = (adj + N32 - N31) / STEP3;
4786 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4791 count2 = (adj + N22 - N21) / STEP2;
4792 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4797 count1 = (adj + N12 - N11) / STEP1;
4798 count = count1 * count2 * count3;
4799 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4801 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4802 of the combined loop constructs, just initialize COUNTS array
4803 from the _looptemp_ clauses. */
4805 /* NOTE: It *could* be better to moosh all of the BBs together,
4806 creating one larger BB with all the computation and the unexpected
4807 jump at the end. I.e.
4809 bool zero3, zero2, zero1, zero;
4812 count3 = (N32 - N31) /[cl] STEP3;
4814 count2 = (N22 - N21) /[cl] STEP2;
4816 count1 = (N12 - N11) /[cl] STEP1;
4817 zero = zero3 || zero2 || zero1;
4818 count = count1 * count2 * count3;
4819 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4821 After all, we expect the zero=false, and thus we expect to have to
4822 evaluate all of the comparison expressions, so short-circuiting
4823 oughtn't be a win. Since the condition isn't protecting a
4824 denominator, we're not concerned about divide-by-zero, so we can
4825 fully evaluate count even if a numerator turned out to be wrong.
4827 It seems like putting this all together would create much better
4828 scheduling opportunities, and less pressure on the chip's branch
4832 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4833 basic_block
&entry_bb
, tree
*counts
,
4834 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4835 basic_block
&l2_dom_bb
)
4837 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
4842 /* Collapsed loops need work for expansion into SSA form. */
4843 gcc_assert (!gimple_in_ssa_p (cfun
));
4845 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
4846 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
4848 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4849 isn't supposed to be handled, as the inner loop doesn't
4851 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
4852 OMP_CLAUSE__LOOPTEMP_
);
4853 gcc_assert (innerc
);
4854 for (i
= 0; i
< fd
->collapse
; i
++)
4856 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
4857 OMP_CLAUSE__LOOPTEMP_
);
4858 gcc_assert (innerc
);
4860 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
4862 counts
[0] = NULL_TREE
;
4867 for (i
= 0; i
< fd
->collapse
; i
++)
4869 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
4871 if (SSA_VAR_P (fd
->loop
.n2
)
4872 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
4873 fold_convert (itype
, fd
->loops
[i
].n1
),
4874 fold_convert (itype
, fd
->loops
[i
].n2
)))
4875 == NULL_TREE
|| !integer_onep (t
)))
4878 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
4879 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
4880 true, GSI_SAME_STMT
);
4881 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
4882 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
4883 true, GSI_SAME_STMT
);
4884 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
4885 NULL_TREE
, NULL_TREE
);
4886 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4887 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
4888 expand_omp_regimplify_p
, NULL
, NULL
)
4889 || walk_tree (gimple_cond_rhs_ptr (stmt
),
4890 expand_omp_regimplify_p
, NULL
, NULL
))
4892 *gsi
= gsi_for_stmt (stmt
);
4893 gimple_regimplify_operands (stmt
, gsi
);
4895 e
= split_block (entry_bb
, stmt
);
4896 if (zero_iter_bb
== NULL
)
4898 first_zero_iter
= i
;
4899 zero_iter_bb
= create_empty_bb (entry_bb
);
4901 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
4902 *gsi
= gsi_after_labels (zero_iter_bb
);
4903 stmt
= gimple_build_assign (fd
->loop
.n2
,
4904 build_zero_cst (type
));
4905 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4906 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
4909 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
4910 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
4911 e
->flags
= EDGE_TRUE_VALUE
;
4912 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
4913 if (l2_dom_bb
== NULL
)
4914 l2_dom_bb
= entry_bb
;
4916 *gsi
= gsi_last_bb (entry_bb
);
4919 if (POINTER_TYPE_P (itype
))
4920 itype
= signed_type_for (itype
);
4921 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
4923 t
= fold_build2 (PLUS_EXPR
, itype
,
4924 fold_convert (itype
, fd
->loops
[i
].step
), t
);
4925 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
4926 fold_convert (itype
, fd
->loops
[i
].n2
));
4927 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
4928 fold_convert (itype
, fd
->loops
[i
].n1
));
4929 /* ?? We could probably use CEIL_DIV_EXPR instead of
4930 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
4931 generate the same code in the end because generically we
4932 don't know that the values involved must be negative for
4934 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
4935 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4936 fold_build1 (NEGATE_EXPR
, itype
, t
),
4937 fold_build1 (NEGATE_EXPR
, itype
,
4938 fold_convert (itype
,
4939 fd
->loops
[i
].step
)));
4941 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
4942 fold_convert (itype
, fd
->loops
[i
].step
));
4943 t
= fold_convert (type
, t
);
4944 if (TREE_CODE (t
) == INTEGER_CST
)
4948 counts
[i
] = create_tmp_reg (type
, ".count");
4949 expand_omp_build_assign (gsi
, counts
[i
], t
);
4951 if (SSA_VAR_P (fd
->loop
.n2
))
4956 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
4957 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
4963 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
4965 V3 = N31 + (T % count3) * STEP3;
4967 V2 = N21 + (T % count2) * STEP2;
4969 V1 = N11 + T * STEP1;
4970 if this loop doesn't have an inner loop construct combined with it.
4971 If it does have an inner loop construct combined with it and the
4972 iteration count isn't known constant, store values from counts array
4973 into its _looptemp_ temporaries instead. */
4976 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4977 tree
*counts
, gimple inner_stmt
, tree startvar
)
4980 if (gimple_omp_for_combined_p (fd
->for_stmt
))
4982 /* If fd->loop.n2 is constant, then no propagation of the counts
4983 is needed, they are constant. */
4984 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
4987 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
4988 ? gimple_omp_parallel_clauses (inner_stmt
)
4989 : gimple_omp_for_clauses (inner_stmt
);
4990 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4991 isn't supposed to be handled, as the inner loop doesn't
4993 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
4994 gcc_assert (innerc
);
4995 for (i
= 0; i
< fd
->collapse
; i
++)
4997 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
4998 OMP_CLAUSE__LOOPTEMP_
);
4999 gcc_assert (innerc
);
5002 tree tem
= OMP_CLAUSE_DECL (innerc
);
5003 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5004 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5005 false, GSI_CONTINUE_LINKING
);
5006 gimple stmt
= gimple_build_assign (tem
, t
);
5007 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5013 tree type
= TREE_TYPE (fd
->loop
.v
);
5014 tree tem
= create_tmp_reg (type
, ".tem");
5015 gimple stmt
= gimple_build_assign (tem
, startvar
);
5016 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5018 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5020 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5022 if (POINTER_TYPE_P (vtype
))
5023 itype
= signed_type_for (vtype
);
5025 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5028 t
= fold_convert (itype
, t
);
5029 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5030 fold_convert (itype
, fd
->loops
[i
].step
));
5031 if (POINTER_TYPE_P (vtype
))
5032 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5034 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5035 t
= force_gimple_operand_gsi (gsi
, t
,
5036 DECL_P (fd
->loops
[i
].v
)
5037 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5039 GSI_CONTINUE_LINKING
);
5040 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5041 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5044 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5045 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5046 false, GSI_CONTINUE_LINKING
);
5047 stmt
= gimple_build_assign (tem
, t
);
5048 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5054 /* Helper function for expand_omp_for_*. Generate code like:
5057 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5061 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5068 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5069 basic_block body_bb
)
5071 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5073 gimple_stmt_iterator gsi
;
5079 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5081 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5083 bb
= create_empty_bb (last_bb
);
5085 add_bb_to_loop (bb
, last_bb
->loop_father
);
5086 gsi
= gsi_start_bb (bb
);
5088 if (i
< fd
->collapse
- 1)
5090 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5091 e
->probability
= REG_BR_PROB_BASE
/ 8;
5093 t
= fd
->loops
[i
+ 1].n1
;
5094 t
= force_gimple_operand_gsi (&gsi
, t
,
5095 DECL_P (fd
->loops
[i
+ 1].v
)
5096 && TREE_ADDRESSABLE (fd
->loops
[i
5099 GSI_CONTINUE_LINKING
);
5100 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5101 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5106 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5108 if (POINTER_TYPE_P (vtype
))
5109 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5111 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5112 t
= force_gimple_operand_gsi (&gsi
, t
,
5113 DECL_P (fd
->loops
[i
].v
)
5114 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5115 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5116 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5117 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5121 t
= fd
->loops
[i
].n2
;
5122 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5123 false, GSI_CONTINUE_LINKING
);
5124 tree v
= fd
->loops
[i
].v
;
5125 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5126 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5127 false, GSI_CONTINUE_LINKING
);
5128 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5129 stmt
= gimple_build_cond_empty (t
);
5130 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5131 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5132 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5135 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5143 /* A subroutine of expand_omp_for. Generate code for a parallel
5144 loop with any schedule. Given parameters:
5146 for (V = N1; V cond N2; V += STEP) BODY;
5148 where COND is "<" or ">", we generate pseudocode
5150 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5151 if (more) goto L0; else goto L3;
5158 if (V cond iend) goto L1; else goto L2;
5160 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5163 If this is a combined omp parallel loop, instead of the call to
5164 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5165 If this is gimple_omp_for_combined_p loop, then instead of assigning
5166 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5167 inner GIMPLE_OMP_FOR and V += STEP; and
5168 if (V cond iend) goto L1; else goto L2; are removed.
5170 For collapsed loops, given parameters:
5172 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5173 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5174 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5177 we generate pseudocode
5179 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5184 count3 = (adj + N32 - N31) / STEP3;
5185 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5190 count2 = (adj + N22 - N21) / STEP2;
5191 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5196 count1 = (adj + N12 - N11) / STEP1;
5197 count = count1 * count2 * count3;
5202 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5203 if (more) goto L0; else goto L3;
5207 V3 = N31 + (T % count3) * STEP3;
5209 V2 = N21 + (T % count2) * STEP2;
5211 V1 = N11 + T * STEP1;
5216 if (V < iend) goto L10; else goto L2;
5219 if (V3 cond3 N32) goto L1; else goto L11;
5223 if (V2 cond2 N22) goto L1; else goto L12;
5229 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5235 expand_omp_for_generic (struct omp_region
*region
,
5236 struct omp_for_data
*fd
,
5237 enum built_in_function start_fn
,
5238 enum built_in_function next_fn
,
5241 tree type
, istart0
, iend0
, iend
;
5242 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5243 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5244 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5245 gimple_stmt_iterator gsi
;
5247 bool in_combined_parallel
= is_combined_parallel (region
);
5248 bool broken_loop
= region
->cont
== NULL
;
5250 tree
*counts
= NULL
;
5253 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5254 gcc_assert (fd
->iter_type
== long_integer_type_node
5255 || !in_combined_parallel
);
5257 type
= TREE_TYPE (fd
->loop
.v
);
5258 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5259 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5260 TREE_ADDRESSABLE (istart0
) = 1;
5261 TREE_ADDRESSABLE (iend0
) = 1;
5263 /* See if we need to bias by LLONG_MIN. */
5264 if (fd
->iter_type
== long_long_unsigned_type_node
5265 && TREE_CODE (type
) == INTEGER_TYPE
5266 && !TYPE_UNSIGNED (type
))
5270 if (fd
->loop
.cond_code
== LT_EXPR
)
5273 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5277 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5280 if (TREE_CODE (n1
) != INTEGER_CST
5281 || TREE_CODE (n2
) != INTEGER_CST
5282 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5283 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5286 entry_bb
= region
->entry
;
5287 cont_bb
= region
->cont
;
5289 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5290 gcc_assert (broken_loop
5291 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5292 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5293 l1_bb
= single_succ (l0_bb
);
5296 l2_bb
= create_empty_bb (cont_bb
);
5297 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5298 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5302 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5303 exit_bb
= region
->exit
;
5305 gsi
= gsi_last_bb (entry_bb
);
5307 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5308 if (fd
->collapse
> 1)
5310 int first_zero_iter
= -1;
5311 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5313 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5314 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5315 zero_iter_bb
, first_zero_iter
,
5320 /* Some counts[i] vars might be uninitialized if
5321 some loop has zero iterations. But the body shouldn't
5322 be executed in that case, so just avoid uninit warnings. */
5323 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5324 if (SSA_VAR_P (counts
[i
]))
5325 TREE_NO_WARNING (counts
[i
]) = 1;
5327 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5329 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5330 gsi
= gsi_last_bb (entry_bb
);
5331 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5332 get_immediate_dominator (CDI_DOMINATORS
,
5336 if (in_combined_parallel
)
5338 /* In a combined parallel loop, emit a call to
5339 GOMP_loop_foo_next. */
5340 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5341 build_fold_addr_expr (istart0
),
5342 build_fold_addr_expr (iend0
));
5346 tree t0
, t1
, t2
, t3
, t4
;
5347 /* If this is not a combined parallel loop, emit a call to
5348 GOMP_loop_foo_start in ENTRY_BB. */
5349 t4
= build_fold_addr_expr (iend0
);
5350 t3
= build_fold_addr_expr (istart0
);
5351 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5354 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5356 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5357 OMP_CLAUSE__LOOPTEMP_
);
5358 gcc_assert (innerc
);
5359 t0
= OMP_CLAUSE_DECL (innerc
);
5360 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5361 OMP_CLAUSE__LOOPTEMP_
);
5362 gcc_assert (innerc
);
5363 t1
= OMP_CLAUSE_DECL (innerc
);
5365 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5366 && TYPE_PRECISION (TREE_TYPE (t0
))
5367 != TYPE_PRECISION (fd
->iter_type
))
5369 /* Avoid casting pointers to integer of a different size. */
5370 tree itype
= signed_type_for (type
);
5371 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5372 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5376 t1
= fold_convert (fd
->iter_type
, t1
);
5377 t0
= fold_convert (fd
->iter_type
, t0
);
5381 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5382 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5384 if (fd
->iter_type
== long_integer_type_node
)
5388 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5389 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5390 6, t0
, t1
, t2
, t
, t3
, t4
);
5393 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5394 5, t0
, t1
, t2
, t3
, t4
);
5402 /* The GOMP_loop_ull_*start functions have additional boolean
5403 argument, true for < loops and false for > loops.
5404 In Fortran, the C bool type can be different from
5405 boolean_type_node. */
5406 bfn_decl
= builtin_decl_explicit (start_fn
);
5407 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5408 t5
= build_int_cst (c_bool_type
,
5409 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5412 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5413 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5414 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5417 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5418 6, t5
, t0
, t1
, t2
, t3
, t4
);
5421 if (TREE_TYPE (t
) != boolean_type_node
)
5422 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5423 t
, build_int_cst (TREE_TYPE (t
), 0));
5424 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5425 true, GSI_SAME_STMT
);
5426 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5428 /* Remove the GIMPLE_OMP_FOR statement. */
5429 gsi_remove (&gsi
, true);
5431 /* Iteration setup for sequential loop goes in L0_BB. */
5432 tree startvar
= fd
->loop
.v
;
5433 tree endvar
= NULL_TREE
;
5435 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5437 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5438 && gimple_omp_for_kind (inner_stmt
)
5439 == GF_OMP_FOR_KIND_SIMD
);
5440 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5441 OMP_CLAUSE__LOOPTEMP_
);
5442 gcc_assert (innerc
);
5443 startvar
= OMP_CLAUSE_DECL (innerc
);
5444 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5445 OMP_CLAUSE__LOOPTEMP_
);
5446 gcc_assert (innerc
);
5447 endvar
= OMP_CLAUSE_DECL (innerc
);
5450 gsi
= gsi_start_bb (l0_bb
);
5453 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5454 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5455 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5456 t
= fold_convert (TREE_TYPE (startvar
), t
);
5457 t
= force_gimple_operand_gsi (&gsi
, t
,
5459 && TREE_ADDRESSABLE (startvar
),
5460 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5461 stmt
= gimple_build_assign (startvar
, t
);
5462 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5466 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5467 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5468 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5469 t
= fold_convert (TREE_TYPE (startvar
), t
);
5470 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5471 false, GSI_CONTINUE_LINKING
);
5474 stmt
= gimple_build_assign (endvar
, iend
);
5475 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5477 if (fd
->collapse
> 1)
5478 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5482 /* Code to control the increment and predicate for the sequential
5483 loop goes in the CONT_BB. */
5484 gsi
= gsi_last_bb (cont_bb
);
5485 stmt
= gsi_stmt (gsi
);
5486 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5487 vmain
= gimple_omp_continue_control_use (stmt
);
5488 vback
= gimple_omp_continue_control_def (stmt
);
5490 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5492 if (POINTER_TYPE_P (type
))
5493 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5495 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5496 t
= force_gimple_operand_gsi (&gsi
, t
,
5498 && TREE_ADDRESSABLE (vback
),
5499 NULL_TREE
, true, GSI_SAME_STMT
);
5500 stmt
= gimple_build_assign (vback
, t
);
5501 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5503 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5504 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5506 stmt
= gimple_build_cond_empty (t
);
5507 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5510 /* Remove GIMPLE_OMP_CONTINUE. */
5511 gsi_remove (&gsi
, true);
5513 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5514 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5516 /* Emit code to get the next parallel iteration in L2_BB. */
5517 gsi
= gsi_start_bb (l2_bb
);
5519 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5520 build_fold_addr_expr (istart0
),
5521 build_fold_addr_expr (iend0
));
5522 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5523 false, GSI_CONTINUE_LINKING
);
5524 if (TREE_TYPE (t
) != boolean_type_node
)
5525 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5526 t
, build_int_cst (TREE_TYPE (t
), 0));
5527 stmt
= gimple_build_cond_empty (t
);
5528 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5531 /* Add the loop cleanup function. */
5532 gsi
= gsi_last_bb (exit_bb
);
5533 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5534 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5535 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5536 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5538 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5539 stmt
= gimple_build_call (t
, 0);
5540 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5541 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5542 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5543 gsi_remove (&gsi
, true);
5545 /* Connect the new blocks. */
5546 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5547 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5553 e
= find_edge (cont_bb
, l3_bb
);
5554 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5556 phis
= phi_nodes (l3_bb
);
5557 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5559 gimple phi
= gsi_stmt (gsi
);
5560 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5561 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5565 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5567 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5568 e
= find_edge (cont_bb
, l1_bb
);
5569 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5574 else if (fd
->collapse
> 1)
5577 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5580 e
->flags
= EDGE_TRUE_VALUE
;
5583 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5584 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5588 e
= find_edge (cont_bb
, l2_bb
);
5589 e
->flags
= EDGE_FALLTHRU
;
5591 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5593 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5594 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5595 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5596 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5597 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5598 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5599 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5600 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5602 struct loop
*outer_loop
= alloc_loop ();
5603 outer_loop
->header
= l0_bb
;
5604 outer_loop
->latch
= l2_bb
;
5605 add_loop (outer_loop
, l0_bb
->loop_father
);
5607 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5609 struct loop
*loop
= alloc_loop ();
5610 loop
->header
= l1_bb
;
5611 /* The loop may have multiple latches. */
5612 add_loop (loop
, outer_loop
);
5618 /* A subroutine of expand_omp_for. Generate code for a parallel
5619 loop with static schedule and no specified chunk size. Given
5622 for (V = N1; V cond N2; V += STEP) BODY;
5624 where COND is "<" or ">", we generate pseudocode
5626 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5631 if ((__typeof (V)) -1 > 0 && cond is >)
5632 n = -(adj + N2 - N1) / -STEP;
5634 n = (adj + N2 - N1) / STEP;
5637 if (threadid < tt) goto L3; else goto L4;
5642 s0 = q * threadid + tt;
5645 if (s0 >= e0) goto L2; else goto L0;
5651 if (V cond e) goto L1;
5656 expand_omp_for_static_nochunk (struct omp_region
*region
,
5657 struct omp_for_data
*fd
,
5660 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5661 tree type
, itype
, vmain
, vback
;
5662 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5663 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5665 gimple_stmt_iterator gsi
;
5668 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5669 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5670 bool broken_loop
= region
->cont
== NULL
;
5671 tree
*counts
= NULL
;
5674 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5675 if (POINTER_TYPE_P (type
))
5676 itype
= signed_type_for (type
);
5678 entry_bb
= region
->entry
;
5679 cont_bb
= region
->cont
;
5680 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5681 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5682 gcc_assert (broken_loop
5683 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5684 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5685 body_bb
= single_succ (seq_start_bb
);
5688 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5689 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5691 exit_bb
= region
->exit
;
5693 /* Iteration space partitioning goes in ENTRY_BB. */
5694 gsi
= gsi_last_bb (entry_bb
);
5695 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5697 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5699 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5700 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5703 if (fd
->collapse
> 1)
5705 int first_zero_iter
= -1;
5706 basic_block l2_dom_bb
= NULL
;
5708 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5709 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5710 fin_bb
, first_zero_iter
,
5714 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5715 t
= integer_one_node
;
5717 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5718 fold_convert (type
, fd
->loop
.n1
),
5719 fold_convert (type
, fd
->loop
.n2
));
5720 if (fd
->collapse
== 1
5721 && TYPE_UNSIGNED (type
)
5722 && (t
== NULL_TREE
|| !integer_onep (t
)))
5724 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5725 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5726 true, GSI_SAME_STMT
);
5727 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5728 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5729 true, GSI_SAME_STMT
);
5730 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5731 NULL_TREE
, NULL_TREE
);
5732 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5733 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5734 expand_omp_regimplify_p
, NULL
, NULL
)
5735 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5736 expand_omp_regimplify_p
, NULL
, NULL
))
5738 gsi
= gsi_for_stmt (stmt
);
5739 gimple_regimplify_operands (stmt
, &gsi
);
5741 ep
= split_block (entry_bb
, stmt
);
5742 ep
->flags
= EDGE_TRUE_VALUE
;
5743 entry_bb
= ep
->dest
;
5744 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5745 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5746 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5747 if (gimple_in_ssa_p (cfun
))
5749 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5750 for (gsi
= gsi_start_phis (fin_bb
);
5751 !gsi_end_p (gsi
); gsi_next (&gsi
))
5753 gimple phi
= gsi_stmt (gsi
);
5754 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5755 ep
, UNKNOWN_LOCATION
);
5758 gsi
= gsi_last_bb (entry_bb
);
5761 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5762 t
= fold_convert (itype
, t
);
5763 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5764 true, GSI_SAME_STMT
);
5766 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5767 t
= fold_convert (itype
, t
);
5768 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5769 true, GSI_SAME_STMT
);
5773 step
= fd
->loop
.step
;
5774 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5776 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5777 OMP_CLAUSE__LOOPTEMP_
);
5778 gcc_assert (innerc
);
5779 n1
= OMP_CLAUSE_DECL (innerc
);
5780 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5781 OMP_CLAUSE__LOOPTEMP_
);
5782 gcc_assert (innerc
);
5783 n2
= OMP_CLAUSE_DECL (innerc
);
5785 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5786 true, NULL_TREE
, true, GSI_SAME_STMT
);
5787 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5788 true, NULL_TREE
, true, GSI_SAME_STMT
);
5789 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5790 true, NULL_TREE
, true, GSI_SAME_STMT
);
5792 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5793 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5794 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5795 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5796 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5797 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5798 fold_build1 (NEGATE_EXPR
, itype
, t
),
5799 fold_build1 (NEGATE_EXPR
, itype
, step
));
5801 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5802 t
= fold_convert (itype
, t
);
5803 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5805 q
= create_tmp_reg (itype
, "q");
5806 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5807 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5808 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5810 tt
= create_tmp_reg (itype
, "tt");
5811 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5812 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5813 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5815 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5816 stmt
= gimple_build_cond_empty (t
);
5817 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5819 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5820 gsi
= gsi_last_bb (second_bb
);
5821 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5823 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5825 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5826 build_int_cst (itype
, 1));
5827 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5829 third_bb
= split_block (second_bb
, stmt
)->dest
;
5830 gsi
= gsi_last_bb (third_bb
);
5831 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5833 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
5834 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
5835 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5837 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
5838 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5840 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
5841 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5843 /* Remove the GIMPLE_OMP_FOR statement. */
5844 gsi_remove (&gsi
, true);
5846 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5847 gsi
= gsi_start_bb (seq_start_bb
);
5849 tree startvar
= fd
->loop
.v
;
5850 tree endvar
= NULL_TREE
;
5852 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5854 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5855 ? gimple_omp_parallel_clauses (inner_stmt
)
5856 : gimple_omp_for_clauses (inner_stmt
);
5857 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5858 gcc_assert (innerc
);
5859 startvar
= OMP_CLAUSE_DECL (innerc
);
5860 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5861 OMP_CLAUSE__LOOPTEMP_
);
5862 gcc_assert (innerc
);
5863 endvar
= OMP_CLAUSE_DECL (innerc
);
5865 t
= fold_convert (itype
, s0
);
5866 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5867 if (POINTER_TYPE_P (type
))
5868 t
= fold_build_pointer_plus (n1
, t
);
5870 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5871 t
= fold_convert (TREE_TYPE (startvar
), t
);
5872 t
= force_gimple_operand_gsi (&gsi
, t
,
5874 && TREE_ADDRESSABLE (startvar
),
5875 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5876 stmt
= gimple_build_assign (startvar
, t
);
5877 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5879 t
= fold_convert (itype
, e0
);
5880 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5881 if (POINTER_TYPE_P (type
))
5882 t
= fold_build_pointer_plus (n1
, t
);
5884 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5885 t
= fold_convert (TREE_TYPE (startvar
), t
);
5886 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5887 false, GSI_CONTINUE_LINKING
);
5890 stmt
= gimple_build_assign (endvar
, e
);
5891 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5893 if (fd
->collapse
> 1)
5894 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5898 /* The code controlling the sequential loop replaces the
5899 GIMPLE_OMP_CONTINUE. */
5900 gsi
= gsi_last_bb (cont_bb
);
5901 stmt
= gsi_stmt (gsi
);
5902 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5903 vmain
= gimple_omp_continue_control_use (stmt
);
5904 vback
= gimple_omp_continue_control_def (stmt
);
5906 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5908 if (POINTER_TYPE_P (type
))
5909 t
= fold_build_pointer_plus (vmain
, step
);
5911 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
5912 t
= force_gimple_operand_gsi (&gsi
, t
,
5914 && TREE_ADDRESSABLE (vback
),
5915 NULL_TREE
, true, GSI_SAME_STMT
);
5916 stmt
= gimple_build_assign (vback
, t
);
5917 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5919 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5920 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
5922 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5925 /* Remove the GIMPLE_OMP_CONTINUE statement. */
5926 gsi_remove (&gsi
, true);
5928 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5929 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
5932 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5933 gsi
= gsi_last_bb (exit_bb
);
5934 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5936 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
5937 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
5939 gsi_remove (&gsi
, true);
5941 /* Connect all the blocks. */
5942 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
5943 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
5944 ep
= find_edge (entry_bb
, second_bb
);
5945 ep
->flags
= EDGE_TRUE_VALUE
;
5946 ep
->probability
= REG_BR_PROB_BASE
/ 4;
5947 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
5948 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
5952 ep
= find_edge (cont_bb
, body_bb
);
5953 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5958 else if (fd
->collapse
> 1)
5961 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5964 ep
->flags
= EDGE_TRUE_VALUE
;
5965 find_edge (cont_bb
, fin_bb
)->flags
5966 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
5969 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
5970 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
5971 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
5973 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
5974 recompute_dominator (CDI_DOMINATORS
, body_bb
));
5975 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
5976 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
5978 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
5980 struct loop
*loop
= alloc_loop ();
5981 loop
->header
= body_bb
;
5982 if (collapse_bb
== NULL
)
5983 loop
->latch
= cont_bb
;
5984 add_loop (loop
, body_bb
->loop_father
);
5989 /* A subroutine of expand_omp_for. Generate code for a parallel
5990 loop with static schedule and a specified chunk size. Given
5993 for (V = N1; V cond N2; V += STEP) BODY;
5995 where COND is "<" or ">", we generate pseudocode
5997 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6002 if ((__typeof (V)) -1 > 0 && cond is >)
6003 n = -(adj + N2 - N1) / -STEP;
6005 n = (adj + N2 - N1) / STEP;
6007 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6008 here so that V is defined
6009 if the loop is not entered
6011 s0 = (trip * nthreads + threadid) * CHUNK;
6012 e0 = min(s0 + CHUNK, n);
6013 if (s0 < n) goto L1; else goto L4;
6020 if (V cond e) goto L2; else goto L3;
6028 expand_omp_for_static_chunk (struct omp_region
*region
,
6029 struct omp_for_data
*fd
, gimple inner_stmt
)
6031 tree n
, s0
, e0
, e
, t
;
6032 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6033 tree type
, itype
, v_main
, v_back
, v_extra
;
6034 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6035 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6036 gimple_stmt_iterator si
;
6039 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6040 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6041 bool broken_loop
= region
->cont
== NULL
;
6042 tree
*counts
= NULL
;
6045 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6046 if (POINTER_TYPE_P (type
))
6047 itype
= signed_type_for (type
);
6049 entry_bb
= region
->entry
;
6050 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6052 iter_part_bb
= se
->dest
;
6053 cont_bb
= region
->cont
;
6054 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6055 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6056 gcc_assert (broken_loop
6057 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6058 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6059 body_bb
= single_succ (seq_start_bb
);
6062 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6063 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6064 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6066 exit_bb
= region
->exit
;
6068 /* Trip and adjustment setup goes in ENTRY_BB. */
6069 si
= gsi_last_bb (entry_bb
);
6070 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6072 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6074 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6075 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6078 if (fd
->collapse
> 1)
6080 int first_zero_iter
= -1;
6081 basic_block l2_dom_bb
= NULL
;
6083 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6084 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6085 fin_bb
, first_zero_iter
,
6089 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6090 t
= integer_one_node
;
6092 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6093 fold_convert (type
, fd
->loop
.n1
),
6094 fold_convert (type
, fd
->loop
.n2
));
6095 if (fd
->collapse
== 1
6096 && TYPE_UNSIGNED (type
)
6097 && (t
== NULL_TREE
|| !integer_onep (t
)))
6099 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6100 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6101 true, GSI_SAME_STMT
);
6102 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6103 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6104 true, GSI_SAME_STMT
);
6105 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6106 NULL_TREE
, NULL_TREE
);
6107 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6108 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6109 expand_omp_regimplify_p
, NULL
, NULL
)
6110 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6111 expand_omp_regimplify_p
, NULL
, NULL
))
6113 si
= gsi_for_stmt (stmt
);
6114 gimple_regimplify_operands (stmt
, &si
);
6116 se
= split_block (entry_bb
, stmt
);
6117 se
->flags
= EDGE_TRUE_VALUE
;
6118 entry_bb
= se
->dest
;
6119 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6120 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6121 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6122 if (gimple_in_ssa_p (cfun
))
6124 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6125 for (si
= gsi_start_phis (fin_bb
);
6126 !gsi_end_p (si
); gsi_next (&si
))
6128 gimple phi
= gsi_stmt (si
);
6129 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6130 se
, UNKNOWN_LOCATION
);
6133 si
= gsi_last_bb (entry_bb
);
6136 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6137 t
= fold_convert (itype
, t
);
6138 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6139 true, GSI_SAME_STMT
);
6141 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6142 t
= fold_convert (itype
, t
);
6143 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6144 true, GSI_SAME_STMT
);
6148 step
= fd
->loop
.step
;
6149 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6151 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6152 OMP_CLAUSE__LOOPTEMP_
);
6153 gcc_assert (innerc
);
6154 n1
= OMP_CLAUSE_DECL (innerc
);
6155 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6156 OMP_CLAUSE__LOOPTEMP_
);
6157 gcc_assert (innerc
);
6158 n2
= OMP_CLAUSE_DECL (innerc
);
6160 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6161 true, NULL_TREE
, true, GSI_SAME_STMT
);
6162 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6163 true, NULL_TREE
, true, GSI_SAME_STMT
);
6164 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6165 true, NULL_TREE
, true, GSI_SAME_STMT
);
6167 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6168 true, NULL_TREE
, true, GSI_SAME_STMT
);
6170 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6171 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6172 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6173 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6174 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6175 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6176 fold_build1 (NEGATE_EXPR
, itype
, t
),
6177 fold_build1 (NEGATE_EXPR
, itype
, step
));
6179 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6180 t
= fold_convert (itype
, t
);
6181 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6182 true, GSI_SAME_STMT
);
6184 trip_var
= create_tmp_reg (itype
, ".trip");
6185 if (gimple_in_ssa_p (cfun
))
6187 trip_init
= make_ssa_name (trip_var
, NULL
);
6188 trip_main
= make_ssa_name (trip_var
, NULL
);
6189 trip_back
= make_ssa_name (trip_var
, NULL
);
6193 trip_init
= trip_var
;
6194 trip_main
= trip_var
;
6195 trip_back
= trip_var
;
6198 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6199 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6201 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6202 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6203 if (POINTER_TYPE_P (type
))
6204 t
= fold_build_pointer_plus (n1
, t
);
6206 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6207 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6208 true, GSI_SAME_STMT
);
6210 /* Remove the GIMPLE_OMP_FOR. */
6211 gsi_remove (&si
, true);
6213 /* Iteration space partitioning goes in ITER_PART_BB. */
6214 si
= gsi_last_bb (iter_part_bb
);
6216 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6217 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6218 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6219 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6220 false, GSI_CONTINUE_LINKING
);
6222 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6223 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6224 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6225 false, GSI_CONTINUE_LINKING
);
6227 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6228 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6230 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6231 si
= gsi_start_bb (seq_start_bb
);
6233 tree startvar
= fd
->loop
.v
;
6234 tree endvar
= NULL_TREE
;
6236 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6238 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6239 ? gimple_omp_parallel_clauses (inner_stmt
)
6240 : gimple_omp_for_clauses (inner_stmt
);
6241 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6242 gcc_assert (innerc
);
6243 startvar
= OMP_CLAUSE_DECL (innerc
);
6244 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6245 OMP_CLAUSE__LOOPTEMP_
);
6246 gcc_assert (innerc
);
6247 endvar
= OMP_CLAUSE_DECL (innerc
);
6250 t
= fold_convert (itype
, s0
);
6251 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6252 if (POINTER_TYPE_P (type
))
6253 t
= fold_build_pointer_plus (n1
, t
);
6255 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6256 t
= fold_convert (TREE_TYPE (startvar
), t
);
6257 t
= force_gimple_operand_gsi (&si
, t
,
6259 && TREE_ADDRESSABLE (startvar
),
6260 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6261 stmt
= gimple_build_assign (startvar
, t
);
6262 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6264 t
= fold_convert (itype
, e0
);
6265 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6266 if (POINTER_TYPE_P (type
))
6267 t
= fold_build_pointer_plus (n1
, t
);
6269 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6270 t
= fold_convert (TREE_TYPE (startvar
), t
);
6271 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6272 false, GSI_CONTINUE_LINKING
);
6275 stmt
= gimple_build_assign (endvar
, e
);
6276 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6278 if (fd
->collapse
> 1)
6279 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6283 /* The code controlling the sequential loop goes in CONT_BB,
6284 replacing the GIMPLE_OMP_CONTINUE. */
6285 si
= gsi_last_bb (cont_bb
);
6286 stmt
= gsi_stmt (si
);
6287 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6288 v_main
= gimple_omp_continue_control_use (stmt
);
6289 v_back
= gimple_omp_continue_control_def (stmt
);
6291 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6293 if (POINTER_TYPE_P (type
))
6294 t
= fold_build_pointer_plus (v_main
, step
);
6296 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6297 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6298 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6299 true, GSI_SAME_STMT
);
6300 stmt
= gimple_build_assign (v_back
, t
);
6301 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6303 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6304 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6306 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6309 /* Remove GIMPLE_OMP_CONTINUE. */
6310 gsi_remove (&si
, true);
6312 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6313 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6315 /* Trip update code goes into TRIP_UPDATE_BB. */
6316 si
= gsi_start_bb (trip_update_bb
);
6318 t
= build_int_cst (itype
, 1);
6319 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6320 stmt
= gimple_build_assign (trip_back
, t
);
6321 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6324 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6325 si
= gsi_last_bb (exit_bb
);
6326 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6328 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6329 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6331 gsi_remove (&si
, true);
6333 /* Connect the new blocks. */
6334 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6335 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6339 se
= find_edge (cont_bb
, body_bb
);
6340 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6345 else if (fd
->collapse
> 1)
6348 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6351 se
->flags
= EDGE_TRUE_VALUE
;
6352 find_edge (cont_bb
, trip_update_bb
)->flags
6353 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6355 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6358 if (gimple_in_ssa_p (cfun
))
6360 gimple_stmt_iterator psi
;
6363 edge_var_map_vector
*head
;
6367 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6369 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6370 remove arguments of the phi nodes in fin_bb. We need to create
6371 appropriate phi nodes in iter_part_bb instead. */
6372 se
= single_pred_edge (fin_bb
);
6373 re
= single_succ_edge (trip_update_bb
);
6374 head
= redirect_edge_var_map_vector (re
);
6375 ene
= single_succ_edge (entry_bb
);
6377 psi
= gsi_start_phis (fin_bb
);
6378 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6379 gsi_next (&psi
), ++i
)
6382 source_location locus
;
6384 phi
= gsi_stmt (psi
);
6385 t
= gimple_phi_result (phi
);
6386 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6387 nphi
= create_phi_node (t
, iter_part_bb
);
6389 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6390 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6392 /* A special case -- fd->loop.v is not yet computed in
6393 iter_part_bb, we need to use v_extra instead. */
6394 if (t
== fd
->loop
.v
)
6396 add_phi_arg (nphi
, t
, ene
, locus
);
6397 locus
= redirect_edge_var_map_location (vm
);
6398 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6400 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6401 redirect_edge_var_map_clear (re
);
6404 psi
= gsi_start_phis (fin_bb
);
6405 if (gsi_end_p (psi
))
6407 remove_phi_node (&psi
, false);
6410 /* Make phi node for trip. */
6411 phi
= create_phi_node (trip_main
, iter_part_bb
);
6412 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6414 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6419 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6420 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6421 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6422 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6423 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6424 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6425 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6426 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6427 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6431 struct loop
*trip_loop
= alloc_loop ();
6432 trip_loop
->header
= iter_part_bb
;
6433 trip_loop
->latch
= trip_update_bb
;
6434 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6436 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6438 struct loop
*loop
= alloc_loop ();
6439 loop
->header
= body_bb
;
6440 loop
->latch
= cont_bb
;
6441 add_loop (loop
, trip_loop
);
6447 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6448 loop. Given parameters:
6450 for (V = N1; V cond N2; V += STEP) BODY;
6452 where COND is "<" or ">", we generate pseudocode
6460 if (V cond N2) goto L0; else goto L2;
6463 For collapsed loops, given parameters:
6465 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6466 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6467 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6470 we generate pseudocode
6476 count3 = (adj + N32 - N31) / STEP3;
6481 count2 = (adj + N22 - N21) / STEP2;
6486 count1 = (adj + N12 - N11) / STEP1;
6487 count = count1 * count2 * count3;
6497 V2 += (V3 cond3 N32) ? 0 : STEP2;
6498 V3 = (V3 cond3 N32) ? V3 : N31;
6499 V1 += (V2 cond2 N22) ? 0 : STEP1;
6500 V2 = (V2 cond2 N22) ? V2 : N21;
6502 if (V < count) goto L0; else goto L2;
6508 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6511 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6512 gimple_stmt_iterator gsi
;
6514 bool broken_loop
= region
->cont
== NULL
;
6516 tree
*counts
= NULL
;
6518 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6519 OMP_CLAUSE_SAFELEN
);
6520 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6521 OMP_CLAUSE__SIMDUID_
);
6524 type
= TREE_TYPE (fd
->loop
.v
);
6525 entry_bb
= region
->entry
;
6526 cont_bb
= region
->cont
;
6527 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6528 gcc_assert (broken_loop
6529 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6530 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6533 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6534 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6535 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6536 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6540 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6541 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6542 l2_bb
= single_succ (l1_bb
);
6544 exit_bb
= region
->exit
;
6547 gsi
= gsi_last_bb (entry_bb
);
6549 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6550 /* Not needed in SSA form right now. */
6551 gcc_assert (!gimple_in_ssa_p (cfun
));
6552 if (fd
->collapse
> 1)
6554 int first_zero_iter
= -1;
6555 basic_block zero_iter_bb
= l2_bb
;
6557 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6558 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6559 zero_iter_bb
, first_zero_iter
,
6562 if (l2_dom_bb
== NULL
)
6567 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6569 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6570 OMP_CLAUSE__LOOPTEMP_
);
6571 gcc_assert (innerc
);
6572 n1
= OMP_CLAUSE_DECL (innerc
);
6573 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6574 OMP_CLAUSE__LOOPTEMP_
);
6575 gcc_assert (innerc
);
6576 n2
= OMP_CLAUSE_DECL (innerc
);
6577 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6578 fold_convert (type
, n1
));
6579 if (fd
->collapse
> 1)
6582 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6588 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6589 fold_convert (type
, fd
->loop
.n1
));
6590 if (fd
->collapse
> 1)
6591 for (i
= 0; i
< fd
->collapse
; i
++)
6593 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6594 if (POINTER_TYPE_P (itype
))
6595 itype
= signed_type_for (itype
);
6596 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6597 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6601 /* Remove the GIMPLE_OMP_FOR statement. */
6602 gsi_remove (&gsi
, true);
6606 /* Code to control the increment goes in the CONT_BB. */
6607 gsi
= gsi_last_bb (cont_bb
);
6608 stmt
= gsi_stmt (gsi
);
6609 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6611 if (POINTER_TYPE_P (type
))
6612 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6614 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6615 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6617 if (fd
->collapse
> 1)
6619 i
= fd
->collapse
- 1;
6620 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6622 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6623 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6627 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6629 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6632 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6634 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6636 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6637 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6638 if (POINTER_TYPE_P (itype2
))
6639 itype2
= signed_type_for (itype2
);
6640 t
= build3 (COND_EXPR
, itype2
,
6641 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6643 fold_convert (itype
, fd
->loops
[i
].n2
)),
6644 build_int_cst (itype2
, 0),
6645 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6646 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6647 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6649 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6650 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6652 t
= build3 (COND_EXPR
, itype
,
6653 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6655 fold_convert (itype
, fd
->loops
[i
].n2
)),
6657 fold_convert (itype
, fd
->loops
[i
].n1
));
6658 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6662 /* Remove GIMPLE_OMP_CONTINUE. */
6663 gsi_remove (&gsi
, true);
6666 /* Emit the condition in L1_BB. */
6667 gsi
= gsi_start_bb (l1_bb
);
6669 t
= fold_convert (type
, n2
);
6670 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6671 false, GSI_CONTINUE_LINKING
);
6672 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6673 stmt
= gimple_build_cond_empty (t
);
6674 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6675 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6677 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6680 gsi
= gsi_for_stmt (stmt
);
6681 gimple_regimplify_operands (stmt
, &gsi
);
6684 /* Remove GIMPLE_OMP_RETURN. */
6685 gsi
= gsi_last_bb (exit_bb
);
6686 gsi_remove (&gsi
, true);
6688 /* Connect the new blocks. */
6689 remove_edge (FALLTHRU_EDGE (entry_bb
));
6693 remove_edge (BRANCH_EDGE (entry_bb
));
6694 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6696 e
= BRANCH_EDGE (l1_bb
);
6697 ne
= FALLTHRU_EDGE (l1_bb
);
6698 e
->flags
= EDGE_TRUE_VALUE
;
6702 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6704 ne
= single_succ_edge (l1_bb
);
6705 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6708 ne
->flags
= EDGE_FALSE_VALUE
;
6709 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6710 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6712 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6713 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6714 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6718 struct loop
*loop
= alloc_loop ();
6719 loop
->header
= l1_bb
;
6720 loop
->latch
= e
->dest
;
6721 add_loop (loop
, l1_bb
->loop_father
);
6722 if (safelen
== NULL_TREE
)
6723 loop
->safelen
= INT_MAX
;
6726 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6727 if (!host_integerp (safelen
, 1)
6728 || (unsigned HOST_WIDE_INT
) tree_low_cst (safelen
, 1)
6730 loop
->safelen
= INT_MAX
;
6732 loop
->safelen
= tree_low_cst (safelen
, 1);
6733 if (loop
->safelen
== 1)
6738 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6739 cfun
->has_simduid_loops
= true;
6741 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6743 if ((flag_tree_loop_vectorize
6744 || (!global_options_set
.x_flag_tree_loop_vectorize
6745 && !global_options_set
.x_flag_tree_vectorize
))
6746 && loop
->safelen
> 1)
6748 loop
->force_vect
= true;
6749 cfun
->has_force_vect_loops
= true;
6755 /* Expand the OpenMP loop defined by REGION. */
6758 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6760 struct omp_for_data fd
;
6761 struct omp_for_data_loop
*loops
;
6764 = (struct omp_for_data_loop
*)
6765 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6766 * sizeof (struct omp_for_data_loop
));
6767 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6768 region
->sched_kind
= fd
.sched_kind
;
6770 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6771 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6772 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6775 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6776 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6777 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6780 /* If there isn't a continue then this is a degerate case where
6781 the introduction of abnormal edges during lowering will prevent
6782 original loops from being detected. Fix that up. */
6783 loops_state_set (LOOPS_NEED_FIXUP
);
6785 if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_SIMD
)
6786 expand_omp_simd (region
, &fd
);
6787 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6788 && !fd
.have_ordered
)
6790 if (fd
.chunk_size
== NULL
)
6791 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6793 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6797 int fn_index
, start_ix
, next_ix
;
6799 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6800 == GF_OMP_FOR_KIND_FOR
);
6801 if (fd
.chunk_size
== NULL
6802 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6803 fd
.chunk_size
= integer_zero_node
;
6804 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6805 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6806 ? 3 : fd
.sched_kind
;
6807 fn_index
+= fd
.have_ordered
* 4;
6808 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6809 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6810 if (fd
.iter_type
== long_long_unsigned_type_node
)
6812 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6813 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6814 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6815 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6817 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6818 (enum built_in_function
) next_ix
, inner_stmt
);
6821 if (gimple_in_ssa_p (cfun
))
6822 update_ssa (TODO_update_ssa_only_virtuals
);
6826 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6828 v = GOMP_sections_start (n);
6845 v = GOMP_sections_next ();
6850 If this is a combined parallel sections, replace the call to
6851 GOMP_sections_start with call to GOMP_sections_next. */
6854 expand_omp_sections (struct omp_region
*region
)
6856 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
6857 vec
<tree
> label_vec
;
6859 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
6860 gimple_stmt_iterator si
, switch_si
;
6861 gimple sections_stmt
, stmt
, cont
;
6864 struct omp_region
*inner
;
6866 bool exit_reachable
= region
->cont
!= NULL
;
6868 gcc_assert (region
->exit
!= NULL
);
6869 entry_bb
= region
->entry
;
6870 l0_bb
= single_succ (entry_bb
);
6871 l1_bb
= region
->cont
;
6872 l2_bb
= region
->exit
;
6873 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
6874 l2
= gimple_block_label (l2_bb
);
6877 /* This can happen if there are reductions. */
6878 len
= EDGE_COUNT (l0_bb
->succs
);
6879 gcc_assert (len
> 0);
6880 e
= EDGE_SUCC (l0_bb
, len
- 1);
6881 si
= gsi_last_bb (e
->dest
);
6884 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6885 l2
= gimple_block_label (e
->dest
);
6887 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
6889 si
= gsi_last_bb (e
->dest
);
6891 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6893 l2
= gimple_block_label (e
->dest
);
6899 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
6901 default_bb
= create_empty_bb (l0_bb
);
6903 /* We will build a switch() with enough cases for all the
6904 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
6905 and a default case to abort if something goes wrong. */
6906 len
= EDGE_COUNT (l0_bb
->succs
);
6908 /* Use vec::quick_push on label_vec throughout, since we know the size
6910 label_vec
.create (len
);
6912 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
6913 GIMPLE_OMP_SECTIONS statement. */
6914 si
= gsi_last_bb (entry_bb
);
6915 sections_stmt
= gsi_stmt (si
);
6916 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
6917 vin
= gimple_omp_sections_control (sections_stmt
);
6918 if (!is_combined_parallel (region
))
6920 /* If we are not inside a combined parallel+sections region,
6921 call GOMP_sections_start. */
6922 t
= build_int_cst (unsigned_type_node
, len
- 1);
6923 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
6924 stmt
= gimple_build_call (u
, 1, t
);
6928 /* Otherwise, call GOMP_sections_next. */
6929 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
6930 stmt
= gimple_build_call (u
, 0);
6932 gimple_call_set_lhs (stmt
, vin
);
6933 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
6934 gsi_remove (&si
, true);
6936 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
6938 switch_si
= gsi_last_bb (l0_bb
);
6939 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
6942 cont
= last_stmt (l1_bb
);
6943 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
6944 vmain
= gimple_omp_continue_control_use (cont
);
6945 vnext
= gimple_omp_continue_control_def (cont
);
6953 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
6954 label_vec
.quick_push (t
);
6957 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
6958 for (inner
= region
->inner
, casei
= 1;
6960 inner
= inner
->next
, i
++, casei
++)
6962 basic_block s_entry_bb
, s_exit_bb
;
6964 /* Skip optional reduction region. */
6965 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
6972 s_entry_bb
= inner
->entry
;
6973 s_exit_bb
= inner
->exit
;
6975 t
= gimple_block_label (s_entry_bb
);
6976 u
= build_int_cst (unsigned_type_node
, casei
);
6977 u
= build_case_label (u
, NULL
, t
);
6978 label_vec
.quick_push (u
);
6980 si
= gsi_last_bb (s_entry_bb
);
6981 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
6982 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
6983 gsi_remove (&si
, true);
6984 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
6986 if (s_exit_bb
== NULL
)
6989 si
= gsi_last_bb (s_exit_bb
);
6990 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
6991 gsi_remove (&si
, true);
6993 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
6996 /* Error handling code goes in DEFAULT_BB. */
6997 t
= gimple_block_label (default_bb
);
6998 u
= build_case_label (NULL
, NULL
, t
);
6999 make_edge (l0_bb
, default_bb
, 0);
7001 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7003 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7004 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7005 gsi_remove (&switch_si
, true);
7006 label_vec
.release ();
7008 si
= gsi_start_bb (default_bb
);
7009 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7010 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7016 /* Code to get the next section goes in L1_BB. */
7017 si
= gsi_last_bb (l1_bb
);
7018 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7020 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7021 stmt
= gimple_build_call (bfn_decl
, 0);
7022 gimple_call_set_lhs (stmt
, vnext
);
7023 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7024 gsi_remove (&si
, true);
7026 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7029 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7030 si
= gsi_last_bb (l2_bb
);
7031 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7032 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7033 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7034 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7036 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7037 stmt
= gimple_build_call (t
, 0);
7038 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7039 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7040 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7041 gsi_remove (&si
, true);
7043 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7047 /* Expand code for an OpenMP single directive. We've already expanded
7048 much of the code, here we simply place the GOMP_barrier call. */
7051 expand_omp_single (struct omp_region
*region
)
7053 basic_block entry_bb
, exit_bb
;
7054 gimple_stmt_iterator si
;
7056 entry_bb
= region
->entry
;
7057 exit_bb
= region
->exit
;
7059 si
= gsi_last_bb (entry_bb
);
7060 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7061 gsi_remove (&si
, true);
7062 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7064 si
= gsi_last_bb (exit_bb
);
7065 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7067 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7068 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7070 gsi_remove (&si
, true);
7071 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7075 /* Generic expansion for OpenMP synchronization directives: master,
7076 ordered and critical. All we need to do here is remove the entry
7077 and exit markers for REGION. */
7080 expand_omp_synch (struct omp_region
*region
)
7082 basic_block entry_bb
, exit_bb
;
7083 gimple_stmt_iterator si
;
7085 entry_bb
= region
->entry
;
7086 exit_bb
= region
->exit
;
7088 si
= gsi_last_bb (entry_bb
);
7089 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7090 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7091 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7092 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7093 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7094 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7095 gsi_remove (&si
, true);
7096 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7100 si
= gsi_last_bb (exit_bb
);
7101 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7102 gsi_remove (&si
, true);
7103 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7107 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7108 operation as a normal volatile load. */
7111 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7112 tree loaded_val
, int index
)
7114 enum built_in_function tmpbase
;
7115 gimple_stmt_iterator gsi
;
7116 basic_block store_bb
;
7119 tree decl
, call
, type
, itype
;
7121 gsi
= gsi_last_bb (load_bb
);
7122 stmt
= gsi_stmt (gsi
);
7123 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7124 loc
= gimple_location (stmt
);
7126 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7127 is smaller than word size, then expand_atomic_load assumes that the load
7128 is atomic. We could avoid the builtin entirely in this case. */
7130 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7131 decl
= builtin_decl_explicit (tmpbase
);
7132 if (decl
== NULL_TREE
)
7135 type
= TREE_TYPE (loaded_val
);
7136 itype
= TREE_TYPE (TREE_TYPE (decl
));
7138 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7139 build_int_cst (NULL
,
7140 gimple_omp_atomic_seq_cst_p (stmt
)
7142 : MEMMODEL_RELAXED
));
7143 if (!useless_type_conversion_p (type
, itype
))
7144 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7145 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7147 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7148 gsi_remove (&gsi
, true);
7150 store_bb
= single_succ (load_bb
);
7151 gsi
= gsi_last_bb (store_bb
);
7152 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7153 gsi_remove (&gsi
, true);
7155 if (gimple_in_ssa_p (cfun
))
7156 update_ssa (TODO_update_ssa_no_phi
);
7161 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7162 operation as a normal volatile store. */
7165 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7166 tree loaded_val
, tree stored_val
, int index
)
7168 enum built_in_function tmpbase
;
7169 gimple_stmt_iterator gsi
;
7170 basic_block store_bb
= single_succ (load_bb
);
7173 tree decl
, call
, type
, itype
;
7174 enum machine_mode imode
;
7177 gsi
= gsi_last_bb (load_bb
);
7178 stmt
= gsi_stmt (gsi
);
7179 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7181 /* If the load value is needed, then this isn't a store but an exchange. */
7182 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7184 gsi
= gsi_last_bb (store_bb
);
7185 stmt
= gsi_stmt (gsi
);
7186 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7187 loc
= gimple_location (stmt
);
7189 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7190 is smaller than word size, then expand_atomic_store assumes that the store
7191 is atomic. We could avoid the builtin entirely in this case. */
7193 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7194 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7195 decl
= builtin_decl_explicit (tmpbase
);
7196 if (decl
== NULL_TREE
)
7199 type
= TREE_TYPE (stored_val
);
7201 /* Dig out the type of the function's second argument. */
7202 itype
= TREE_TYPE (decl
);
7203 itype
= TYPE_ARG_TYPES (itype
);
7204 itype
= TREE_CHAIN (itype
);
7205 itype
= TREE_VALUE (itype
);
7206 imode
= TYPE_MODE (itype
);
7208 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7211 if (!useless_type_conversion_p (itype
, type
))
7212 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7213 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7214 build_int_cst (NULL
,
7215 gimple_omp_atomic_seq_cst_p (stmt
)
7217 : MEMMODEL_RELAXED
));
7220 if (!useless_type_conversion_p (type
, itype
))
7221 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7222 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7225 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7226 gsi_remove (&gsi
, true);
7228 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7229 gsi
= gsi_last_bb (load_bb
);
7230 gsi_remove (&gsi
, true);
7232 if (gimple_in_ssa_p (cfun
))
7233 update_ssa (TODO_update_ssa_no_phi
);
7238 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7239 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7240 size of the data type, and thus usable to find the index of the builtin
7241 decl. Returns false if the expression is not of the proper form. */
7244 expand_omp_atomic_fetch_op (basic_block load_bb
,
7245 tree addr
, tree loaded_val
,
7246 tree stored_val
, int index
)
7248 enum built_in_function oldbase
, newbase
, tmpbase
;
7249 tree decl
, itype
, call
;
7251 basic_block store_bb
= single_succ (load_bb
);
7252 gimple_stmt_iterator gsi
;
7255 enum tree_code code
;
7256 bool need_old
, need_new
;
7257 enum machine_mode imode
;
7260 /* We expect to find the following sequences:
7263 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7266 val = tmp OP something; (or: something OP tmp)
7267 GIMPLE_OMP_STORE (val)
7269 ???FIXME: Allow a more flexible sequence.
7270 Perhaps use data flow to pick the statements.
7274 gsi
= gsi_after_labels (store_bb
);
7275 stmt
= gsi_stmt (gsi
);
7276 loc
= gimple_location (stmt
);
7277 if (!is_gimple_assign (stmt
))
7280 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7282 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7283 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7284 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7285 gcc_checking_assert (!need_old
|| !need_new
);
7287 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7290 /* Check for one of the supported fetch-op operations. */
7291 code
= gimple_assign_rhs_code (stmt
);
7295 case POINTER_PLUS_EXPR
:
7296 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7297 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7300 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7301 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7304 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7305 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7308 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7309 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7312 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7313 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7319 /* Make sure the expression is of the proper form. */
7320 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7321 rhs
= gimple_assign_rhs2 (stmt
);
7322 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7323 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7324 rhs
= gimple_assign_rhs1 (stmt
);
7328 tmpbase
= ((enum built_in_function
)
7329 ((need_new
? newbase
: oldbase
) + index
+ 1));
7330 decl
= builtin_decl_explicit (tmpbase
);
7331 if (decl
== NULL_TREE
)
7333 itype
= TREE_TYPE (TREE_TYPE (decl
));
7334 imode
= TYPE_MODE (itype
);
7336 /* We could test all of the various optabs involved, but the fact of the
7337 matter is that (with the exception of i486 vs i586 and xadd) all targets
7338 that support any atomic operaton optab also implements compare-and-swap.
7339 Let optabs.c take care of expanding any compare-and-swap loop. */
7340 if (!can_compare_and_swap_p (imode
, true))
7343 gsi
= gsi_last_bb (load_bb
);
7344 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7346 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7347 It only requires that the operation happen atomically. Thus we can
7348 use the RELAXED memory model. */
7349 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7350 fold_convert_loc (loc
, itype
, rhs
),
7351 build_int_cst (NULL
,
7352 seq_cst
? MEMMODEL_SEQ_CST
7353 : MEMMODEL_RELAXED
));
7355 if (need_old
|| need_new
)
7357 lhs
= need_old
? loaded_val
: stored_val
;
7358 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7359 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7362 call
= fold_convert_loc (loc
, void_type_node
, call
);
7363 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7364 gsi_remove (&gsi
, true);
7366 gsi
= gsi_last_bb (store_bb
);
7367 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7368 gsi_remove (&gsi
, true);
7369 gsi
= gsi_last_bb (store_bb
);
7370 gsi_remove (&gsi
, true);
7372 if (gimple_in_ssa_p (cfun
))
7373 update_ssa (TODO_update_ssa_no_phi
);
7378 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7382 newval = rhs; // with oldval replacing *addr in rhs
7383 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7384 if (oldval != newval)
7387 INDEX is log2 of the size of the data type, and thus usable to find the
7388 index of the builtin decl. */
7391 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7392 tree addr
, tree loaded_val
, tree stored_val
,
7395 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7396 tree type
, itype
, cmpxchg
, iaddr
;
7397 gimple_stmt_iterator si
;
7398 basic_block loop_header
= single_succ (load_bb
);
7401 enum built_in_function fncode
;
7403 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7404 order to use the RELAXED memory model effectively. */
7405 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7407 cmpxchg
= builtin_decl_explicit (fncode
);
7408 if (cmpxchg
== NULL_TREE
)
7410 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7411 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7413 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7416 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7417 si
= gsi_last_bb (load_bb
);
7418 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7420 /* For floating-point values, we'll need to view-convert them to integers
7421 so that we can perform the atomic compare and swap. Simplify the
7422 following code by always setting up the "i"ntegral variables. */
7423 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7427 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7430 = force_gimple_operand_gsi (&si
,
7431 fold_convert (TREE_TYPE (iaddr
), addr
),
7432 false, NULL_TREE
, true, GSI_SAME_STMT
);
7433 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7434 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7435 loadedi
= create_tmp_var (itype
, NULL
);
7436 if (gimple_in_ssa_p (cfun
))
7437 loadedi
= make_ssa_name (loadedi
, NULL
);
7442 loadedi
= loaded_val
;
7446 = force_gimple_operand_gsi (&si
,
7447 build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)),
7449 build_int_cst (TREE_TYPE (iaddr
), 0)),
7450 true, NULL_TREE
, true, GSI_SAME_STMT
);
7452 /* Move the value to the LOADEDI temporary. */
7453 if (gimple_in_ssa_p (cfun
))
7455 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7456 phi
= create_phi_node (loadedi
, loop_header
);
7457 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7461 gsi_insert_before (&si
,
7462 gimple_build_assign (loadedi
, initial
),
7464 if (loadedi
!= loaded_val
)
7466 gimple_stmt_iterator gsi2
;
7469 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7470 gsi2
= gsi_start_bb (loop_header
);
7471 if (gimple_in_ssa_p (cfun
))
7474 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7475 true, GSI_SAME_STMT
);
7476 stmt
= gimple_build_assign (loaded_val
, x
);
7477 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7481 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7482 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7483 true, GSI_SAME_STMT
);
7486 gsi_remove (&si
, true);
7488 si
= gsi_last_bb (store_bb
);
7489 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7492 storedi
= stored_val
;
7495 force_gimple_operand_gsi (&si
,
7496 build1 (VIEW_CONVERT_EXPR
, itype
,
7497 stored_val
), true, NULL_TREE
, true,
7500 /* Build the compare&swap statement. */
7501 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7502 new_storedi
= force_gimple_operand_gsi (&si
,
7503 fold_convert (TREE_TYPE (loadedi
),
7506 true, GSI_SAME_STMT
);
7508 if (gimple_in_ssa_p (cfun
))
7512 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7513 stmt
= gimple_build_assign (old_vali
, loadedi
);
7514 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7516 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7517 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7520 /* Note that we always perform the comparison as an integer, even for
7521 floating point. This allows the atomic operation to properly
7522 succeed even with NaNs and -0.0. */
7523 stmt
= gimple_build_cond_empty
7524 (build2 (NE_EXPR
, boolean_type_node
,
7525 new_storedi
, old_vali
));
7526 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7529 e
= single_succ_edge (store_bb
);
7530 e
->flags
&= ~EDGE_FALLTHRU
;
7531 e
->flags
|= EDGE_FALSE_VALUE
;
7533 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7535 /* Copy the new value to loadedi (we already did that before the condition
7536 if we are not in SSA). */
7537 if (gimple_in_ssa_p (cfun
))
7539 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7540 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7543 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7544 gsi_remove (&si
, true);
7546 struct loop
*loop
= alloc_loop ();
7547 loop
->header
= loop_header
;
7548 loop
->latch
= store_bb
;
7549 add_loop (loop
, loop_header
->loop_father
);
7551 if (gimple_in_ssa_p (cfun
))
7552 update_ssa (TODO_update_ssa_no_phi
);
7557 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7559 GOMP_atomic_start ();
7563 The result is not globally atomic, but works so long as all parallel
7564 references are within #pragma omp atomic directives. According to
7565 responses received from omp@openmp.org, appears to be within spec.
7566 Which makes sense, since that's how several other compilers handle
7567 this situation as well.
7568 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7569 expanding. STORED_VAL is the operand of the matching
7570 GIMPLE_OMP_ATOMIC_STORE.
7573 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7577 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7582 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7583 tree addr
, tree loaded_val
, tree stored_val
)
7585 gimple_stmt_iterator si
;
7589 si
= gsi_last_bb (load_bb
);
7590 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7592 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7593 t
= build_call_expr (t
, 0);
7594 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7596 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7597 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7598 gsi_remove (&si
, true);
7600 si
= gsi_last_bb (store_bb
);
7601 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7603 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7605 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7607 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7608 t
= build_call_expr (t
, 0);
7609 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7610 gsi_remove (&si
, true);
7612 if (gimple_in_ssa_p (cfun
))
7613 update_ssa (TODO_update_ssa_no_phi
);
7617 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7618 using expand_omp_atomic_fetch_op. If it failed, we try to
7619 call expand_omp_atomic_pipeline, and if it fails too, the
7620 ultimate fallback is wrapping the operation in a mutex
7621 (expand_omp_atomic_mutex). REGION is the atomic region built
7622 by build_omp_regions_1(). */
7625 expand_omp_atomic (struct omp_region
*region
)
7627 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7628 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7629 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7630 tree addr
= gimple_omp_atomic_load_rhs (load
);
7631 tree stored_val
= gimple_omp_atomic_store_val (store
);
7632 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7633 HOST_WIDE_INT index
;
7635 /* Make sure the type is one of the supported sizes. */
7636 index
= tree_low_cst (TYPE_SIZE_UNIT (type
), 1);
7637 index
= exact_log2 (index
);
7638 if (index
>= 0 && index
<= 4)
7640 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7642 /* __sync builtins require strict data alignment. */
7643 if (exact_log2 (align
) >= index
)
7646 if (loaded_val
== stored_val
7647 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7648 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7649 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7650 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7654 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7655 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7656 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7657 && store_bb
== single_succ (load_bb
)
7658 && first_stmt (store_bb
) == store
7659 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7663 /* When possible, use specialized atomic update functions. */
7664 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7665 && store_bb
== single_succ (load_bb
)
7666 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7667 loaded_val
, stored_val
, index
))
7670 /* If we don't have specialized __sync builtins, try and implement
7671 as a compare and swap loop. */
7672 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7673 loaded_val
, stored_val
, index
))
7678 /* The ultimate fallback is wrapping the operation in a mutex. */
7679 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7683 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7686 expand_omp_target (struct omp_region
*region
)
7688 basic_block entry_bb
, exit_bb
, new_bb
;
7689 struct function
*child_cfun
= NULL
;
7690 tree child_fn
= NULL_TREE
, block
, t
;
7691 gimple_stmt_iterator gsi
;
7692 gimple entry_stmt
, stmt
;
7695 entry_stmt
= last_stmt (region
->entry
);
7696 new_bb
= region
->entry
;
7697 int kind
= gimple_omp_target_kind (entry_stmt
);
7698 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7700 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7701 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7704 entry_bb
= region
->entry
;
7705 exit_bb
= region
->exit
;
7707 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7709 unsigned srcidx
, dstidx
, num
;
7711 /* If the target region needs data sent from the parent
7712 function, then the very first statement (except possible
7713 tree profile counter updates) of the parallel body
7714 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7715 &.OMP_DATA_O is passed as an argument to the child function,
7716 we need to replace it with the argument as seen by the child
7719 In most cases, this will end up being the identity assignment
7720 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7721 a function call that has been inlined, the original PARM_DECL
7722 .OMP_DATA_I may have been converted into a different local
7723 variable. In which case, we need to keep the assignment. */
7724 if (gimple_omp_target_data_arg (entry_stmt
))
7726 basic_block entry_succ_bb
= single_succ (entry_bb
);
7727 gimple_stmt_iterator gsi
;
7729 gimple tgtcopy_stmt
= NULL
;
7731 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7733 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7735 gcc_assert (!gsi_end_p (gsi
));
7736 stmt
= gsi_stmt (gsi
);
7737 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7740 if (gimple_num_ops (stmt
) == 2)
7742 tree arg
= gimple_assign_rhs1 (stmt
);
7744 /* We're ignoring the subcode because we're
7745 effectively doing a STRIP_NOPS. */
7747 if (TREE_CODE (arg
) == ADDR_EXPR
7748 && TREE_OPERAND (arg
, 0) == sender
)
7750 tgtcopy_stmt
= stmt
;
7756 gcc_assert (tgtcopy_stmt
!= NULL
);
7757 arg
= DECL_ARGUMENTS (child_fn
);
7759 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7760 gsi_remove (&gsi
, true);
7763 /* Declare local variables needed in CHILD_CFUN. */
7764 block
= DECL_INITIAL (child_fn
);
7765 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7766 /* The gimplifier could record temporaries in target block
7767 rather than in containing function's local_decls chain,
7768 which would mean cgraph missed finalizing them. Do it now. */
7769 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7770 if (TREE_CODE (t
) == VAR_DECL
7772 && !DECL_EXTERNAL (t
))
7773 varpool_finalize_decl (t
);
7774 DECL_SAVED_TREE (child_fn
) = NULL
;
7775 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7776 gimple_set_body (child_fn
, NULL
);
7777 TREE_USED (block
) = 1;
7779 /* Reset DECL_CONTEXT on function arguments. */
7780 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7781 DECL_CONTEXT (t
) = child_fn
;
7783 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7784 so that it can be moved to the child function. */
7785 gsi
= gsi_last_bb (entry_bb
);
7786 stmt
= gsi_stmt (gsi
);
7787 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7788 && gimple_omp_target_kind (stmt
)
7789 == GF_OMP_TARGET_KIND_REGION
);
7790 gsi_remove (&gsi
, true);
7791 e
= split_block (entry_bb
, stmt
);
7793 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7795 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7798 gsi
= gsi_last_bb (exit_bb
);
7799 gcc_assert (!gsi_end_p (gsi
)
7800 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7801 stmt
= gimple_build_return (NULL
);
7802 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7803 gsi_remove (&gsi
, true);
7806 /* Move the target region into CHILD_CFUN. */
7808 block
= gimple_block (entry_stmt
);
7810 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7812 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7813 /* When the OMP expansion process cannot guarantee an up-to-date
7814 loop tree arrange for the child function to fixup loops. */
7815 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7816 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7818 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7819 num
= vec_safe_length (child_cfun
->local_decls
);
7820 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7822 t
= (*child_cfun
->local_decls
)[srcidx
];
7823 if (DECL_CONTEXT (t
) == cfun
->decl
)
7825 if (srcidx
!= dstidx
)
7826 (*child_cfun
->local_decls
)[dstidx
] = t
;
7830 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7832 /* Inform the callgraph about the new function. */
7833 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
7834 cgraph_add_new_function (child_fn
, true);
7836 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7837 fixed in a following pass. */
7838 push_cfun (child_cfun
);
7839 rebuild_cgraph_edges ();
7841 /* Some EH regions might become dead, see PR34608. If
7842 pass_cleanup_cfg isn't the first pass to happen with the
7843 new child, these dead EH edges might cause problems.
7844 Clean them up now. */
7845 if (flag_exceptions
)
7848 bool changed
= false;
7851 changed
|= gimple_purge_dead_eh_edges (bb
);
7853 cleanup_tree_cfg ();
7858 /* Emit a library call to launch the target region, or do data
7860 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
7861 enum built_in_function start_ix
;
7862 location_t clause_loc
;
7864 clauses
= gimple_omp_target_clauses (entry_stmt
);
7866 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7867 start_ix
= BUILT_IN_GOMP_TARGET
;
7868 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
7869 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
7871 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
7873 /* By default, the value of DEVICE is -1 (let runtime library choose)
7874 and there is no conditional. */
7876 device
= build_int_cst (integer_type_node
, -1);
7878 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
7880 cond
= OMP_CLAUSE_IF_EXPR (c
);
7882 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
7885 device
= OMP_CLAUSE_DEVICE_ID (c
);
7886 clause_loc
= OMP_CLAUSE_LOCATION (c
);
7889 clause_loc
= gimple_location (entry_stmt
);
7891 /* Ensure 'device' is of the correct type. */
7892 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
7894 /* If we found the clause 'if (cond)', build
7895 (cond ? device : -2). */
7898 cond
= gimple_boolify (cond
);
7900 basic_block cond_bb
, then_bb
, else_bb
;
7904 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
7905 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
7907 gsi
= gsi_last_bb (new_bb
);
7909 e
= split_block (new_bb
, gsi_stmt (gsi
));
7912 e
= split_block (new_bb
, NULL
);
7917 then_bb
= create_empty_bb (cond_bb
);
7918 else_bb
= create_empty_bb (then_bb
);
7919 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
7920 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
7922 stmt
= gimple_build_cond_empty (cond
);
7923 gsi
= gsi_last_bb (cond_bb
);
7924 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7926 gsi
= gsi_start_bb (then_bb
);
7927 stmt
= gimple_build_assign (tmp_var
, device
);
7928 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7930 gsi
= gsi_start_bb (else_bb
);
7931 stmt
= gimple_build_assign (tmp_var
,
7932 build_int_cst (integer_type_node
, -2));
7933 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7935 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
7936 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
7939 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
7940 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
7942 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
7943 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
7948 gsi
= gsi_last_bb (new_bb
);
7949 t
= gimple_omp_target_data_arg (entry_stmt
);
7952 t1
= size_zero_node
;
7953 t2
= build_zero_cst (ptr_type_node
);
7959 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
7960 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
7961 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
7962 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
7963 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
7967 /* FIXME: This will be address of
7968 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
7969 symbol, as soon as the linker plugin is able to create it for us. */
7970 tree openmp_target
= build_zero_cst (ptr_type_node
);
7971 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7973 tree fnaddr
= build_fold_addr_expr (child_fn
);
7974 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
7975 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
7978 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
7979 device
, openmp_target
, t1
, t2
, t3
, t4
);
7980 gimple_set_location (g
, gimple_location (entry_stmt
));
7981 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
7982 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
7985 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
7986 gsi_remove (&gsi
, true);
7988 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
7990 gsi
= gsi_last_bb (region
->exit
);
7992 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
7993 gsi_remove (&gsi
, true);
7998 /* Expand the parallel region tree rooted at REGION. Expansion
7999 proceeds in depth-first order. Innermost regions are expanded
8000 first. This way, parallel regions that require a new function to
8001 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8002 internal dependencies in their body. */
8005 expand_omp (struct omp_region
*region
)
8009 location_t saved_location
;
8010 gimple inner_stmt
= NULL
;
8012 /* First, determine whether this is a combined parallel+workshare
8014 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8015 determine_parallel_type (region
);
8017 if (region
->type
== GIMPLE_OMP_FOR
8018 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8019 inner_stmt
= last_stmt (region
->inner
->entry
);
8022 expand_omp (region
->inner
);
8024 saved_location
= input_location
;
8025 if (gimple_has_location (last_stmt (region
->entry
)))
8026 input_location
= gimple_location (last_stmt (region
->entry
));
8028 switch (region
->type
)
8030 case GIMPLE_OMP_PARALLEL
:
8031 case GIMPLE_OMP_TASK
:
8032 expand_omp_taskreg (region
);
8035 case GIMPLE_OMP_FOR
:
8036 expand_omp_for (region
, inner_stmt
);
8039 case GIMPLE_OMP_SECTIONS
:
8040 expand_omp_sections (region
);
8043 case GIMPLE_OMP_SECTION
:
8044 /* Individual omp sections are handled together with their
8045 parent GIMPLE_OMP_SECTIONS region. */
8048 case GIMPLE_OMP_SINGLE
:
8049 expand_omp_single (region
);
8052 case GIMPLE_OMP_MASTER
:
8053 case GIMPLE_OMP_TASKGROUP
:
8054 case GIMPLE_OMP_ORDERED
:
8055 case GIMPLE_OMP_CRITICAL
:
8056 case GIMPLE_OMP_TEAMS
:
8057 expand_omp_synch (region
);
8060 case GIMPLE_OMP_ATOMIC_LOAD
:
8061 expand_omp_atomic (region
);
8064 case GIMPLE_OMP_TARGET
:
8065 expand_omp_target (region
);
8072 input_location
= saved_location
;
8073 region
= region
->next
;
8078 /* Helper for build_omp_regions. Scan the dominator tree starting at
8079 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8080 true, the function ends once a single tree is built (otherwise, whole
8081 forest of OMP constructs may be built). */
8084 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8087 gimple_stmt_iterator gsi
;
8091 gsi
= gsi_last_bb (bb
);
8092 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8094 struct omp_region
*region
;
8095 enum gimple_code code
;
8097 stmt
= gsi_stmt (gsi
);
8098 code
= gimple_code (stmt
);
8099 if (code
== GIMPLE_OMP_RETURN
)
8101 /* STMT is the return point out of region PARENT. Mark it
8102 as the exit point and make PARENT the immediately
8103 enclosing region. */
8104 gcc_assert (parent
);
8107 parent
= parent
->outer
;
8109 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8111 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8112 GIMPLE_OMP_RETURN, but matches with
8113 GIMPLE_OMP_ATOMIC_LOAD. */
8114 gcc_assert (parent
);
8115 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8118 parent
= parent
->outer
;
8121 else if (code
== GIMPLE_OMP_CONTINUE
)
8123 gcc_assert (parent
);
8126 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8128 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8129 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8132 else if (code
== GIMPLE_OMP_TARGET
8133 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8134 new_omp_region (bb
, code
, parent
);
8137 /* Otherwise, this directive becomes the parent for a new
8139 region
= new_omp_region (bb
, code
, parent
);
8144 if (single_tree
&& !parent
)
8147 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8149 son
= next_dom_son (CDI_DOMINATORS
, son
))
8150 build_omp_regions_1 (son
, parent
, single_tree
);
8153 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8157 build_omp_regions_root (basic_block root
)
8159 gcc_assert (root_omp_region
== NULL
);
8160 build_omp_regions_1 (root
, NULL
, true);
8161 gcc_assert (root_omp_region
!= NULL
);
8164 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8167 omp_expand_local (basic_block head
)
8169 build_omp_regions_root (head
);
8170 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8172 fprintf (dump_file
, "\nOMP region tree\n\n");
8173 dump_omp_region (dump_file
, root_omp_region
, 0);
8174 fprintf (dump_file
, "\n");
8177 remove_exit_barriers (root_omp_region
);
8178 expand_omp (root_omp_region
);
8180 free_omp_regions ();
8183 /* Scan the CFG and build a tree of OMP regions. Return the root of
8184 the OMP region tree. */
8187 build_omp_regions (void)
8189 gcc_assert (root_omp_region
== NULL
);
8190 calculate_dominance_info (CDI_DOMINATORS
);
8191 build_omp_regions_1 (ENTRY_BLOCK_PTR
, NULL
, false);
8194 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8197 execute_expand_omp (void)
8199 build_omp_regions ();
8201 if (!root_omp_region
)
8206 fprintf (dump_file
, "\nOMP region tree\n\n");
8207 dump_omp_region (dump_file
, root_omp_region
, 0);
8208 fprintf (dump_file
, "\n");
8211 remove_exit_barriers (root_omp_region
);
8213 expand_omp (root_omp_region
);
8215 cleanup_tree_cfg ();
8217 free_omp_regions ();
8222 /* OMP expansion -- the default pass, run before creation of SSA form. */
8225 gate_expand_omp (void)
8227 return (flag_openmp
!= 0 && !seen_error ());
8232 const pass_data pass_data_expand_omp
=
8234 GIMPLE_PASS
, /* type */
8235 "ompexp", /* name */
8236 OPTGROUP_NONE
, /* optinfo_flags */
8237 true, /* has_gate */
8238 true, /* has_execute */
8239 TV_NONE
, /* tv_id */
8240 PROP_gimple_any
, /* properties_required */
8241 0, /* properties_provided */
8242 0, /* properties_destroyed */
8243 0, /* todo_flags_start */
8244 0, /* todo_flags_finish */
8247 class pass_expand_omp
: public gimple_opt_pass
8250 pass_expand_omp (gcc::context
*ctxt
)
8251 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8254 /* opt_pass methods: */
8255 bool gate () { return gate_expand_omp (); }
8256 unsigned int execute () { return execute_expand_omp (); }
8258 }; // class pass_expand_omp
8263 make_pass_expand_omp (gcc::context
*ctxt
)
8265 return new pass_expand_omp (ctxt
);
8268 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8270 /* If ctx is a worksharing context inside of a cancellable parallel
8271 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8272 and conditional branch to parallel's cancel_label to handle
8273 cancellation in the implicit barrier. */
8276 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8278 gimple omp_return
= gimple_seq_last_stmt (*body
);
8279 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8280 if (gimple_omp_return_nowait_p (omp_return
))
8283 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8284 && ctx
->outer
->cancellable
)
8286 tree lhs
= create_tmp_var (boolean_type_node
, NULL
);
8287 gimple_omp_return_set_lhs (omp_return
, lhs
);
8288 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8289 gimple g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
8290 ctx
->outer
->cancel_label
, fallthru_label
);
8291 gimple_seq_add_stmt (body
, g
);
8292 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8296 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8297 CTX is the enclosing OMP context for the current statement. */
8300 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8302 tree block
, control
;
8303 gimple_stmt_iterator tgsi
;
8304 gimple stmt
, new_stmt
, bind
, t
;
8305 gimple_seq ilist
, dlist
, olist
, new_body
;
8306 struct gimplify_ctx gctx
;
8308 stmt
= gsi_stmt (*gsi_p
);
8310 push_gimplify_context (&gctx
);
8314 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8315 &ilist
, &dlist
, ctx
, NULL
);
8317 new_body
= gimple_omp_body (stmt
);
8318 gimple_omp_set_body (stmt
, NULL
);
8319 tgsi
= gsi_start (new_body
);
8320 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8325 sec_start
= gsi_stmt (tgsi
);
8326 sctx
= maybe_lookup_ctx (sec_start
);
8329 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8330 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8331 GSI_CONTINUE_LINKING
);
8332 gimple_omp_set_body (sec_start
, NULL
);
8334 if (gsi_one_before_end_p (tgsi
))
8336 gimple_seq l
= NULL
;
8337 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8339 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8340 gimple_omp_section_set_last (sec_start
);
8343 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8344 GSI_CONTINUE_LINKING
);
8347 block
= make_node (BLOCK
);
8348 bind
= gimple_build_bind (NULL
, new_body
, block
);
8351 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8353 block
= make_node (BLOCK
);
8354 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8355 gsi_replace (gsi_p
, new_stmt
, true);
8357 pop_gimplify_context (new_stmt
);
8358 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8359 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8360 if (BLOCK_VARS (block
))
8361 TREE_USED (block
) = 1;
8364 gimple_seq_add_seq (&new_body
, ilist
);
8365 gimple_seq_add_stmt (&new_body
, stmt
);
8366 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8367 gimple_seq_add_stmt (&new_body
, bind
);
8369 control
= create_tmp_var (unsigned_type_node
, ".section");
8370 t
= gimple_build_omp_continue (control
, control
);
8371 gimple_omp_sections_set_control (stmt
, control
);
8372 gimple_seq_add_stmt (&new_body
, t
);
8374 gimple_seq_add_seq (&new_body
, olist
);
8375 if (ctx
->cancellable
)
8376 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8377 gimple_seq_add_seq (&new_body
, dlist
);
8379 new_body
= maybe_catch_exception (new_body
);
8381 t
= gimple_build_omp_return
8382 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8383 OMP_CLAUSE_NOWAIT
));
8384 gimple_seq_add_stmt (&new_body
, t
);
8385 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8387 gimple_bind_set_body (new_stmt
, new_body
);
8391 /* A subroutine of lower_omp_single. Expand the simple form of
8392 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8394 if (GOMP_single_start ())
8396 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8398 FIXME. It may be better to delay expanding the logic of this until
8399 pass_expand_omp. The expanded logic may make the job more difficult
8400 to a synchronization analysis pass. */
8403 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8405 location_t loc
= gimple_location (single_stmt
);
8406 tree tlabel
= create_artificial_label (loc
);
8407 tree flabel
= create_artificial_label (loc
);
8411 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8412 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8413 call
= gimple_build_call (decl
, 0);
8414 gimple_call_set_lhs (call
, lhs
);
8415 gimple_seq_add_stmt (pre_p
, call
);
8417 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8418 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8421 gimple_seq_add_stmt (pre_p
, cond
);
8422 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8423 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8424 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8428 /* A subroutine of lower_omp_single. Expand the simple form of
8429 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8431 #pragma omp single copyprivate (a, b, c)
8433 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8436 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8442 GOMP_single_copy_end (©out);
8453 FIXME. It may be better to delay expanding the logic of this until
8454 pass_expand_omp. The expanded logic may make the job more difficult
8455 to a synchronization analysis pass. */
8458 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8460 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8461 gimple_seq copyin_seq
;
8462 location_t loc
= gimple_location (single_stmt
);
8464 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8466 ptr_type
= build_pointer_type (ctx
->record_type
);
8467 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8469 l0
= create_artificial_label (loc
);
8470 l1
= create_artificial_label (loc
);
8471 l2
= create_artificial_label (loc
);
8473 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8474 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8475 t
= fold_convert_loc (loc
, ptr_type
, t
);
8476 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8478 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8479 build_int_cst (ptr_type
, 0));
8480 t
= build3 (COND_EXPR
, void_type_node
, t
,
8481 build_and_jump (&l0
), build_and_jump (&l1
));
8482 gimplify_and_add (t
, pre_p
);
8484 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8486 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8489 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8492 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8493 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8494 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8495 gimplify_and_add (t
, pre_p
);
8497 t
= build_and_jump (&l2
);
8498 gimplify_and_add (t
, pre_p
);
8500 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8502 gimple_seq_add_seq (pre_p
, copyin_seq
);
8504 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8508 /* Expand code for an OpenMP single directive. */
8511 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8514 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8515 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8516 struct gimplify_ctx gctx
;
8518 push_gimplify_context (&gctx
);
8520 block
= make_node (BLOCK
);
8521 bind
= gimple_build_bind (NULL
, NULL
, block
);
8522 gsi_replace (gsi_p
, bind
, true);
8525 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8526 &bind_body
, &dlist
, ctx
, NULL
);
8527 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8529 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8531 if (ctx
->record_type
)
8532 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8534 lower_omp_single_simple (single_stmt
, &bind_body
);
8536 gimple_omp_set_body (single_stmt
, NULL
);
8538 gimple_seq_add_seq (&bind_body
, dlist
);
8540 bind_body
= maybe_catch_exception (bind_body
);
8542 t
= gimple_build_omp_return
8543 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8544 OMP_CLAUSE_NOWAIT
));
8545 gimple_seq_add_stmt (&bind_body_tail
, t
);
8546 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8547 if (ctx
->record_type
)
8549 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8550 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8551 TREE_THIS_VOLATILE (clobber
) = 1;
8552 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8553 clobber
), GSI_SAME_STMT
);
8555 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8556 gimple_bind_set_body (bind
, bind_body
);
8558 pop_gimplify_context (bind
);
8560 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8561 BLOCK_VARS (block
) = ctx
->block_vars
;
8562 if (BLOCK_VARS (block
))
8563 TREE_USED (block
) = 1;
8567 /* Expand code for an OpenMP master directive. */
8570 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8572 tree block
, lab
= NULL
, x
, bfn_decl
;
8573 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8574 location_t loc
= gimple_location (stmt
);
8576 struct gimplify_ctx gctx
;
8578 push_gimplify_context (&gctx
);
8580 block
= make_node (BLOCK
);
8581 bind
= gimple_build_bind (NULL
, NULL
, block
);
8582 gsi_replace (gsi_p
, bind
, true);
8583 gimple_bind_add_stmt (bind
, stmt
);
8585 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8586 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8587 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8588 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8590 gimplify_and_add (x
, &tseq
);
8591 gimple_bind_add_seq (bind
, tseq
);
8593 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8594 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8595 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8596 gimple_omp_set_body (stmt
, NULL
);
8598 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8600 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8602 pop_gimplify_context (bind
);
8604 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8605 BLOCK_VARS (block
) = ctx
->block_vars
;
8609 /* Expand code for an OpenMP taskgroup directive. */
8612 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8614 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8615 tree block
= make_node (BLOCK
);
8617 bind
= gimple_build_bind (NULL
, NULL
, block
);
8618 gsi_replace (gsi_p
, bind
, true);
8619 gimple_bind_add_stmt (bind
, stmt
);
8621 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8623 gimple_bind_add_stmt (bind
, x
);
8625 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8626 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8627 gimple_omp_set_body (stmt
, NULL
);
8629 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8631 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8632 BLOCK_VARS (block
) = ctx
->block_vars
;
8636 /* Expand code for an OpenMP ordered directive. */
8639 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8642 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8643 struct gimplify_ctx gctx
;
8645 push_gimplify_context (&gctx
);
8647 block
= make_node (BLOCK
);
8648 bind
= gimple_build_bind (NULL
, NULL
, block
);
8649 gsi_replace (gsi_p
, bind
, true);
8650 gimple_bind_add_stmt (bind
, stmt
);
8652 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8654 gimple_bind_add_stmt (bind
, x
);
8656 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8657 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8658 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8659 gimple_omp_set_body (stmt
, NULL
);
8661 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8662 gimple_bind_add_stmt (bind
, x
);
8664 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8666 pop_gimplify_context (bind
);
8668 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8669 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8673 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8674 substitution of a couple of function calls. But in the NAMED case,
8675 requires that languages coordinate a symbol name. It is therefore
8676 best put here in common code. */
8678 static GTY((param1_is (tree
), param2_is (tree
)))
8679 splay_tree critical_name_mutexes
;
8682 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8685 tree name
, lock
, unlock
;
8686 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8687 location_t loc
= gimple_location (stmt
);
8689 struct gimplify_ctx gctx
;
8691 name
= gimple_omp_critical_name (stmt
);
8697 if (!critical_name_mutexes
)
8698 critical_name_mutexes
8699 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8700 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8701 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8703 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8708 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8710 new_str
= ACONCAT ((".gomp_critical_user_",
8711 IDENTIFIER_POINTER (name
), NULL
));
8712 DECL_NAME (decl
) = get_identifier (new_str
);
8713 TREE_PUBLIC (decl
) = 1;
8714 TREE_STATIC (decl
) = 1;
8715 DECL_COMMON (decl
) = 1;
8716 DECL_ARTIFICIAL (decl
) = 1;
8717 DECL_IGNORED_P (decl
) = 1;
8718 varpool_finalize_decl (decl
);
8720 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8721 (splay_tree_value
) decl
);
8724 decl
= (tree
) n
->value
;
8726 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8727 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8729 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8730 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8731 build_fold_addr_expr_loc (loc
, decl
));
8735 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8736 lock
= build_call_expr_loc (loc
, lock
, 0);
8738 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8739 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8742 push_gimplify_context (&gctx
);
8744 block
= make_node (BLOCK
);
8745 bind
= gimple_build_bind (NULL
, NULL
, block
);
8746 gsi_replace (gsi_p
, bind
, true);
8747 gimple_bind_add_stmt (bind
, stmt
);
8749 tbody
= gimple_bind_body (bind
);
8750 gimplify_and_add (lock
, &tbody
);
8751 gimple_bind_set_body (bind
, tbody
);
8753 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8754 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8755 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8756 gimple_omp_set_body (stmt
, NULL
);
8758 tbody
= gimple_bind_body (bind
);
8759 gimplify_and_add (unlock
, &tbody
);
8760 gimple_bind_set_body (bind
, tbody
);
8762 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8764 pop_gimplify_context (bind
);
8765 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8766 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8770 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8771 for a lastprivate clause. Given a loop control predicate of (V
8772 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8773 is appended to *DLIST, iterator initialization is appended to
8777 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8778 gimple_seq
*dlist
, struct omp_context
*ctx
)
8780 tree clauses
, cond
, vinit
;
8781 enum tree_code cond_code
;
8784 cond_code
= fd
->loop
.cond_code
;
8785 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8787 /* When possible, use a strict equality expression. This can let VRP
8788 type optimizations deduce the value and remove a copy. */
8789 if (host_integerp (fd
->loop
.step
, 0))
8791 HOST_WIDE_INT step
= TREE_INT_CST_LOW (fd
->loop
.step
);
8792 if (step
== 1 || step
== -1)
8793 cond_code
= EQ_EXPR
;
8796 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8798 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8800 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8801 if (!gimple_seq_empty_p (stmts
))
8803 gimple_seq_add_seq (&stmts
, *dlist
);
8806 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8807 vinit
= fd
->loop
.n1
;
8808 if (cond_code
== EQ_EXPR
8809 && host_integerp (fd
->loop
.n2
, 0)
8810 && ! integer_zerop (fd
->loop
.n2
))
8811 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8813 vinit
= unshare_expr (vinit
);
8815 /* Initialize the iterator variable, so that threads that don't execute
8816 any iterations don't execute the lastprivate clauses by accident. */
8817 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
8822 /* Lower code for an OpenMP loop directive. */
8825 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8828 struct omp_for_data fd
, *fdp
= NULL
;
8829 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
8830 gimple_seq omp_for_body
, body
, dlist
;
8832 struct gimplify_ctx gctx
;
8834 push_gimplify_context (&gctx
);
8836 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
8838 block
= make_node (BLOCK
);
8839 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8840 /* Replace at gsi right away, so that 'stmt' is no member
8841 of a sequence anymore as we're going to add to to a different
8843 gsi_replace (gsi_p
, new_stmt
, true);
8845 /* Move declaration of temporaries in the loop body before we make
8847 omp_for_body
= gimple_omp_body (stmt
);
8848 if (!gimple_seq_empty_p (omp_for_body
)
8849 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
8851 tree vars
= gimple_bind_vars (gimple_seq_first_stmt (omp_for_body
));
8852 gimple_bind_append_vars (new_stmt
, vars
);
8855 if (gimple_omp_for_combined_into_p (stmt
))
8857 extract_omp_for_data (stmt
, &fd
, NULL
);
8860 /* We need two temporaries with fd.loop.v type (istart/iend)
8861 and then (fd.collapse - 1) temporaries with the same
8862 type for count2 ... countN-1 vars if not constant. */
8864 tree type
= fd
.iter_type
;
8866 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
8867 count
+= fd
.collapse
- 1;
8868 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
8869 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
8873 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
8874 OMP_CLAUSE__LOOPTEMP_
);
8875 for (i
= 0; i
< count
; i
++)
8880 gcc_assert (outerc
);
8881 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
8882 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
8883 OMP_CLAUSE__LOOPTEMP_
);
8886 temp
= create_tmp_var (type
, NULL
);
8887 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
8888 OMP_CLAUSE_DECL (*pc
) = temp
;
8889 pc
= &OMP_CLAUSE_CHAIN (*pc
);
8894 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
8897 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
8899 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
8901 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8903 /* Lower the header expressions. At this point, we can assume that
8904 the header is of the form:
8906 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
8908 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
8909 using the .omp_data_s mapping, if needed. */
8910 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
8912 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
8913 if (!is_gimple_min_invariant (*rhs_p
))
8914 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8916 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
8917 if (!is_gimple_min_invariant (*rhs_p
))
8918 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8920 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
8921 if (!is_gimple_min_invariant (*rhs_p
))
8922 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8925 /* Once lowered, extract the bounds and clauses. */
8926 extract_omp_for_data (stmt
, &fd
, NULL
);
8928 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
8930 gimple_seq_add_stmt (&body
, stmt
);
8931 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
8933 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
8936 /* After the loop, add exit clauses. */
8937 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
8939 if (ctx
->cancellable
)
8940 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
8942 gimple_seq_add_seq (&body
, dlist
);
8944 body
= maybe_catch_exception (body
);
8946 /* Region exit marker goes at the end of the loop body. */
8947 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
8948 maybe_add_implicit_barrier_cancel (ctx
, &body
);
8949 pop_gimplify_context (new_stmt
);
8951 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8952 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
8953 if (BLOCK_VARS (block
))
8954 TREE_USED (block
) = 1;
8956 gimple_bind_set_body (new_stmt
, body
);
8957 gimple_omp_set_body (stmt
, NULL
);
8958 gimple_omp_for_set_pre_body (stmt
, NULL
);
8961 /* Callback for walk_stmts. Check if the current statement only contains
8962 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
8965 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
8966 bool *handled_ops_p
,
8967 struct walk_stmt_info
*wi
)
8969 int *info
= (int *) wi
->info
;
8970 gimple stmt
= gsi_stmt (*gsi_p
);
8972 *handled_ops_p
= true;
8973 switch (gimple_code (stmt
))
8977 case GIMPLE_OMP_FOR
:
8978 case GIMPLE_OMP_SECTIONS
:
8979 *info
= *info
== 0 ? 1 : -1;
8988 struct omp_taskcopy_context
8990 /* This field must be at the beginning, as we do "inheritance": Some
8991 callback functions for tree-inline.c (e.g., omp_copy_decl)
8992 receive a copy_body_data pointer that is up-casted to an
8993 omp_context pointer. */
8999 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9001 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9003 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9004 return create_tmp_var (TREE_TYPE (var
), NULL
);
9010 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9012 tree name
, new_fields
= NULL
, type
, f
;
9014 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9015 name
= DECL_NAME (TYPE_NAME (orig_type
));
9016 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9017 TYPE_DECL
, name
, type
);
9018 TYPE_NAME (type
) = name
;
9020 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9022 tree new_f
= copy_node (f
);
9023 DECL_CONTEXT (new_f
) = type
;
9024 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9025 TREE_CHAIN (new_f
) = new_fields
;
9026 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9027 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9028 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9031 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9033 TYPE_FIELDS (type
) = nreverse (new_fields
);
9038 /* Create task copyfn. */
9041 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9043 struct function
*child_cfun
;
9044 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9045 tree record_type
, srecord_type
, bind
, list
;
9046 bool record_needs_remap
= false, srecord_needs_remap
= false;
9048 struct omp_taskcopy_context tcctx
;
9049 struct gimplify_ctx gctx
;
9050 location_t loc
= gimple_location (task_stmt
);
9052 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9053 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9054 gcc_assert (child_cfun
->cfg
== NULL
);
9055 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9057 /* Reset DECL_CONTEXT on function arguments. */
9058 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9059 DECL_CONTEXT (t
) = child_fn
;
9061 /* Populate the function. */
9062 push_gimplify_context (&gctx
);
9063 push_cfun (child_cfun
);
9065 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9066 TREE_SIDE_EFFECTS (bind
) = 1;
9068 DECL_SAVED_TREE (child_fn
) = bind
;
9069 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9071 /* Remap src and dst argument types if needed. */
9072 record_type
= ctx
->record_type
;
9073 srecord_type
= ctx
->srecord_type
;
9074 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9075 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9077 record_needs_remap
= true;
9080 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9081 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9083 srecord_needs_remap
= true;
9087 if (record_needs_remap
|| srecord_needs_remap
)
9089 memset (&tcctx
, '\0', sizeof (tcctx
));
9090 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9091 tcctx
.cb
.dst_fn
= child_fn
;
9092 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9093 gcc_checking_assert (tcctx
.cb
.src_node
);
9094 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9095 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9096 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9097 tcctx
.cb
.eh_lp_nr
= 0;
9098 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9099 tcctx
.cb
.decl_map
= pointer_map_create ();
9102 if (record_needs_remap
)
9103 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9104 if (srecord_needs_remap
)
9105 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9108 tcctx
.cb
.decl_map
= NULL
;
9110 arg
= DECL_ARGUMENTS (child_fn
);
9111 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9112 sarg
= DECL_CHAIN (arg
);
9113 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9115 /* First pass: initialize temporaries used in record_type and srecord_type
9116 sizes and field offsets. */
9117 if (tcctx
.cb
.decl_map
)
9118 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9119 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9123 decl
= OMP_CLAUSE_DECL (c
);
9124 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9127 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9128 sf
= (tree
) n
->value
;
9129 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9130 src
= build_simple_mem_ref_loc (loc
, sarg
);
9131 src
= omp_build_component_ref (src
, sf
);
9132 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9133 append_to_statement_list (t
, &list
);
9136 /* Second pass: copy shared var pointers and copy construct non-VLA
9137 firstprivate vars. */
9138 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9139 switch (OMP_CLAUSE_CODE (c
))
9141 case OMP_CLAUSE_SHARED
:
9142 decl
= OMP_CLAUSE_DECL (c
);
9143 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9146 f
= (tree
) n
->value
;
9147 if (tcctx
.cb
.decl_map
)
9148 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9149 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9150 sf
= (tree
) n
->value
;
9151 if (tcctx
.cb
.decl_map
)
9152 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9153 src
= build_simple_mem_ref_loc (loc
, sarg
);
9154 src
= omp_build_component_ref (src
, sf
);
9155 dst
= build_simple_mem_ref_loc (loc
, arg
);
9156 dst
= omp_build_component_ref (dst
, f
);
9157 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9158 append_to_statement_list (t
, &list
);
9160 case OMP_CLAUSE_FIRSTPRIVATE
:
9161 decl
= OMP_CLAUSE_DECL (c
);
9162 if (is_variable_sized (decl
))
9164 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9167 f
= (tree
) n
->value
;
9168 if (tcctx
.cb
.decl_map
)
9169 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9170 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9173 sf
= (tree
) n
->value
;
9174 if (tcctx
.cb
.decl_map
)
9175 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9176 src
= build_simple_mem_ref_loc (loc
, sarg
);
9177 src
= omp_build_component_ref (src
, sf
);
9178 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9179 src
= build_simple_mem_ref_loc (loc
, src
);
9183 dst
= build_simple_mem_ref_loc (loc
, arg
);
9184 dst
= omp_build_component_ref (dst
, f
);
9185 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9186 append_to_statement_list (t
, &list
);
9188 case OMP_CLAUSE_PRIVATE
:
9189 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9191 decl
= OMP_CLAUSE_DECL (c
);
9192 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9193 f
= (tree
) n
->value
;
9194 if (tcctx
.cb
.decl_map
)
9195 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9196 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9199 sf
= (tree
) n
->value
;
9200 if (tcctx
.cb
.decl_map
)
9201 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9202 src
= build_simple_mem_ref_loc (loc
, sarg
);
9203 src
= omp_build_component_ref (src
, sf
);
9204 if (use_pointer_for_field (decl
, NULL
))
9205 src
= build_simple_mem_ref_loc (loc
, src
);
9209 dst
= build_simple_mem_ref_loc (loc
, arg
);
9210 dst
= omp_build_component_ref (dst
, f
);
9211 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9212 append_to_statement_list (t
, &list
);
9218 /* Last pass: handle VLA firstprivates. */
9219 if (tcctx
.cb
.decl_map
)
9220 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9221 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9225 decl
= OMP_CLAUSE_DECL (c
);
9226 if (!is_variable_sized (decl
))
9228 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9231 f
= (tree
) n
->value
;
9232 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9233 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9234 ind
= DECL_VALUE_EXPR (decl
);
9235 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9236 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9237 n
= splay_tree_lookup (ctx
->sfield_map
,
9238 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9239 sf
= (tree
) n
->value
;
9240 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9241 src
= build_simple_mem_ref_loc (loc
, sarg
);
9242 src
= omp_build_component_ref (src
, sf
);
9243 src
= build_simple_mem_ref_loc (loc
, src
);
9244 dst
= build_simple_mem_ref_loc (loc
, arg
);
9245 dst
= omp_build_component_ref (dst
, f
);
9246 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9247 append_to_statement_list (t
, &list
);
9248 n
= splay_tree_lookup (ctx
->field_map
,
9249 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9250 df
= (tree
) n
->value
;
9251 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9252 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9253 ptr
= omp_build_component_ref (ptr
, df
);
9254 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9255 build_fold_addr_expr_loc (loc
, dst
));
9256 append_to_statement_list (t
, &list
);
9259 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9260 append_to_statement_list (t
, &list
);
9262 if (tcctx
.cb
.decl_map
)
9263 pointer_map_destroy (tcctx
.cb
.decl_map
);
9264 pop_gimplify_context (NULL
);
9265 BIND_EXPR_BODY (bind
) = list
;
9270 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9274 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9276 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9278 gcc_assert (clauses
);
9279 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9280 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9281 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9283 case OMP_CLAUSE_DEPEND_IN
:
9286 case OMP_CLAUSE_DEPEND_OUT
:
9287 case OMP_CLAUSE_DEPEND_INOUT
:
9293 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9294 tree array
= create_tmp_var (type
, NULL
);
9295 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9297 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9298 gimple_seq_add_stmt (iseq
, g
);
9299 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9301 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9302 gimple_seq_add_stmt (iseq
, g
);
9303 for (i
= 0; i
< 2; i
++)
9305 if ((i
? n_in
: n_out
) == 0)
9307 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9308 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9309 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9311 tree t
= OMP_CLAUSE_DECL (c
);
9312 t
= fold_convert (ptr_type_node
, t
);
9313 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9314 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9315 NULL_TREE
, NULL_TREE
);
9316 g
= gimple_build_assign (r
, t
);
9317 gimple_seq_add_stmt (iseq
, g
);
9320 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9321 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9322 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9323 OMP_CLAUSE_CHAIN (c
) = *p
;
9325 tree clobber
= build_constructor (type
, NULL
);
9326 TREE_THIS_VOLATILE (clobber
) = 1;
9327 g
= gimple_build_assign (array
, clobber
);
9328 gimple_seq_add_stmt (oseq
, g
);
9331 /* Lower the OpenMP parallel or task directive in the current statement
9332 in GSI_P. CTX holds context information for the directive. */
9335 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9339 gimple stmt
= gsi_stmt (*gsi_p
);
9340 gimple par_bind
, bind
, dep_bind
= NULL
;
9341 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9342 struct gimplify_ctx gctx
, dep_gctx
;
9343 location_t loc
= gimple_location (stmt
);
9345 clauses
= gimple_omp_taskreg_clauses (stmt
);
9346 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9347 par_body
= gimple_bind_body (par_bind
);
9348 child_fn
= ctx
->cb
.dst_fn
;
9349 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9350 && !gimple_omp_parallel_combined_p (stmt
))
9352 struct walk_stmt_info wi
;
9355 memset (&wi
, 0, sizeof (wi
));
9358 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9360 gimple_omp_parallel_set_combined_p (stmt
, true);
9362 gimple_seq dep_ilist
= NULL
;
9363 gimple_seq dep_olist
= NULL
;
9364 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9365 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9367 push_gimplify_context (&dep_gctx
);
9368 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9369 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9372 if (ctx
->srecord_type
)
9373 create_task_copyfn (stmt
, ctx
);
9375 push_gimplify_context (&gctx
);
9380 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9381 lower_omp (&par_body
, ctx
);
9382 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9383 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9385 /* Declare all the variables created by mapping and the variables
9386 declared in the scope of the parallel body. */
9387 record_vars_into (ctx
->block_vars
, child_fn
);
9388 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9390 if (ctx
->record_type
)
9393 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9394 : ctx
->record_type
, ".omp_data_o");
9395 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9396 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9397 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9402 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9403 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9405 if (ctx
->record_type
)
9407 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9408 TREE_THIS_VOLATILE (clobber
) = 1;
9409 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9413 /* Once all the expansions are done, sequence all the different
9414 fragments inside gimple_omp_body. */
9418 if (ctx
->record_type
)
9420 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9421 /* fixup_child_record_type might have changed receiver_decl's type. */
9422 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9423 gimple_seq_add_stmt (&new_body
,
9424 gimple_build_assign (ctx
->receiver_decl
, t
));
9427 gimple_seq_add_seq (&new_body
, par_ilist
);
9428 gimple_seq_add_seq (&new_body
, par_body
);
9429 gimple_seq_add_seq (&new_body
, par_rlist
);
9430 if (ctx
->cancellable
)
9431 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9432 gimple_seq_add_seq (&new_body
, par_olist
);
9433 new_body
= maybe_catch_exception (new_body
);
9434 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9435 gimple_omp_set_body (stmt
, new_body
);
9437 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9438 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9439 gimple_bind_add_seq (bind
, ilist
);
9440 gimple_bind_add_stmt (bind
, stmt
);
9441 gimple_bind_add_seq (bind
, olist
);
9443 pop_gimplify_context (NULL
);
9447 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9448 gimple_bind_add_stmt (dep_bind
, bind
);
9449 gimple_bind_add_seq (dep_bind
, dep_olist
);
9450 pop_gimplify_context (dep_bind
);
9454 /* Lower the OpenMP target directive in the current statement
9455 in GSI_P. CTX holds context information for the directive. */
9458 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9461 tree child_fn
, t
, c
;
9462 gimple stmt
= gsi_stmt (*gsi_p
);
9463 gimple tgt_bind
= NULL
, bind
;
9464 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9465 struct gimplify_ctx gctx
;
9466 location_t loc
= gimple_location (stmt
);
9467 int kind
= gimple_omp_target_kind (stmt
);
9468 unsigned int map_cnt
= 0;
9470 clauses
= gimple_omp_target_clauses (stmt
);
9471 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9473 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9474 tgt_body
= gimple_bind_body (tgt_bind
);
9476 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9477 tgt_body
= gimple_omp_body (stmt
);
9478 child_fn
= ctx
->cb
.dst_fn
;
9480 push_gimplify_context (&gctx
);
9482 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9483 switch (OMP_CLAUSE_CODE (c
))
9489 case OMP_CLAUSE_MAP
:
9491 case OMP_CLAUSE_FROM
:
9492 var
= OMP_CLAUSE_DECL (c
);
9495 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9496 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9502 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9504 tree var2
= DECL_VALUE_EXPR (var
);
9505 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9506 var2
= TREE_OPERAND (var2
, 0);
9507 gcc_assert (DECL_P (var2
));
9511 if (!maybe_lookup_field (var
, ctx
))
9514 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9516 x
= build_receiver_ref (var
, true, ctx
);
9517 tree new_var
= lookup_decl (var
, ctx
);
9518 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9519 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9520 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9521 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9522 x
= build_simple_mem_ref (x
);
9523 SET_DECL_VALUE_EXPR (new_var
, x
);
9524 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9529 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9531 target_nesting_level
++;
9532 lower_omp (&tgt_body
, ctx
);
9533 target_nesting_level
--;
9535 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9536 lower_omp (&tgt_body
, ctx
);
9538 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9540 /* Declare all the variables created by mapping and the variables
9541 declared in the scope of the target body. */
9542 record_vars_into (ctx
->block_vars
, child_fn
);
9543 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9548 if (ctx
->record_type
)
9551 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9552 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9553 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9554 t
= make_tree_vec (3);
9555 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9557 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9559 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9560 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9561 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9563 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9566 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9567 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9568 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9569 gimple_omp_target_set_data_arg (stmt
, t
);
9571 vec
<constructor_elt
, va_gc
> *vsize
;
9572 vec
<constructor_elt
, va_gc
> *vkind
;
9573 vec_alloc (vsize
, map_cnt
);
9574 vec_alloc (vkind
, map_cnt
);
9575 unsigned int map_idx
= 0;
9577 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9578 switch (OMP_CLAUSE_CODE (c
))
9584 case OMP_CLAUSE_MAP
:
9586 case OMP_CLAUSE_FROM
:
9588 ovar
= OMP_CLAUSE_DECL (c
);
9591 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9592 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9594 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9595 == get_base_address (ovar
));
9596 nc
= OMP_CLAUSE_CHAIN (c
);
9597 ovar
= OMP_CLAUSE_DECL (nc
);
9601 tree x
= build_sender_ref (ovar
, ctx
);
9603 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9604 gimplify_assign (x
, v
, &ilist
);
9610 if (DECL_SIZE (ovar
)
9611 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9613 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9614 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9615 ovar2
= TREE_OPERAND (ovar2
, 0);
9616 gcc_assert (DECL_P (ovar2
));
9619 if (!maybe_lookup_field (ovar
, ctx
))
9625 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9626 tree x
= build_sender_ref (ovar
, ctx
);
9627 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9628 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9629 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9630 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9632 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9634 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9635 mark_addressable (avar
);
9636 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9637 avar
= build_fold_addr_expr (avar
);
9638 gimplify_assign (x
, avar
, &ilist
);
9640 else if (is_gimple_reg (var
))
9642 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9643 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9644 mark_addressable (avar
);
9645 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9646 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9647 gimplify_assign (avar
, var
, &ilist
);
9648 avar
= build_fold_addr_expr (avar
);
9649 gimplify_assign (x
, avar
, &ilist
);
9650 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9651 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9652 && !TYPE_READONLY (TREE_TYPE (var
)))
9654 x
= build_sender_ref (ovar
, ctx
);
9655 x
= build_simple_mem_ref (x
);
9656 gimplify_assign (var
, x
, &olist
);
9661 var
= build_fold_addr_expr (var
);
9662 gimplify_assign (x
, var
, &ilist
);
9665 tree s
= OMP_CLAUSE_SIZE (c
);
9667 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9668 s
= fold_convert (size_type_node
, s
);
9669 tree purpose
= size_int (map_idx
++);
9670 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9671 if (TREE_CODE (s
) != INTEGER_CST
)
9672 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9674 unsigned char tkind
= 0;
9675 switch (OMP_CLAUSE_CODE (c
))
9677 case OMP_CLAUSE_MAP
:
9678 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9681 tkind
= OMP_CLAUSE_MAP_TO
;
9683 case OMP_CLAUSE_FROM
:
9684 tkind
= OMP_CLAUSE_MAP_FROM
;
9689 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9690 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9691 talign
= DECL_ALIGN_UNIT (ovar
);
9692 talign
= ceil_log2 (talign
);
9693 tkind
|= talign
<< 3;
9694 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9695 build_int_cst (unsigned_char_type_node
,
9701 gcc_assert (map_idx
== map_cnt
);
9703 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9704 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9705 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9706 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9707 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9709 gimple_seq initlist
= NULL
;
9710 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9711 TREE_VEC_ELT (t
, 1)),
9712 &initlist
, true, NULL_TREE
);
9713 gimple_seq_add_seq (&ilist
, initlist
);
9716 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9717 TREE_THIS_VOLATILE (clobber
) = 1;
9718 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9722 /* Once all the expansions are done, sequence all the different
9723 fragments inside gimple_omp_body. */
9727 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9729 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9730 /* fixup_child_record_type might have changed receiver_decl's type. */
9731 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9732 gimple_seq_add_stmt (&new_body
,
9733 gimple_build_assign (ctx
->receiver_decl
, t
));
9736 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9738 gimple_seq_add_seq (&new_body
, tgt_body
);
9739 new_body
= maybe_catch_exception (new_body
);
9741 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9742 new_body
= tgt_body
;
9743 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9745 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9746 gimple_omp_set_body (stmt
, new_body
);
9749 bind
= gimple_build_bind (NULL
, NULL
,
9750 tgt_bind
? gimple_bind_block (tgt_bind
)
9752 gsi_replace (gsi_p
, bind
, true);
9753 gimple_bind_add_seq (bind
, ilist
);
9754 gimple_bind_add_stmt (bind
, stmt
);
9755 gimple_bind_add_seq (bind
, olist
);
9757 pop_gimplify_context (NULL
);
9760 /* Expand code for an OpenMP teams directive. */
9763 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9765 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9766 struct gimplify_ctx gctx
;
9767 push_gimplify_context (&gctx
);
9769 tree block
= make_node (BLOCK
);
9770 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9771 gsi_replace (gsi_p
, bind
, true);
9772 gimple_seq bind_body
= NULL
;
9773 gimple_seq dlist
= NULL
;
9774 gimple_seq olist
= NULL
;
9776 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9777 OMP_CLAUSE_NUM_TEAMS
);
9778 if (num_teams
== NULL_TREE
)
9779 num_teams
= build_int_cst (unsigned_type_node
, 0);
9782 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9783 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9784 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9786 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9787 OMP_CLAUSE_THREAD_LIMIT
);
9788 if (thread_limit
== NULL_TREE
)
9789 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9792 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9793 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9794 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9798 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9799 &bind_body
, &dlist
, ctx
, NULL
);
9800 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9801 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9802 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9804 location_t loc
= gimple_location (teams_stmt
);
9805 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9806 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9807 gimple_set_location (call
, loc
);
9808 gimple_seq_add_stmt (&bind_body
, call
);
9810 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
9811 gimple_omp_set_body (teams_stmt
, NULL
);
9812 gimple_seq_add_seq (&bind_body
, olist
);
9813 gimple_seq_add_seq (&bind_body
, dlist
);
9814 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
9815 gimple_bind_set_body (bind
, bind_body
);
9817 pop_gimplify_context (bind
);
9819 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9820 BLOCK_VARS (block
) = ctx
->block_vars
;
9821 if (BLOCK_VARS (block
))
9822 TREE_USED (block
) = 1;
9826 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9827 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9828 of OpenMP context, but with task_shared_vars set. */
9831 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
9836 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9837 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
9840 if (task_shared_vars
9842 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
9845 /* If a global variable has been privatized, TREE_CONSTANT on
9846 ADDR_EXPR might be wrong. */
9847 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
9848 recompute_tree_invariant_for_addr_expr (t
);
9850 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
9855 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9857 gimple stmt
= gsi_stmt (*gsi_p
);
9858 struct walk_stmt_info wi
;
9860 if (gimple_has_location (stmt
))
9861 input_location
= gimple_location (stmt
);
9863 if (task_shared_vars
)
9864 memset (&wi
, '\0', sizeof (wi
));
9866 /* If we have issued syntax errors, avoid doing any heavy lifting.
9867 Just replace the OpenMP directives with a NOP to avoid
9868 confusing RTL expansion. */
9869 if (seen_error () && is_gimple_omp (stmt
))
9871 gsi_replace (gsi_p
, gimple_build_nop (), true);
9875 switch (gimple_code (stmt
))
9878 if ((ctx
|| task_shared_vars
)
9879 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
9880 ctx
? NULL
: &wi
, NULL
)
9881 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
9882 ctx
? NULL
: &wi
, NULL
)))
9883 gimple_regimplify_operands (stmt
, gsi_p
);
9886 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
9888 case GIMPLE_EH_FILTER
:
9889 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
9892 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
9893 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
9895 case GIMPLE_TRANSACTION
:
9896 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
9899 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
9901 case GIMPLE_OMP_PARALLEL
:
9902 case GIMPLE_OMP_TASK
:
9903 ctx
= maybe_lookup_ctx (stmt
);
9905 if (ctx
->cancellable
)
9906 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9907 lower_omp_taskreg (gsi_p
, ctx
);
9909 case GIMPLE_OMP_FOR
:
9910 ctx
= maybe_lookup_ctx (stmt
);
9912 if (ctx
->cancellable
)
9913 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9914 lower_omp_for (gsi_p
, ctx
);
9916 case GIMPLE_OMP_SECTIONS
:
9917 ctx
= maybe_lookup_ctx (stmt
);
9919 if (ctx
->cancellable
)
9920 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9921 lower_omp_sections (gsi_p
, ctx
);
9923 case GIMPLE_OMP_SINGLE
:
9924 ctx
= maybe_lookup_ctx (stmt
);
9926 lower_omp_single (gsi_p
, ctx
);
9928 case GIMPLE_OMP_MASTER
:
9929 ctx
= maybe_lookup_ctx (stmt
);
9931 lower_omp_master (gsi_p
, ctx
);
9933 case GIMPLE_OMP_TASKGROUP
:
9934 ctx
= maybe_lookup_ctx (stmt
);
9936 lower_omp_taskgroup (gsi_p
, ctx
);
9938 case GIMPLE_OMP_ORDERED
:
9939 ctx
= maybe_lookup_ctx (stmt
);
9941 lower_omp_ordered (gsi_p
, ctx
);
9943 case GIMPLE_OMP_CRITICAL
:
9944 ctx
= maybe_lookup_ctx (stmt
);
9946 lower_omp_critical (gsi_p
, ctx
);
9948 case GIMPLE_OMP_ATOMIC_LOAD
:
9949 if ((ctx
|| task_shared_vars
)
9950 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
9951 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
9952 gimple_regimplify_operands (stmt
, gsi_p
);
9954 case GIMPLE_OMP_TARGET
:
9955 ctx
= maybe_lookup_ctx (stmt
);
9957 lower_omp_target (gsi_p
, ctx
);
9959 case GIMPLE_OMP_TEAMS
:
9960 ctx
= maybe_lookup_ctx (stmt
);
9962 lower_omp_teams (gsi_p
, ctx
);
9966 fndecl
= gimple_call_fndecl (stmt
);
9968 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
9969 switch (DECL_FUNCTION_CODE (fndecl
))
9971 case BUILT_IN_GOMP_BARRIER
:
9975 case BUILT_IN_GOMP_CANCEL
:
9976 case BUILT_IN_GOMP_CANCELLATION_POINT
:
9979 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
9981 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
9982 if (!cctx
->cancellable
)
9984 if (DECL_FUNCTION_CODE (fndecl
)
9985 == BUILT_IN_GOMP_CANCELLATION_POINT
)
9987 stmt
= gimple_build_nop ();
9988 gsi_replace (gsi_p
, stmt
, false);
9993 lhs
= create_tmp_var (boolean_type_node
, NULL
);
9994 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
9996 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
9997 gimple_call_set_fndecl (stmt
, fndecl
);
9998 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10000 gimple_call_set_lhs (stmt
, lhs
);
10001 tree fallthru_label
;
10002 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10004 g
= gimple_build_label (fallthru_label
);
10005 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10006 g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
10007 cctx
->cancel_label
, fallthru_label
);
10008 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10015 if ((ctx
|| task_shared_vars
)
10016 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10018 gimple_regimplify_operands (stmt
, gsi_p
);
10024 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10026 location_t saved_location
= input_location
;
10027 gimple_stmt_iterator gsi
;
10028 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10029 lower_omp_1 (&gsi
, ctx
);
10030 /* Inside target region we haven't called fold_stmt during gimplification,
10031 because it can break code by adding decl references that weren't in the
10032 source. Call fold_stmt now. */
10033 if (target_nesting_level
)
10034 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10036 input_location
= saved_location
;
10039 /* Main entry point. */
10041 static unsigned int
10042 execute_lower_omp (void)
10046 /* This pass always runs, to provide PROP_gimple_lomp.
10047 But there is nothing to do unless -fopenmp is given. */
10048 if (flag_openmp
== 0)
10051 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10052 delete_omp_context
);
10054 body
= gimple_body (current_function_decl
);
10055 scan_omp (&body
, NULL
);
10056 gcc_assert (taskreg_nesting_level
== 0);
10058 if (all_contexts
->root
)
10060 struct gimplify_ctx gctx
;
10062 if (task_shared_vars
)
10063 push_gimplify_context (&gctx
);
10064 lower_omp (&body
, NULL
);
10065 if (task_shared_vars
)
10066 pop_gimplify_context (NULL
);
10071 splay_tree_delete (all_contexts
);
10072 all_contexts
= NULL
;
10074 BITMAP_FREE (task_shared_vars
);
10080 const pass_data pass_data_lower_omp
=
10082 GIMPLE_PASS
, /* type */
10083 "omplower", /* name */
10084 OPTGROUP_NONE
, /* optinfo_flags */
10085 false, /* has_gate */
10086 true, /* has_execute */
10087 TV_NONE
, /* tv_id */
10088 PROP_gimple_any
, /* properties_required */
10089 PROP_gimple_lomp
, /* properties_provided */
10090 0, /* properties_destroyed */
10091 0, /* todo_flags_start */
10092 0, /* todo_flags_finish */
10095 class pass_lower_omp
: public gimple_opt_pass
10098 pass_lower_omp (gcc::context
*ctxt
)
10099 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10102 /* opt_pass methods: */
10103 unsigned int execute () { return execute_lower_omp (); }
10105 }; // class pass_lower_omp
10107 } // anon namespace
10110 make_pass_lower_omp (gcc::context
*ctxt
)
10112 return new pass_lower_omp (ctxt
);
10115 /* The following is a utility to diagnose OpenMP structured block violations.
10116 It is not part of the "omplower" pass, as that's invoked too late. It
10117 should be invoked by the respective front ends after gimplification. */
10119 static splay_tree all_labels
;
10121 /* Check for mismatched contexts and generate an error if needed. Return
10122 true if an error is detected. */
10125 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10126 gimple branch_ctx
, gimple label_ctx
)
10128 if (label_ctx
== branch_ctx
)
10133 Previously we kept track of the label's entire context in diagnose_sb_[12]
10134 so we could traverse it and issue a correct "exit" or "enter" error
10135 message upon a structured block violation.
10137 We built the context by building a list with tree_cons'ing, but there is
10138 no easy counterpart in gimple tuples. It seems like far too much work
10139 for issuing exit/enter error messages. If someone really misses the
10140 distinct error message... patches welcome.
10144 /* Try to avoid confusing the user by producing and error message
10145 with correct "exit" or "enter" verbiage. We prefer "exit"
10146 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10147 if (branch_ctx
== NULL
)
10153 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10158 label_ctx
= TREE_CHAIN (label_ctx
);
10163 error ("invalid exit from OpenMP structured block");
10165 error ("invalid entry to OpenMP structured block");
10168 /* If it's obvious we have an invalid entry, be specific about the error. */
10169 if (branch_ctx
== NULL
)
10170 error ("invalid entry to OpenMP structured block");
10172 /* Otherwise, be vague and lazy, but efficient. */
10173 error ("invalid branch to/from an OpenMP structured block");
10175 gsi_replace (gsi_p
, gimple_build_nop (), false);
10179 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10180 where each label is found. */
10183 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10184 struct walk_stmt_info
*wi
)
10186 gimple context
= (gimple
) wi
->info
;
10187 gimple inner_context
;
10188 gimple stmt
= gsi_stmt (*gsi_p
);
10190 *handled_ops_p
= true;
10192 switch (gimple_code (stmt
))
10196 case GIMPLE_OMP_PARALLEL
:
10197 case GIMPLE_OMP_TASK
:
10198 case GIMPLE_OMP_SECTIONS
:
10199 case GIMPLE_OMP_SINGLE
:
10200 case GIMPLE_OMP_SECTION
:
10201 case GIMPLE_OMP_MASTER
:
10202 case GIMPLE_OMP_ORDERED
:
10203 case GIMPLE_OMP_CRITICAL
:
10204 case GIMPLE_OMP_TARGET
:
10205 case GIMPLE_OMP_TEAMS
:
10206 case GIMPLE_OMP_TASKGROUP
:
10207 /* The minimal context here is just the current OMP construct. */
10208 inner_context
= stmt
;
10209 wi
->info
= inner_context
;
10210 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10211 wi
->info
= context
;
10214 case GIMPLE_OMP_FOR
:
10215 inner_context
= stmt
;
10216 wi
->info
= inner_context
;
10217 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10219 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10220 diagnose_sb_1
, NULL
, wi
);
10221 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10222 wi
->info
= context
;
10226 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10227 (splay_tree_value
) context
);
10237 /* Pass 2: Check each branch and see if its context differs from that of
10238 the destination label's context. */
10241 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10242 struct walk_stmt_info
*wi
)
10244 gimple context
= (gimple
) wi
->info
;
10246 gimple stmt
= gsi_stmt (*gsi_p
);
10248 *handled_ops_p
= true;
10250 switch (gimple_code (stmt
))
10254 case GIMPLE_OMP_PARALLEL
:
10255 case GIMPLE_OMP_TASK
:
10256 case GIMPLE_OMP_SECTIONS
:
10257 case GIMPLE_OMP_SINGLE
:
10258 case GIMPLE_OMP_SECTION
:
10259 case GIMPLE_OMP_MASTER
:
10260 case GIMPLE_OMP_ORDERED
:
10261 case GIMPLE_OMP_CRITICAL
:
10262 case GIMPLE_OMP_TARGET
:
10263 case GIMPLE_OMP_TEAMS
:
10264 case GIMPLE_OMP_TASKGROUP
:
10266 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10267 wi
->info
= context
;
10270 case GIMPLE_OMP_FOR
:
10272 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10274 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10275 diagnose_sb_2
, NULL
, wi
);
10276 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10277 wi
->info
= context
;
10282 tree lab
= gimple_cond_true_label (stmt
);
10285 n
= splay_tree_lookup (all_labels
,
10286 (splay_tree_key
) lab
);
10287 diagnose_sb_0 (gsi_p
, context
,
10288 n
? (gimple
) n
->value
: NULL
);
10290 lab
= gimple_cond_false_label (stmt
);
10293 n
= splay_tree_lookup (all_labels
,
10294 (splay_tree_key
) lab
);
10295 diagnose_sb_0 (gsi_p
, context
,
10296 n
? (gimple
) n
->value
: NULL
);
10303 tree lab
= gimple_goto_dest (stmt
);
10304 if (TREE_CODE (lab
) != LABEL_DECL
)
10307 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10308 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10312 case GIMPLE_SWITCH
:
10315 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10317 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10318 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10319 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10325 case GIMPLE_RETURN
:
10326 diagnose_sb_0 (gsi_p
, context
, NULL
);
10336 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10339 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
)
10341 gimple last
= last_stmt (bb
);
10342 enum gimple_code code
= gimple_code (last
);
10343 struct omp_region
*cur_region
= *region
;
10344 bool fallthru
= false;
10348 case GIMPLE_OMP_PARALLEL
:
10349 case GIMPLE_OMP_TASK
:
10350 case GIMPLE_OMP_FOR
:
10351 case GIMPLE_OMP_SINGLE
:
10352 case GIMPLE_OMP_TEAMS
:
10353 case GIMPLE_OMP_MASTER
:
10354 case GIMPLE_OMP_TASKGROUP
:
10355 case GIMPLE_OMP_ORDERED
:
10356 case GIMPLE_OMP_CRITICAL
:
10357 case GIMPLE_OMP_SECTION
:
10358 cur_region
= new_omp_region (bb
, code
, cur_region
);
10362 case GIMPLE_OMP_TARGET
:
10363 cur_region
= new_omp_region (bb
, code
, cur_region
);
10365 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10366 cur_region
= cur_region
->outer
;
10369 case GIMPLE_OMP_SECTIONS
:
10370 cur_region
= new_omp_region (bb
, code
, cur_region
);
10374 case GIMPLE_OMP_SECTIONS_SWITCH
:
10378 case GIMPLE_OMP_ATOMIC_LOAD
:
10379 case GIMPLE_OMP_ATOMIC_STORE
:
10383 case GIMPLE_OMP_RETURN
:
10384 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10385 somewhere other than the next block. This will be
10387 cur_region
->exit
= bb
;
10388 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10389 cur_region
= cur_region
->outer
;
10392 case GIMPLE_OMP_CONTINUE
:
10393 cur_region
->cont
= bb
;
10394 switch (cur_region
->type
)
10396 case GIMPLE_OMP_FOR
:
10397 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10398 succs edges as abnormal to prevent splitting
10400 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10401 /* Make the loopback edge. */
10402 make_edge (bb
, single_succ (cur_region
->entry
),
10405 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10406 corresponds to the case that the body of the loop
10407 is not executed at all. */
10408 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10409 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10413 case GIMPLE_OMP_SECTIONS
:
10414 /* Wire up the edges into and out of the nested sections. */
10416 basic_block switch_bb
= single_succ (cur_region
->entry
);
10418 struct omp_region
*i
;
10419 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10421 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10422 make_edge (switch_bb
, i
->entry
, 0);
10423 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10426 /* Make the loopback edge to the block with
10427 GIMPLE_OMP_SECTIONS_SWITCH. */
10428 make_edge (bb
, switch_bb
, 0);
10430 /* Make the edge from the switch to exit. */
10431 make_edge (switch_bb
, bb
->next_bb
, 0);
10437 gcc_unreachable ();
10442 gcc_unreachable ();
10445 if (*region
!= cur_region
)
10446 *region
= cur_region
;
10451 static unsigned int
10452 diagnose_omp_structured_block_errors (void)
10454 struct walk_stmt_info wi
;
10455 gimple_seq body
= gimple_body (current_function_decl
);
10457 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10459 memset (&wi
, 0, sizeof (wi
));
10460 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10462 memset (&wi
, 0, sizeof (wi
));
10463 wi
.want_locations
= true;
10464 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10466 gimple_set_body (current_function_decl
, body
);
10468 splay_tree_delete (all_labels
);
10475 gate_diagnose_omp_blocks (void)
10477 return flag_openmp
!= 0;
10482 const pass_data pass_data_diagnose_omp_blocks
=
10484 GIMPLE_PASS
, /* type */
10485 "*diagnose_omp_blocks", /* name */
10486 OPTGROUP_NONE
, /* optinfo_flags */
10487 true, /* has_gate */
10488 true, /* has_execute */
10489 TV_NONE
, /* tv_id */
10490 PROP_gimple_any
, /* properties_required */
10491 0, /* properties_provided */
10492 0, /* properties_destroyed */
10493 0, /* todo_flags_start */
10494 0, /* todo_flags_finish */
10497 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10500 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10501 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10504 /* opt_pass methods: */
10505 bool gate () { return gate_diagnose_omp_blocks (); }
10506 unsigned int execute () {
10507 return diagnose_omp_structured_block_errors ();
10510 }; // class pass_diagnose_omp_blocks
10512 } // anon namespace
10515 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10517 return new pass_diagnose_omp_blocks (ctxt
);
10520 #include "gt-omp-low.h"