1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
34 #include "gimple-iterator.h"
35 #include "gimplify-me.h"
36 #include "gimple-walk.h"
37 #include "tree-iterator.h"
38 #include "tree-inline.h"
39 #include "langhooks.h"
40 #include "diagnostic-core.h"
41 #include "gimple-ssa.h"
44 #include "tree-phinodes.h"
45 #include "ssa-iterators.h"
46 #include "tree-ssanames.h"
47 #include "tree-into-ssa.h"
54 #include "tree-pass.h"
57 #include "splay-tree.h"
62 #include "gimple-low.h"
63 #include "tree-cfgcleanup.h"
64 #include "tree-nested.h"
67 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
68 phases. The first phase scans the function looking for OMP statements
69 and then for variables that must be replaced to satisfy data sharing
70 clauses. The second phase expands code for the constructs, as well as
71 re-gimplifying things when variables have been replaced with complex
74 Final code generation is done by pass_expand_omp. The flowgraph is
75 scanned for parallel regions which are then moved to a new
76 function, to be invoked by the thread library. */
78 /* Parallel region information. Every parallel and workshare
79 directive is enclosed between two markers, the OMP_* directive
80 and a corresponding OMP_RETURN statement. */
84 /* The enclosing region. */
85 struct omp_region
*outer
;
87 /* First child region. */
88 struct omp_region
*inner
;
90 /* Next peer region. */
91 struct omp_region
*next
;
93 /* Block containing the omp directive as its last stmt. */
96 /* Block containing the OMP_RETURN as its last stmt. */
99 /* Block containing the OMP_CONTINUE as its last stmt. */
102 /* If this is a combined parallel+workshare region, this is a list
103 of additional arguments needed by the combined parallel+workshare
105 vec
<tree
, va_gc
> *ws_args
;
107 /* The code for the omp directive of this region. */
108 enum gimple_code type
;
110 /* Schedule kind, only used for OMP_FOR type regions. */
111 enum omp_clause_schedule_kind sched_kind
;
113 /* True if this is a combined parallel+workshare region. */
114 bool is_combined_parallel
;
117 /* Context structure. Used to store information about each parallel
118 directive in the code. */
120 typedef struct omp_context
122 /* This field must be at the beginning, as we do "inheritance": Some
123 callback functions for tree-inline.c (e.g., omp_copy_decl)
124 receive a copy_body_data pointer that is up-casted to an
125 omp_context pointer. */
128 /* The tree of contexts corresponding to the encountered constructs. */
129 struct omp_context
*outer
;
132 /* Map variables to fields in a structure that allows communication
133 between sending and receiving threads. */
134 splay_tree field_map
;
139 /* These are used just by task contexts, if task firstprivate fn is
140 needed. srecord_type is used to communicate from the thread
141 that encountered the task construct to task firstprivate fn,
142 record_type is allocated by GOMP_task, initialized by task firstprivate
143 fn and passed to the task body fn. */
144 splay_tree sfield_map
;
147 /* A chain of variables to add to the top-level block surrounding the
148 construct. In the case of a parallel, this is in the child function. */
151 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
152 barriers should jump to during omplower pass. */
155 /* What to do with variables with implicitly determined sharing
157 enum omp_clause_default_kind default_kind
;
159 /* Nesting depth of this context. Used to beautify error messages re
160 invalid gotos. The outermost ctx is depth 1, with depth 0 being
161 reserved for the main body of the function. */
164 /* True if this parallel directive is nested within another. */
167 /* True if this construct can be cancelled. */
172 struct omp_for_data_loop
174 tree v
, n1
, n2
, step
;
175 enum tree_code cond_code
;
178 /* A structure describing the main elements of a parallel loop. */
182 struct omp_for_data_loop loop
;
187 bool have_nowait
, have_ordered
;
188 enum omp_clause_schedule_kind sched_kind
;
189 struct omp_for_data_loop
*loops
;
193 static splay_tree all_contexts
;
194 static int taskreg_nesting_level
;
195 static int target_nesting_level
;
196 static struct omp_region
*root_omp_region
;
197 static bitmap task_shared_vars
;
199 static void scan_omp (gimple_seq
*, omp_context
*);
200 static tree
scan_omp_1_op (tree
*, int *, void *);
202 #define WALK_SUBSTMTS \
206 case GIMPLE_EH_FILTER: \
207 case GIMPLE_TRANSACTION: \
208 /* The sub-statements for these should be walked. */ \
209 *handled_ops_p = false; \
212 /* Convenience function for calling scan_omp_1_op on tree operands. */
215 scan_omp_op (tree
*tp
, omp_context
*ctx
)
217 struct walk_stmt_info wi
;
219 memset (&wi
, 0, sizeof (wi
));
221 wi
.want_locations
= true;
223 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
226 static void lower_omp (gimple_seq
*, omp_context
*);
227 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
228 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
230 /* Find an OpenMP clause of type KIND within CLAUSES. */
233 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
235 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
236 if (OMP_CLAUSE_CODE (clauses
) == kind
)
242 /* Return true if CTX is for an omp parallel. */
245 is_parallel_ctx (omp_context
*ctx
)
247 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
251 /* Return true if CTX is for an omp task. */
254 is_task_ctx (omp_context
*ctx
)
256 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
260 /* Return true if CTX is for an omp parallel or omp task. */
263 is_taskreg_ctx (omp_context
*ctx
)
265 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
266 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
270 /* Return true if REGION is a combined parallel+workshare region. */
273 is_combined_parallel (struct omp_region
*region
)
275 return region
->is_combined_parallel
;
279 /* Extract the header elements of parallel loop FOR_STMT and store
283 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
284 struct omp_for_data_loop
*loops
)
286 tree t
, var
, *collapse_iter
, *collapse_count
;
287 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
288 struct omp_for_data_loop
*loop
;
290 struct omp_for_data_loop dummy_loop
;
291 location_t loc
= gimple_location (for_stmt
);
292 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_KIND_SIMD
;
293 bool distribute
= gimple_omp_for_kind (for_stmt
)
294 == GF_OMP_FOR_KIND_DISTRIBUTE
;
296 fd
->for_stmt
= for_stmt
;
298 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
299 if (fd
->collapse
> 1)
302 fd
->loops
= &fd
->loop
;
304 fd
->have_nowait
= distribute
|| simd
;
305 fd
->have_ordered
= false;
306 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
307 fd
->chunk_size
= NULL_TREE
;
308 collapse_iter
= NULL
;
309 collapse_count
= NULL
;
311 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
312 switch (OMP_CLAUSE_CODE (t
))
314 case OMP_CLAUSE_NOWAIT
:
315 fd
->have_nowait
= true;
317 case OMP_CLAUSE_ORDERED
:
318 fd
->have_ordered
= true;
320 case OMP_CLAUSE_SCHEDULE
:
321 gcc_assert (!distribute
);
322 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
323 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
325 case OMP_CLAUSE_DIST_SCHEDULE
:
326 gcc_assert (distribute
);
327 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
329 case OMP_CLAUSE_COLLAPSE
:
330 if (fd
->collapse
> 1)
332 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
333 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
339 /* FIXME: for now map schedule(auto) to schedule(static).
340 There should be analysis to determine whether all iterations
341 are approximately the same amount of work (then schedule(static)
342 is best) or if it varies (then schedule(dynamic,N) is better). */
343 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
345 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
346 gcc_assert (fd
->chunk_size
== NULL
);
348 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
349 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
350 gcc_assert (fd
->chunk_size
== NULL
);
351 else if (fd
->chunk_size
== NULL
)
353 /* We only need to compute a default chunk size for ordered
354 static loops and dynamic loops. */
355 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
357 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
358 ? integer_zero_node
: integer_one_node
;
361 for (i
= 0; i
< fd
->collapse
; i
++)
363 if (fd
->collapse
== 1)
365 else if (loops
!= NULL
)
370 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
371 gcc_assert (SSA_VAR_P (loop
->v
));
372 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
373 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
374 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
375 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
377 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
378 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
379 switch (loop
->cond_code
)
385 gcc_assert (gimple_omp_for_kind (for_stmt
)
386 == GF_OMP_FOR_KIND_CILKSIMD
);
389 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
390 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
392 loop
->n2
= fold_build2_loc (loc
,
393 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
394 build_int_cst (TREE_TYPE (loop
->n2
), 1));
395 loop
->cond_code
= LT_EXPR
;
398 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
399 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
401 loop
->n2
= fold_build2_loc (loc
,
402 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
403 build_int_cst (TREE_TYPE (loop
->n2
), 1));
404 loop
->cond_code
= GT_EXPR
;
410 t
= gimple_omp_for_incr (for_stmt
, i
);
411 gcc_assert (TREE_OPERAND (t
, 0) == var
);
412 switch (TREE_CODE (t
))
415 loop
->step
= TREE_OPERAND (t
, 1);
417 case POINTER_PLUS_EXPR
:
418 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
421 loop
->step
= TREE_OPERAND (t
, 1);
422 loop
->step
= fold_build1_loc (loc
,
423 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
431 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
432 && !fd
->have_ordered
))
434 if (fd
->collapse
== 1)
435 iter_type
= TREE_TYPE (loop
->v
);
437 || TYPE_PRECISION (iter_type
)
438 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
440 = build_nonstandard_integer_type
441 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
443 else if (iter_type
!= long_long_unsigned_type_node
)
445 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
446 iter_type
= long_long_unsigned_type_node
;
447 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
448 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
449 >= TYPE_PRECISION (iter_type
))
453 if (loop
->cond_code
== LT_EXPR
)
454 n
= fold_build2_loc (loc
,
455 PLUS_EXPR
, TREE_TYPE (loop
->v
),
456 loop
->n2
, loop
->step
);
459 if (TREE_CODE (n
) != INTEGER_CST
460 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
461 iter_type
= long_long_unsigned_type_node
;
463 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
464 > TYPE_PRECISION (iter_type
))
468 if (loop
->cond_code
== LT_EXPR
)
471 n2
= fold_build2_loc (loc
,
472 PLUS_EXPR
, TREE_TYPE (loop
->v
),
473 loop
->n2
, loop
->step
);
477 n1
= fold_build2_loc (loc
,
478 MINUS_EXPR
, TREE_TYPE (loop
->v
),
479 loop
->n2
, loop
->step
);
482 if (TREE_CODE (n1
) != INTEGER_CST
483 || TREE_CODE (n2
) != INTEGER_CST
484 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
485 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
486 iter_type
= long_long_unsigned_type_node
;
490 if (collapse_count
&& *collapse_count
== NULL
)
492 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
493 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
494 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
495 if (t
&& integer_zerop (t
))
496 count
= build_zero_cst (long_long_unsigned_type_node
);
497 else if ((i
== 0 || count
!= NULL_TREE
)
498 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
499 && TREE_CONSTANT (loop
->n1
)
500 && TREE_CONSTANT (loop
->n2
)
501 && TREE_CODE (loop
->step
) == INTEGER_CST
)
503 tree itype
= TREE_TYPE (loop
->v
);
505 if (POINTER_TYPE_P (itype
))
506 itype
= signed_type_for (itype
);
507 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
508 t
= fold_build2_loc (loc
,
510 fold_convert_loc (loc
, itype
, loop
->step
), t
);
511 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
512 fold_convert_loc (loc
, itype
, loop
->n2
));
513 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
514 fold_convert_loc (loc
, itype
, loop
->n1
));
515 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
516 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
517 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
518 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
519 fold_convert_loc (loc
, itype
,
522 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
523 fold_convert_loc (loc
, itype
, loop
->step
));
524 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
525 if (count
!= NULL_TREE
)
526 count
= fold_build2_loc (loc
,
527 MULT_EXPR
, long_long_unsigned_type_node
,
531 if (TREE_CODE (count
) != INTEGER_CST
)
534 else if (count
&& !integer_zerop (count
))
541 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
542 || fd
->have_ordered
))
544 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
545 iter_type
= long_long_unsigned_type_node
;
547 iter_type
= long_integer_type_node
;
549 else if (collapse_iter
&& *collapse_iter
!= NULL
)
550 iter_type
= TREE_TYPE (*collapse_iter
);
551 fd
->iter_type
= iter_type
;
552 if (collapse_iter
&& *collapse_iter
== NULL
)
553 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
554 if (collapse_count
&& *collapse_count
== NULL
)
557 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
559 *collapse_count
= create_tmp_var (iter_type
, ".count");
562 if (fd
->collapse
> 1)
564 fd
->loop
.v
= *collapse_iter
;
565 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
566 fd
->loop
.n2
= *collapse_count
;
567 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
568 fd
->loop
.cond_code
= LT_EXPR
;
573 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
574 is the immediate dominator of PAR_ENTRY_BB, return true if there
575 are no data dependencies that would prevent expanding the parallel
576 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
578 When expanding a combined parallel+workshare region, the call to
579 the child function may need additional arguments in the case of
580 GIMPLE_OMP_FOR regions. In some cases, these arguments are
581 computed out of variables passed in from the parent to the child
582 via 'struct .omp_data_s'. For instance:
584 #pragma omp parallel for schedule (guided, i * 4)
589 # BLOCK 2 (PAR_ENTRY_BB)
591 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
593 # BLOCK 3 (WS_ENTRY_BB)
594 .omp_data_i = &.omp_data_o;
595 D.1667 = .omp_data_i->i;
597 #pragma omp for schedule (guided, D.1598)
599 When we outline the parallel region, the call to the child function
600 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
601 that value is computed *after* the call site. So, in principle we
602 cannot do the transformation.
604 To see whether the code in WS_ENTRY_BB blocks the combined
605 parallel+workshare call, we collect all the variables used in the
606 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
607 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
610 FIXME. If we had the SSA form built at this point, we could merely
611 hoist the code in block 3 into block 2 and be done with it. But at
612 this point we don't have dataflow information and though we could
613 hack something up here, it is really not worth the aggravation. */
616 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
618 struct omp_for_data fd
;
619 gimple ws_stmt
= last_stmt (ws_entry_bb
);
621 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
624 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
626 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
628 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
630 if (fd
.iter_type
!= long_integer_type_node
)
633 /* FIXME. We give up too easily here. If any of these arguments
634 are not constants, they will likely involve variables that have
635 been mapped into fields of .omp_data_s for sharing with the child
636 function. With appropriate data flow, it would be possible to
638 if (!is_gimple_min_invariant (fd
.loop
.n1
)
639 || !is_gimple_min_invariant (fd
.loop
.n2
)
640 || !is_gimple_min_invariant (fd
.loop
.step
)
641 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
648 /* Collect additional arguments needed to emit a combined
649 parallel+workshare call. WS_STMT is the workshare directive being
652 static vec
<tree
, va_gc
> *
653 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
656 location_t loc
= gimple_location (ws_stmt
);
657 vec
<tree
, va_gc
> *ws_args
;
659 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
661 struct omp_for_data fd
;
664 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
668 if (gimple_omp_for_combined_into_p (ws_stmt
))
671 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
672 OMP_CLAUSE__LOOPTEMP_
);
674 n1
= OMP_CLAUSE_DECL (innerc
);
675 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
676 OMP_CLAUSE__LOOPTEMP_
);
678 n2
= OMP_CLAUSE_DECL (innerc
);
681 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
683 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
684 ws_args
->quick_push (t
);
686 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
687 ws_args
->quick_push (t
);
689 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
690 ws_args
->quick_push (t
);
694 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
695 ws_args
->quick_push (t
);
700 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
702 /* Number of sections is equal to the number of edges from the
703 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
704 the exit of the sections region. */
705 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
706 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
707 vec_alloc (ws_args
, 1);
708 ws_args
->quick_push (t
);
716 /* Discover whether REGION is a combined parallel+workshare region. */
719 determine_parallel_type (struct omp_region
*region
)
721 basic_block par_entry_bb
, par_exit_bb
;
722 basic_block ws_entry_bb
, ws_exit_bb
;
724 if (region
== NULL
|| region
->inner
== NULL
725 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
726 || region
->inner
->cont
== NULL
)
729 /* We only support parallel+for and parallel+sections. */
730 if (region
->type
!= GIMPLE_OMP_PARALLEL
731 || (region
->inner
->type
!= GIMPLE_OMP_FOR
732 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
735 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
736 WS_EXIT_BB -> PAR_EXIT_BB. */
737 par_entry_bb
= region
->entry
;
738 par_exit_bb
= region
->exit
;
739 ws_entry_bb
= region
->inner
->entry
;
740 ws_exit_bb
= region
->inner
->exit
;
742 if (single_succ (par_entry_bb
) == ws_entry_bb
743 && single_succ (ws_exit_bb
) == par_exit_bb
744 && workshare_safe_to_combine_p (ws_entry_bb
)
745 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
746 || (last_and_only_stmt (ws_entry_bb
)
747 && last_and_only_stmt (par_exit_bb
))))
749 gimple par_stmt
= last_stmt (par_entry_bb
);
750 gimple ws_stmt
= last_stmt (ws_entry_bb
);
752 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
754 /* If this is a combined parallel loop, we need to determine
755 whether or not to use the combined library calls. There
756 are two cases where we do not apply the transformation:
757 static loops and any kind of ordered loop. In the first
758 case, we already open code the loop so there is no need
759 to do anything else. In the latter case, the combined
760 parallel loop call would still need extra synchronization
761 to implement ordered semantics, so there would not be any
762 gain in using the combined call. */
763 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
764 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
766 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
767 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
769 region
->is_combined_parallel
= false;
770 region
->inner
->is_combined_parallel
= false;
775 region
->is_combined_parallel
= true;
776 region
->inner
->is_combined_parallel
= true;
777 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
782 /* Return true if EXPR is variable sized. */
785 is_variable_sized (const_tree expr
)
787 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
790 /* Return true if DECL is a reference type. */
793 is_reference (tree decl
)
795 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
798 /* Lookup variables in the decl or field splay trees. The "maybe" form
799 allows for the variable form to not have been entered, otherwise we
800 assert that the variable must have been entered. */
803 lookup_decl (tree var
, omp_context
*ctx
)
806 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
811 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
814 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
815 return n
? *n
: NULL_TREE
;
819 lookup_field (tree var
, omp_context
*ctx
)
822 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
823 return (tree
) n
->value
;
827 lookup_sfield (tree var
, omp_context
*ctx
)
830 n
= splay_tree_lookup (ctx
->sfield_map
831 ? ctx
->sfield_map
: ctx
->field_map
,
832 (splay_tree_key
) var
);
833 return (tree
) n
->value
;
837 maybe_lookup_field (tree var
, omp_context
*ctx
)
840 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
841 return n
? (tree
) n
->value
: NULL_TREE
;
844 /* Return true if DECL should be copied by pointer. SHARED_CTX is
845 the parallel context if DECL is to be shared. */
848 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
850 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
853 /* We can only use copy-in/copy-out semantics for shared variables
854 when we know the value is not accessible from an outer scope. */
857 /* ??? Trivially accessible from anywhere. But why would we even
858 be passing an address in this case? Should we simply assert
859 this to be false, or should we have a cleanup pass that removes
860 these from the list of mappings? */
861 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
864 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
865 without analyzing the expression whether or not its location
866 is accessible to anyone else. In the case of nested parallel
867 regions it certainly may be. */
868 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
871 /* Do not use copy-in/copy-out for variables that have their
873 if (TREE_ADDRESSABLE (decl
))
876 /* lower_send_shared_vars only uses copy-in, but not copy-out
878 if (TREE_READONLY (decl
)
879 || ((TREE_CODE (decl
) == RESULT_DECL
880 || TREE_CODE (decl
) == PARM_DECL
)
881 && DECL_BY_REFERENCE (decl
)))
884 /* Disallow copy-in/out in nested parallel if
885 decl is shared in outer parallel, otherwise
886 each thread could store the shared variable
887 in its own copy-in location, making the
888 variable no longer really shared. */
889 if (shared_ctx
->is_nested
)
893 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
894 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
901 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
902 c
; c
= OMP_CLAUSE_CHAIN (c
))
903 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
904 && OMP_CLAUSE_DECL (c
) == decl
)
908 goto maybe_mark_addressable_and_ret
;
912 /* For tasks avoid using copy-in/out. As tasks can be
913 deferred or executed in different thread, when GOMP_task
914 returns, the task hasn't necessarily terminated. */
915 if (is_task_ctx (shared_ctx
))
918 maybe_mark_addressable_and_ret
:
919 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
920 if (is_gimple_reg (outer
))
922 /* Taking address of OUTER in lower_send_shared_vars
923 might need regimplification of everything that uses the
925 if (!task_shared_vars
)
926 task_shared_vars
= BITMAP_ALLOC (NULL
);
927 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
928 TREE_ADDRESSABLE (outer
) = 1;
937 /* Construct a new automatic decl similar to VAR. */
940 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
942 tree copy
= copy_var_decl (var
, name
, type
);
944 DECL_CONTEXT (copy
) = current_function_decl
;
945 DECL_CHAIN (copy
) = ctx
->block_vars
;
946 ctx
->block_vars
= copy
;
952 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
954 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
957 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
960 omp_build_component_ref (tree obj
, tree field
)
962 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
963 if (TREE_THIS_VOLATILE (field
))
964 TREE_THIS_VOLATILE (ret
) |= 1;
965 if (TREE_READONLY (field
))
966 TREE_READONLY (ret
) |= 1;
970 /* Build tree nodes to access the field for VAR on the receiver side. */
973 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
975 tree x
, field
= lookup_field (var
, ctx
);
977 /* If the receiver record type was remapped in the child function,
978 remap the field into the new record type. */
979 x
= maybe_lookup_field (field
, ctx
);
983 x
= build_simple_mem_ref (ctx
->receiver_decl
);
984 x
= omp_build_component_ref (x
, field
);
986 x
= build_simple_mem_ref (x
);
991 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
992 of a parallel, this is a component reference; for workshare constructs
993 this is some variable. */
996 build_outer_var_ref (tree var
, omp_context
*ctx
)
1000 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1002 else if (is_variable_sized (var
))
1004 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1005 x
= build_outer_var_ref (x
, ctx
);
1006 x
= build_simple_mem_ref (x
);
1008 else if (is_taskreg_ctx (ctx
))
1010 bool by_ref
= use_pointer_for_field (var
, NULL
);
1011 x
= build_receiver_ref (var
, by_ref
, ctx
);
1013 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1014 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
1016 /* #pragma omp simd isn't a worksharing construct, and can reference even
1017 private vars in its linear etc. clauses. */
1019 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1020 x
= lookup_decl (var
, ctx
->outer
);
1021 else if (ctx
->outer
)
1022 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1026 else if (ctx
->outer
)
1027 x
= lookup_decl (var
, ctx
->outer
);
1028 else if (is_reference (var
))
1029 /* This can happen with orphaned constructs. If var is reference, it is
1030 possible it is shared and as such valid. */
1035 if (is_reference (var
))
1036 x
= build_simple_mem_ref (x
);
1041 /* Build tree nodes to access the field for VAR on the sender side. */
1044 build_sender_ref (tree var
, omp_context
*ctx
)
1046 tree field
= lookup_sfield (var
, ctx
);
1047 return omp_build_component_ref (ctx
->sender_decl
, field
);
1050 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1053 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1055 tree field
, type
, sfield
= NULL_TREE
;
1057 gcc_assert ((mask
& 1) == 0
1058 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1059 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1060 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1062 type
= TREE_TYPE (var
);
1065 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1066 type
= build_pointer_type (build_pointer_type (type
));
1069 type
= build_pointer_type (type
);
1070 else if ((mask
& 3) == 1 && is_reference (var
))
1071 type
= TREE_TYPE (type
);
1073 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1074 FIELD_DECL
, DECL_NAME (var
), type
);
1076 /* Remember what variable this field was created for. This does have a
1077 side effect of making dwarf2out ignore this member, so for helpful
1078 debugging we clear it later in delete_omp_context. */
1079 DECL_ABSTRACT_ORIGIN (field
) = var
;
1080 if (type
== TREE_TYPE (var
))
1082 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1083 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1084 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1087 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1089 if ((mask
& 3) == 3)
1091 insert_field_into_struct (ctx
->record_type
, field
);
1092 if (ctx
->srecord_type
)
1094 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1095 FIELD_DECL
, DECL_NAME (var
), type
);
1096 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1097 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1098 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1099 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1100 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1105 if (ctx
->srecord_type
== NULL_TREE
)
1109 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1110 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1111 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1113 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1114 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1115 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1116 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1117 splay_tree_insert (ctx
->sfield_map
,
1118 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1119 (splay_tree_value
) sfield
);
1123 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1124 : ctx
->srecord_type
, field
);
1128 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1129 (splay_tree_value
) field
);
1130 if ((mask
& 2) && ctx
->sfield_map
)
1131 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1132 (splay_tree_value
) sfield
);
1136 install_var_local (tree var
, omp_context
*ctx
)
1138 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1139 insert_decl_map (&ctx
->cb
, var
, new_var
);
1143 /* Adjust the replacement for DECL in CTX for the new context. This means
1144 copying the DECL_VALUE_EXPR, and fixing up the type. */
1147 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1149 tree new_decl
, size
;
1151 new_decl
= lookup_decl (decl
, ctx
);
1153 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1155 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1156 && DECL_HAS_VALUE_EXPR_P (decl
))
1158 tree ve
= DECL_VALUE_EXPR (decl
);
1159 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1160 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1161 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1164 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1166 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1167 if (size
== error_mark_node
)
1168 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1169 DECL_SIZE (new_decl
) = size
;
1171 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1172 if (size
== error_mark_node
)
1173 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1174 DECL_SIZE_UNIT (new_decl
) = size
;
1178 /* The callback for remap_decl. Search all containing contexts for a
1179 mapping of the variable; this avoids having to duplicate the splay
1180 tree ahead of time. We know a mapping doesn't already exist in the
1181 given context. Create new mappings to implement default semantics. */
1184 omp_copy_decl (tree var
, copy_body_data
*cb
)
1186 omp_context
*ctx
= (omp_context
*) cb
;
1189 if (TREE_CODE (var
) == LABEL_DECL
)
1191 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1192 DECL_CONTEXT (new_var
) = current_function_decl
;
1193 insert_decl_map (&ctx
->cb
, var
, new_var
);
1197 while (!is_taskreg_ctx (ctx
))
1202 new_var
= maybe_lookup_decl (var
, ctx
);
1207 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1210 return error_mark_node
;
1214 /* Return the parallel region associated with STMT. */
1216 /* Debugging dumps for parallel regions. */
1217 void dump_omp_region (FILE *, struct omp_region
*, int);
1218 void debug_omp_region (struct omp_region
*);
1219 void debug_all_omp_regions (void);
1221 /* Dump the parallel region tree rooted at REGION. */
1224 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1226 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1227 gimple_code_name
[region
->type
]);
1230 dump_omp_region (file
, region
->inner
, indent
+ 4);
1234 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1235 region
->cont
->index
);
1239 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1240 region
->exit
->index
);
1242 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1245 dump_omp_region (file
, region
->next
, indent
);
1249 debug_omp_region (struct omp_region
*region
)
1251 dump_omp_region (stderr
, region
, 0);
1255 debug_all_omp_regions (void)
1257 dump_omp_region (stderr
, root_omp_region
, 0);
1261 /* Create a new parallel region starting at STMT inside region PARENT. */
1263 static struct omp_region
*
1264 new_omp_region (basic_block bb
, enum gimple_code type
,
1265 struct omp_region
*parent
)
1267 struct omp_region
*region
= XCNEW (struct omp_region
);
1269 region
->outer
= parent
;
1271 region
->type
= type
;
1275 /* This is a nested region. Add it to the list of inner
1276 regions in PARENT. */
1277 region
->next
= parent
->inner
;
1278 parent
->inner
= region
;
1282 /* This is a toplevel region. Add it to the list of toplevel
1283 regions in ROOT_OMP_REGION. */
1284 region
->next
= root_omp_region
;
1285 root_omp_region
= region
;
1291 /* Release the memory associated with the region tree rooted at REGION. */
1294 free_omp_region_1 (struct omp_region
*region
)
1296 struct omp_region
*i
, *n
;
1298 for (i
= region
->inner
; i
; i
= n
)
1301 free_omp_region_1 (i
);
1307 /* Release the memory for the entire omp region tree. */
1310 free_omp_regions (void)
1312 struct omp_region
*r
, *n
;
1313 for (r
= root_omp_region
; r
; r
= n
)
1316 free_omp_region_1 (r
);
1318 root_omp_region
= NULL
;
1322 /* Create a new context, with OUTER_CTX being the surrounding context. */
1324 static omp_context
*
1325 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1327 omp_context
*ctx
= XCNEW (omp_context
);
1329 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1330 (splay_tree_value
) ctx
);
1335 ctx
->outer
= outer_ctx
;
1336 ctx
->cb
= outer_ctx
->cb
;
1337 ctx
->cb
.block
= NULL
;
1338 ctx
->depth
= outer_ctx
->depth
+ 1;
1342 ctx
->cb
.src_fn
= current_function_decl
;
1343 ctx
->cb
.dst_fn
= current_function_decl
;
1344 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1345 gcc_checking_assert (ctx
->cb
.src_node
);
1346 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1347 ctx
->cb
.src_cfun
= cfun
;
1348 ctx
->cb
.copy_decl
= omp_copy_decl
;
1349 ctx
->cb
.eh_lp_nr
= 0;
1350 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1354 ctx
->cb
.decl_map
= pointer_map_create ();
1359 static gimple_seq
maybe_catch_exception (gimple_seq
);
1361 /* Finalize task copyfn. */
1364 finalize_task_copyfn (gimple task_stmt
)
1366 struct function
*child_cfun
;
1368 gimple_seq seq
= NULL
, new_seq
;
1371 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1372 if (child_fn
== NULL_TREE
)
1375 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1376 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1378 push_cfun (child_cfun
);
1379 bind
= gimplify_body (child_fn
, false);
1380 gimple_seq_add_stmt (&seq
, bind
);
1381 new_seq
= maybe_catch_exception (seq
);
1384 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1386 gimple_seq_add_stmt (&seq
, bind
);
1388 gimple_set_body (child_fn
, seq
);
1391 /* Inform the callgraph about the new function. */
1392 cgraph_add_new_function (child_fn
, false);
1395 /* Destroy a omp_context data structures. Called through the splay tree
1396 value delete callback. */
1399 delete_omp_context (splay_tree_value value
)
1401 omp_context
*ctx
= (omp_context
*) value
;
1403 pointer_map_destroy (ctx
->cb
.decl_map
);
1406 splay_tree_delete (ctx
->field_map
);
1407 if (ctx
->sfield_map
)
1408 splay_tree_delete (ctx
->sfield_map
);
1410 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1411 it produces corrupt debug information. */
1412 if (ctx
->record_type
)
1415 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1416 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1418 if (ctx
->srecord_type
)
1421 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1422 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1425 if (is_task_ctx (ctx
))
1426 finalize_task_copyfn (ctx
->stmt
);
1431 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1435 fixup_child_record_type (omp_context
*ctx
)
1437 tree f
, type
= ctx
->record_type
;
1439 /* ??? It isn't sufficient to just call remap_type here, because
1440 variably_modified_type_p doesn't work the way we expect for
1441 record types. Testing each field for whether it needs remapping
1442 and creating a new record by hand works, however. */
1443 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1444 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1448 tree name
, new_fields
= NULL
;
1450 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1451 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1452 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1453 TYPE_DECL
, name
, type
);
1454 TYPE_NAME (type
) = name
;
1456 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1458 tree new_f
= copy_node (f
);
1459 DECL_CONTEXT (new_f
) = type
;
1460 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1461 DECL_CHAIN (new_f
) = new_fields
;
1462 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1463 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1465 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1469 /* Arrange to be able to look up the receiver field
1470 given the sender field. */
1471 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1472 (splay_tree_value
) new_f
);
1474 TYPE_FIELDS (type
) = nreverse (new_fields
);
1478 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1481 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1482 specified by CLAUSES. */
1485 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1488 bool scan_array_reductions
= false;
1490 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1494 switch (OMP_CLAUSE_CODE (c
))
1496 case OMP_CLAUSE_PRIVATE
:
1497 decl
= OMP_CLAUSE_DECL (c
);
1498 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1500 else if (!is_variable_sized (decl
))
1501 install_var_local (decl
, ctx
);
1504 case OMP_CLAUSE_SHARED
:
1505 /* Ignore shared directives in teams construct. */
1506 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1508 gcc_assert (is_taskreg_ctx (ctx
));
1509 decl
= OMP_CLAUSE_DECL (c
);
1510 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1511 || !is_variable_sized (decl
));
1512 /* Global variables don't need to be copied,
1513 the receiver side will use them directly. */
1514 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1516 by_ref
= use_pointer_for_field (decl
, ctx
);
1517 if (! TREE_READONLY (decl
)
1518 || TREE_ADDRESSABLE (decl
)
1520 || is_reference (decl
))
1522 install_var_field (decl
, by_ref
, 3, ctx
);
1523 install_var_local (decl
, ctx
);
1526 /* We don't need to copy const scalar vars back. */
1527 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1530 case OMP_CLAUSE_LASTPRIVATE
:
1531 /* Let the corresponding firstprivate clause create
1533 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1537 case OMP_CLAUSE_FIRSTPRIVATE
:
1538 case OMP_CLAUSE_REDUCTION
:
1539 case OMP_CLAUSE_LINEAR
:
1540 decl
= OMP_CLAUSE_DECL (c
);
1542 if (is_variable_sized (decl
))
1544 if (is_task_ctx (ctx
))
1545 install_var_field (decl
, false, 1, ctx
);
1548 else if (is_taskreg_ctx (ctx
))
1551 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1552 by_ref
= use_pointer_for_field (decl
, NULL
);
1554 if (is_task_ctx (ctx
)
1555 && (global
|| by_ref
|| is_reference (decl
)))
1557 install_var_field (decl
, false, 1, ctx
);
1559 install_var_field (decl
, by_ref
, 2, ctx
);
1562 install_var_field (decl
, by_ref
, 3, ctx
);
1564 install_var_local (decl
, ctx
);
1567 case OMP_CLAUSE__LOOPTEMP_
:
1568 gcc_assert (is_parallel_ctx (ctx
));
1569 decl
= OMP_CLAUSE_DECL (c
);
1570 install_var_field (decl
, false, 3, ctx
);
1571 install_var_local (decl
, ctx
);
1574 case OMP_CLAUSE_COPYPRIVATE
:
1575 case OMP_CLAUSE_COPYIN
:
1576 decl
= OMP_CLAUSE_DECL (c
);
1577 by_ref
= use_pointer_for_field (decl
, NULL
);
1578 install_var_field (decl
, by_ref
, 3, ctx
);
1581 case OMP_CLAUSE_DEFAULT
:
1582 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1585 case OMP_CLAUSE_FINAL
:
1587 case OMP_CLAUSE_NUM_THREADS
:
1588 case OMP_CLAUSE_NUM_TEAMS
:
1589 case OMP_CLAUSE_THREAD_LIMIT
:
1590 case OMP_CLAUSE_DEVICE
:
1591 case OMP_CLAUSE_SCHEDULE
:
1592 case OMP_CLAUSE_DIST_SCHEDULE
:
1593 case OMP_CLAUSE_DEPEND
:
1595 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1599 case OMP_CLAUSE_FROM
:
1600 case OMP_CLAUSE_MAP
:
1602 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1603 decl
= OMP_CLAUSE_DECL (c
);
1604 /* Global variables with "omp declare target" attribute
1605 don't need to be copied, the receiver side will use them
1607 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1609 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1610 && lookup_attribute ("omp declare target",
1611 DECL_ATTRIBUTES (decl
)))
1613 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1614 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1616 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1617 #pragma omp target data, there is nothing to map for
1619 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1620 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1625 if (DECL_SIZE (decl
)
1626 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1628 tree decl2
= DECL_VALUE_EXPR (decl
);
1629 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1630 decl2
= TREE_OPERAND (decl2
, 0);
1631 gcc_assert (DECL_P (decl2
));
1632 install_var_field (decl2
, true, 3, ctx
);
1633 install_var_local (decl2
, ctx
);
1634 install_var_local (decl
, ctx
);
1638 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1639 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1640 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1641 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1642 install_var_field (decl
, true, 7, ctx
);
1644 install_var_field (decl
, true, 3, ctx
);
1645 if (gimple_omp_target_kind (ctx
->stmt
)
1646 == GF_OMP_TARGET_KIND_REGION
)
1647 install_var_local (decl
, ctx
);
1652 tree base
= get_base_address (decl
);
1653 tree nc
= OMP_CLAUSE_CHAIN (c
);
1656 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1657 && OMP_CLAUSE_DECL (nc
) == base
1658 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1659 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1661 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1662 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1666 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1667 (splay_tree_key
) decl
));
1669 = build_decl (OMP_CLAUSE_LOCATION (c
),
1670 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1671 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1672 insert_field_into_struct (ctx
->record_type
, field
);
1673 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1674 (splay_tree_value
) field
);
1679 case OMP_CLAUSE_NOWAIT
:
1680 case OMP_CLAUSE_ORDERED
:
1681 case OMP_CLAUSE_COLLAPSE
:
1682 case OMP_CLAUSE_UNTIED
:
1683 case OMP_CLAUSE_MERGEABLE
:
1684 case OMP_CLAUSE_PROC_BIND
:
1685 case OMP_CLAUSE_SAFELEN
:
1688 case OMP_CLAUSE_ALIGNED
:
1689 decl
= OMP_CLAUSE_DECL (c
);
1690 if (is_global_var (decl
)
1691 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1692 install_var_local (decl
, ctx
);
1700 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1702 switch (OMP_CLAUSE_CODE (c
))
1704 case OMP_CLAUSE_LASTPRIVATE
:
1705 /* Let the corresponding firstprivate clause create
1707 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1708 scan_array_reductions
= true;
1709 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1713 case OMP_CLAUSE_PRIVATE
:
1714 case OMP_CLAUSE_FIRSTPRIVATE
:
1715 case OMP_CLAUSE_REDUCTION
:
1716 case OMP_CLAUSE_LINEAR
:
1717 decl
= OMP_CLAUSE_DECL (c
);
1718 if (is_variable_sized (decl
))
1719 install_var_local (decl
, ctx
);
1720 fixup_remapped_decl (decl
, ctx
,
1721 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1722 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1723 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1724 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1725 scan_array_reductions
= true;
1728 case OMP_CLAUSE_SHARED
:
1729 /* Ignore shared directives in teams construct. */
1730 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1732 decl
= OMP_CLAUSE_DECL (c
);
1733 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1734 fixup_remapped_decl (decl
, ctx
, false);
1737 case OMP_CLAUSE_MAP
:
1738 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1740 decl
= OMP_CLAUSE_DECL (c
);
1742 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1743 && lookup_attribute ("omp declare target",
1744 DECL_ATTRIBUTES (decl
)))
1748 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1749 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1750 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1752 tree new_decl
= lookup_decl (decl
, ctx
);
1753 TREE_TYPE (new_decl
)
1754 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1756 else if (DECL_SIZE (decl
)
1757 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1759 tree decl2
= DECL_VALUE_EXPR (decl
);
1760 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1761 decl2
= TREE_OPERAND (decl2
, 0);
1762 gcc_assert (DECL_P (decl2
));
1763 fixup_remapped_decl (decl2
, ctx
, false);
1764 fixup_remapped_decl (decl
, ctx
, true);
1767 fixup_remapped_decl (decl
, ctx
, false);
1771 case OMP_CLAUSE_COPYPRIVATE
:
1772 case OMP_CLAUSE_COPYIN
:
1773 case OMP_CLAUSE_DEFAULT
:
1775 case OMP_CLAUSE_NUM_THREADS
:
1776 case OMP_CLAUSE_NUM_TEAMS
:
1777 case OMP_CLAUSE_THREAD_LIMIT
:
1778 case OMP_CLAUSE_DEVICE
:
1779 case OMP_CLAUSE_SCHEDULE
:
1780 case OMP_CLAUSE_DIST_SCHEDULE
:
1781 case OMP_CLAUSE_NOWAIT
:
1782 case OMP_CLAUSE_ORDERED
:
1783 case OMP_CLAUSE_COLLAPSE
:
1784 case OMP_CLAUSE_UNTIED
:
1785 case OMP_CLAUSE_FINAL
:
1786 case OMP_CLAUSE_MERGEABLE
:
1787 case OMP_CLAUSE_PROC_BIND
:
1788 case OMP_CLAUSE_SAFELEN
:
1789 case OMP_CLAUSE_ALIGNED
:
1790 case OMP_CLAUSE_DEPEND
:
1791 case OMP_CLAUSE__LOOPTEMP_
:
1793 case OMP_CLAUSE_FROM
:
1801 if (scan_array_reductions
)
1802 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1803 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1804 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1806 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1807 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1809 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1810 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1811 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1814 /* Create a new name for omp child function. Returns an identifier. */
1816 static GTY(()) unsigned int tmp_ompfn_id_num
;
1819 create_omp_child_function_name (bool task_copy
)
1821 return (clone_function_name (current_function_decl
,
1822 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1825 /* Build a decl for the omp child function. It'll not contain a body
1826 yet, just the bare decl. */
1829 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1831 tree decl
, type
, name
, t
;
1833 name
= create_omp_child_function_name (task_copy
);
1835 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1836 ptr_type_node
, NULL_TREE
);
1838 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1840 decl
= build_decl (gimple_location (ctx
->stmt
),
1841 FUNCTION_DECL
, name
, type
);
1844 ctx
->cb
.dst_fn
= decl
;
1846 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1848 TREE_STATIC (decl
) = 1;
1849 TREE_USED (decl
) = 1;
1850 DECL_ARTIFICIAL (decl
) = 1;
1851 DECL_NAMELESS (decl
) = 1;
1852 DECL_IGNORED_P (decl
) = 0;
1853 TREE_PUBLIC (decl
) = 0;
1854 DECL_UNINLINABLE (decl
) = 1;
1855 DECL_EXTERNAL (decl
) = 0;
1856 DECL_CONTEXT (decl
) = NULL_TREE
;
1857 DECL_INITIAL (decl
) = make_node (BLOCK
);
1858 bool target_p
= false;
1859 if (lookup_attribute ("omp declare target",
1860 DECL_ATTRIBUTES (current_function_decl
)))
1865 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1866 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1867 && gimple_omp_target_kind (octx
->stmt
)
1868 == GF_OMP_TARGET_KIND_REGION
)
1875 DECL_ATTRIBUTES (decl
)
1876 = tree_cons (get_identifier ("omp declare target"),
1877 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1879 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1880 RESULT_DECL
, NULL_TREE
, void_type_node
);
1881 DECL_ARTIFICIAL (t
) = 1;
1882 DECL_IGNORED_P (t
) = 1;
1883 DECL_CONTEXT (t
) = decl
;
1884 DECL_RESULT (decl
) = t
;
1886 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1887 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1888 DECL_ARTIFICIAL (t
) = 1;
1889 DECL_NAMELESS (t
) = 1;
1890 DECL_ARG_TYPE (t
) = ptr_type_node
;
1891 DECL_CONTEXT (t
) = current_function_decl
;
1893 DECL_ARGUMENTS (decl
) = t
;
1895 ctx
->receiver_decl
= t
;
1898 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1899 PARM_DECL
, get_identifier (".omp_data_o"),
1901 DECL_ARTIFICIAL (t
) = 1;
1902 DECL_NAMELESS (t
) = 1;
1903 DECL_ARG_TYPE (t
) = ptr_type_node
;
1904 DECL_CONTEXT (t
) = current_function_decl
;
1906 TREE_ADDRESSABLE (t
) = 1;
1907 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1908 DECL_ARGUMENTS (decl
) = t
;
1911 /* Allocate memory for the function structure. The call to
1912 allocate_struct_function clobbers CFUN, so we need to restore
1914 push_struct_function (decl
);
1915 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1919 /* Callback for walk_gimple_seq. Check if combined parallel
1920 contains gimple_omp_for_combined_into_p OMP_FOR. */
1923 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1924 bool *handled_ops_p
,
1925 struct walk_stmt_info
*wi
)
1927 gimple stmt
= gsi_stmt (*gsi_p
);
1929 *handled_ops_p
= true;
1930 switch (gimple_code (stmt
))
1934 case GIMPLE_OMP_FOR
:
1935 if (gimple_omp_for_combined_into_p (stmt
)
1936 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1939 return integer_zero_node
;
1948 /* Scan an OpenMP parallel directive. */
1951 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1955 gimple stmt
= gsi_stmt (*gsi
);
1957 /* Ignore parallel directives with empty bodies, unless there
1958 are copyin clauses. */
1960 && empty_body_p (gimple_omp_body (stmt
))
1961 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1962 OMP_CLAUSE_COPYIN
) == NULL
)
1964 gsi_replace (gsi
, gimple_build_nop (), false);
1968 if (gimple_omp_parallel_combined_p (stmt
))
1971 struct walk_stmt_info wi
;
1973 memset (&wi
, 0, sizeof (wi
));
1975 walk_gimple_seq (gimple_omp_body (stmt
),
1976 find_combined_for
, NULL
, &wi
);
1977 for_stmt
= (gimple
) wi
.info
;
1980 struct omp_for_data fd
;
1981 extract_omp_for_data (for_stmt
, &fd
, NULL
);
1982 /* We need two temporaries with fd.loop.v type (istart/iend)
1983 and then (fd.collapse - 1) temporaries with the same
1984 type for count2 ... countN-1 vars if not constant. */
1985 size_t count
= 2, i
;
1986 tree type
= fd
.iter_type
;
1988 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
1989 count
+= fd
.collapse
- 1;
1990 for (i
= 0; i
< count
; i
++)
1992 tree temp
= create_tmp_var (type
, NULL
);
1993 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
1994 OMP_CLAUSE__LOOPTEMP_
);
1995 OMP_CLAUSE_DECL (c
) = temp
;
1996 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
1997 gimple_omp_parallel_set_clauses (stmt
, c
);
2002 ctx
= new_omp_context (stmt
, outer_ctx
);
2003 if (taskreg_nesting_level
> 1)
2004 ctx
->is_nested
= true;
2005 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2006 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2007 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2008 name
= create_tmp_var_name (".omp_data_s");
2009 name
= build_decl (gimple_location (stmt
),
2010 TYPE_DECL
, name
, ctx
->record_type
);
2011 DECL_ARTIFICIAL (name
) = 1;
2012 DECL_NAMELESS (name
) = 1;
2013 TYPE_NAME (ctx
->record_type
) = name
;
2014 create_omp_child_function (ctx
, false);
2015 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2017 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2018 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2020 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2021 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2024 layout_type (ctx
->record_type
);
2025 fixup_child_record_type (ctx
);
2029 /* Scan an OpenMP task directive. */
2032 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2036 gimple stmt
= gsi_stmt (*gsi
);
2037 location_t loc
= gimple_location (stmt
);
2039 /* Ignore task directives with empty bodies. */
2041 && empty_body_p (gimple_omp_body (stmt
)))
2043 gsi_replace (gsi
, gimple_build_nop (), false);
2047 ctx
= new_omp_context (stmt
, outer_ctx
);
2048 if (taskreg_nesting_level
> 1)
2049 ctx
->is_nested
= true;
2050 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2051 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2052 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2053 name
= create_tmp_var_name (".omp_data_s");
2054 name
= build_decl (gimple_location (stmt
),
2055 TYPE_DECL
, name
, ctx
->record_type
);
2056 DECL_ARTIFICIAL (name
) = 1;
2057 DECL_NAMELESS (name
) = 1;
2058 TYPE_NAME (ctx
->record_type
) = name
;
2059 create_omp_child_function (ctx
, false);
2060 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2062 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2064 if (ctx
->srecord_type
)
2066 name
= create_tmp_var_name (".omp_data_a");
2067 name
= build_decl (gimple_location (stmt
),
2068 TYPE_DECL
, name
, ctx
->srecord_type
);
2069 DECL_ARTIFICIAL (name
) = 1;
2070 DECL_NAMELESS (name
) = 1;
2071 TYPE_NAME (ctx
->srecord_type
) = name
;
2072 create_omp_child_function (ctx
, true);
2075 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2077 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2079 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2080 t
= build_int_cst (long_integer_type_node
, 0);
2081 gimple_omp_task_set_arg_size (stmt
, t
);
2082 t
= build_int_cst (long_integer_type_node
, 1);
2083 gimple_omp_task_set_arg_align (stmt
, t
);
2087 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2088 /* Move VLA fields to the end. */
2089 p
= &TYPE_FIELDS (ctx
->record_type
);
2091 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2092 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2095 *p
= TREE_CHAIN (*p
);
2096 TREE_CHAIN (*q
) = NULL_TREE
;
2097 q
= &TREE_CHAIN (*q
);
2100 p
= &DECL_CHAIN (*p
);
2102 layout_type (ctx
->record_type
);
2103 fixup_child_record_type (ctx
);
2104 if (ctx
->srecord_type
)
2105 layout_type (ctx
->srecord_type
);
2106 t
= fold_convert_loc (loc
, long_integer_type_node
,
2107 TYPE_SIZE_UNIT (ctx
->record_type
));
2108 gimple_omp_task_set_arg_size (stmt
, t
);
2109 t
= build_int_cst (long_integer_type_node
,
2110 TYPE_ALIGN_UNIT (ctx
->record_type
));
2111 gimple_omp_task_set_arg_align (stmt
, t
);
2116 /* Scan an OpenMP loop directive. */
2119 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2124 ctx
= new_omp_context (stmt
, outer_ctx
);
2126 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2128 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2129 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2131 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2132 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2133 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2134 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2136 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2139 /* Scan an OpenMP sections directive. */
2142 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2146 ctx
= new_omp_context (stmt
, outer_ctx
);
2147 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2148 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2151 /* Scan an OpenMP single directive. */
2154 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2159 ctx
= new_omp_context (stmt
, outer_ctx
);
2160 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2161 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2162 name
= create_tmp_var_name (".omp_copy_s");
2163 name
= build_decl (gimple_location (stmt
),
2164 TYPE_DECL
, name
, ctx
->record_type
);
2165 TYPE_NAME (ctx
->record_type
) = name
;
2167 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2168 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2170 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2171 ctx
->record_type
= NULL
;
2173 layout_type (ctx
->record_type
);
2176 /* Scan an OpenMP target{, data, update} directive. */
2179 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2183 int kind
= gimple_omp_target_kind (stmt
);
2185 ctx
= new_omp_context (stmt
, outer_ctx
);
2186 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2187 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2188 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2189 name
= create_tmp_var_name (".omp_data_t");
2190 name
= build_decl (gimple_location (stmt
),
2191 TYPE_DECL
, name
, ctx
->record_type
);
2192 DECL_ARTIFICIAL (name
) = 1;
2193 DECL_NAMELESS (name
) = 1;
2194 TYPE_NAME (ctx
->record_type
) = name
;
2195 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2197 create_omp_child_function (ctx
, false);
2198 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2201 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2202 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2204 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2205 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2208 TYPE_FIELDS (ctx
->record_type
)
2209 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2210 #ifdef ENABLE_CHECKING
2212 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2213 for (field
= TYPE_FIELDS (ctx
->record_type
);
2215 field
= DECL_CHAIN (field
))
2216 gcc_assert (DECL_ALIGN (field
) == align
);
2218 layout_type (ctx
->record_type
);
2219 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2220 fixup_child_record_type (ctx
);
2224 /* Scan an OpenMP teams directive. */
2227 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2229 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2230 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2231 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2234 /* Check OpenMP nesting restrictions. */
2236 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2240 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2241 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2243 error_at (gimple_location (stmt
),
2244 "OpenMP constructs may not be nested inside simd region");
2247 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2249 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2250 || (gimple_omp_for_kind (stmt
)
2251 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2252 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2254 error_at (gimple_location (stmt
),
2255 "only distribute or parallel constructs are allowed to "
2256 "be closely nested inside teams construct");
2261 switch (gimple_code (stmt
))
2263 case GIMPLE_OMP_FOR
:
2264 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_KIND_SIMD
)
2266 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2268 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2270 error_at (gimple_location (stmt
),
2271 "distribute construct must be closely nested inside "
2279 if (is_gimple_call (stmt
)
2280 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2281 == BUILT_IN_GOMP_CANCEL
2282 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2283 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2285 const char *bad
= NULL
;
2286 const char *kind
= NULL
;
2289 error_at (gimple_location (stmt
), "orphaned %qs construct",
2290 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2291 == BUILT_IN_GOMP_CANCEL
2292 ? "#pragma omp cancel"
2293 : "#pragma omp cancellation point");
2296 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2297 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2301 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2302 bad
= "#pragma omp parallel";
2303 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2304 == BUILT_IN_GOMP_CANCEL
2305 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2306 ctx
->cancellable
= true;
2310 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2311 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2312 bad
= "#pragma omp for";
2313 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2314 == BUILT_IN_GOMP_CANCEL
2315 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2317 ctx
->cancellable
= true;
2318 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2320 warning_at (gimple_location (stmt
), 0,
2321 "%<#pragma omp cancel for%> inside "
2322 "%<nowait%> for construct");
2323 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2324 OMP_CLAUSE_ORDERED
))
2325 warning_at (gimple_location (stmt
), 0,
2326 "%<#pragma omp cancel for%> inside "
2327 "%<ordered%> for construct");
2332 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2333 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2334 bad
= "#pragma omp sections";
2335 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2336 == BUILT_IN_GOMP_CANCEL
2337 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2339 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2341 ctx
->cancellable
= true;
2342 if (find_omp_clause (gimple_omp_sections_clauses
2345 warning_at (gimple_location (stmt
), 0,
2346 "%<#pragma omp cancel sections%> inside "
2347 "%<nowait%> sections construct");
2351 gcc_assert (ctx
->outer
2352 && gimple_code (ctx
->outer
->stmt
)
2353 == GIMPLE_OMP_SECTIONS
);
2354 ctx
->outer
->cancellable
= true;
2355 if (find_omp_clause (gimple_omp_sections_clauses
2358 warning_at (gimple_location (stmt
), 0,
2359 "%<#pragma omp cancel sections%> inside "
2360 "%<nowait%> sections construct");
2366 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2367 bad
= "#pragma omp task";
2369 ctx
->cancellable
= true;
2373 error_at (gimple_location (stmt
), "invalid arguments");
2378 error_at (gimple_location (stmt
),
2379 "%<%s %s%> construct not closely nested inside of %qs",
2380 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2381 == BUILT_IN_GOMP_CANCEL
2382 ? "#pragma omp cancel"
2383 : "#pragma omp cancellation point", kind
, bad
);
2388 case GIMPLE_OMP_SECTIONS
:
2389 case GIMPLE_OMP_SINGLE
:
2390 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2391 switch (gimple_code (ctx
->stmt
))
2393 case GIMPLE_OMP_FOR
:
2394 case GIMPLE_OMP_SECTIONS
:
2395 case GIMPLE_OMP_SINGLE
:
2396 case GIMPLE_OMP_ORDERED
:
2397 case GIMPLE_OMP_MASTER
:
2398 case GIMPLE_OMP_TASK
:
2399 case GIMPLE_OMP_CRITICAL
:
2400 if (is_gimple_call (stmt
))
2402 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2403 != BUILT_IN_GOMP_BARRIER
)
2405 error_at (gimple_location (stmt
),
2406 "barrier region may not be closely nested inside "
2407 "of work-sharing, critical, ordered, master or "
2408 "explicit task region");
2411 error_at (gimple_location (stmt
),
2412 "work-sharing region may not be closely nested inside "
2413 "of work-sharing, critical, ordered, master or explicit "
2416 case GIMPLE_OMP_PARALLEL
:
2422 case GIMPLE_OMP_MASTER
:
2423 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2424 switch (gimple_code (ctx
->stmt
))
2426 case GIMPLE_OMP_FOR
:
2427 case GIMPLE_OMP_SECTIONS
:
2428 case GIMPLE_OMP_SINGLE
:
2429 case GIMPLE_OMP_TASK
:
2430 error_at (gimple_location (stmt
),
2431 "master region may not be closely nested inside "
2432 "of work-sharing or explicit task region");
2434 case GIMPLE_OMP_PARALLEL
:
2440 case GIMPLE_OMP_ORDERED
:
2441 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2442 switch (gimple_code (ctx
->stmt
))
2444 case GIMPLE_OMP_CRITICAL
:
2445 case GIMPLE_OMP_TASK
:
2446 error_at (gimple_location (stmt
),
2447 "ordered region may not be closely nested inside "
2448 "of critical or explicit task region");
2450 case GIMPLE_OMP_FOR
:
2451 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2452 OMP_CLAUSE_ORDERED
) == NULL
)
2454 error_at (gimple_location (stmt
),
2455 "ordered region must be closely nested inside "
2456 "a loop region with an ordered clause");
2460 case GIMPLE_OMP_PARALLEL
:
2461 error_at (gimple_location (stmt
),
2462 "ordered region must be closely nested inside "
2463 "a loop region with an ordered clause");
2469 case GIMPLE_OMP_CRITICAL
:
2470 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2471 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2472 && (gimple_omp_critical_name (stmt
)
2473 == gimple_omp_critical_name (ctx
->stmt
)))
2475 error_at (gimple_location (stmt
),
2476 "critical region may not be nested inside a critical "
2477 "region with the same name");
2481 case GIMPLE_OMP_TEAMS
:
2483 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2484 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2486 error_at (gimple_location (stmt
),
2487 "teams construct not closely nested inside of target "
2499 /* Helper function scan_omp.
2501 Callback for walk_tree or operators in walk_gimple_stmt used to
2502 scan for OpenMP directives in TP. */
2505 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2507 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2508 omp_context
*ctx
= (omp_context
*) wi
->info
;
2511 switch (TREE_CODE (t
))
2518 *tp
= remap_decl (t
, &ctx
->cb
);
2522 if (ctx
&& TYPE_P (t
))
2523 *tp
= remap_type (t
, &ctx
->cb
);
2524 else if (!DECL_P (t
))
2529 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2530 if (tem
!= TREE_TYPE (t
))
2532 if (TREE_CODE (t
) == INTEGER_CST
)
2533 *tp
= build_int_cst_wide (tem
,
2534 TREE_INT_CST_LOW (t
),
2535 TREE_INT_CST_HIGH (t
));
2537 TREE_TYPE (t
) = tem
;
2547 /* Return true if FNDECL is a setjmp or a longjmp. */
2550 setjmp_or_longjmp_p (const_tree fndecl
)
2552 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2553 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2554 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2557 tree declname
= DECL_NAME (fndecl
);
2560 const char *name
= IDENTIFIER_POINTER (declname
);
2561 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2565 /* Helper function for scan_omp.
2567 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2568 the current statement in GSI. */
2571 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2572 struct walk_stmt_info
*wi
)
2574 gimple stmt
= gsi_stmt (*gsi
);
2575 omp_context
*ctx
= (omp_context
*) wi
->info
;
2577 if (gimple_has_location (stmt
))
2578 input_location
= gimple_location (stmt
);
2580 /* Check the OpenMP nesting restrictions. */
2581 bool remove
= false;
2582 if (is_gimple_omp (stmt
))
2583 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2584 else if (is_gimple_call (stmt
))
2586 tree fndecl
= gimple_call_fndecl (stmt
);
2589 if (setjmp_or_longjmp_p (fndecl
)
2591 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2592 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2595 error_at (gimple_location (stmt
),
2596 "setjmp/longjmp inside simd construct");
2598 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2599 switch (DECL_FUNCTION_CODE (fndecl
))
2601 case BUILT_IN_GOMP_BARRIER
:
2602 case BUILT_IN_GOMP_CANCEL
:
2603 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2604 case BUILT_IN_GOMP_TASKYIELD
:
2605 case BUILT_IN_GOMP_TASKWAIT
:
2606 case BUILT_IN_GOMP_TASKGROUP_START
:
2607 case BUILT_IN_GOMP_TASKGROUP_END
:
2608 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2617 stmt
= gimple_build_nop ();
2618 gsi_replace (gsi
, stmt
, false);
2621 *handled_ops_p
= true;
2623 switch (gimple_code (stmt
))
2625 case GIMPLE_OMP_PARALLEL
:
2626 taskreg_nesting_level
++;
2627 scan_omp_parallel (gsi
, ctx
);
2628 taskreg_nesting_level
--;
2631 case GIMPLE_OMP_TASK
:
2632 taskreg_nesting_level
++;
2633 scan_omp_task (gsi
, ctx
);
2634 taskreg_nesting_level
--;
2637 case GIMPLE_OMP_FOR
:
2638 scan_omp_for (stmt
, ctx
);
2641 case GIMPLE_OMP_SECTIONS
:
2642 scan_omp_sections (stmt
, ctx
);
2645 case GIMPLE_OMP_SINGLE
:
2646 scan_omp_single (stmt
, ctx
);
2649 case GIMPLE_OMP_SECTION
:
2650 case GIMPLE_OMP_MASTER
:
2651 case GIMPLE_OMP_TASKGROUP
:
2652 case GIMPLE_OMP_ORDERED
:
2653 case GIMPLE_OMP_CRITICAL
:
2654 ctx
= new_omp_context (stmt
, ctx
);
2655 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2658 case GIMPLE_OMP_TARGET
:
2659 scan_omp_target (stmt
, ctx
);
2662 case GIMPLE_OMP_TEAMS
:
2663 scan_omp_teams (stmt
, ctx
);
2670 *handled_ops_p
= false;
2672 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2673 insert_decl_map (&ctx
->cb
, var
, var
);
2677 *handled_ops_p
= false;
2685 /* Scan all the statements starting at the current statement. CTX
2686 contains context information about the OpenMP directives and
2687 clauses found during the scan. */
2690 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2692 location_t saved_location
;
2693 struct walk_stmt_info wi
;
2695 memset (&wi
, 0, sizeof (wi
));
2697 wi
.want_locations
= true;
2699 saved_location
= input_location
;
2700 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2701 input_location
= saved_location
;
2704 /* Re-gimplification and code generation routines. */
2706 /* Build a call to GOMP_barrier. */
2709 build_omp_barrier (tree lhs
)
2711 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2712 : BUILT_IN_GOMP_BARRIER
);
2713 gimple g
= gimple_build_call (fndecl
, 0);
2715 gimple_call_set_lhs (g
, lhs
);
2719 /* If a context was created for STMT when it was scanned, return it. */
2721 static omp_context
*
2722 maybe_lookup_ctx (gimple stmt
)
2725 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2726 return n
? (omp_context
*) n
->value
: NULL
;
2730 /* Find the mapping for DECL in CTX or the immediately enclosing
2731 context that has a mapping for DECL.
2733 If CTX is a nested parallel directive, we may have to use the decl
2734 mappings created in CTX's parent context. Suppose that we have the
2735 following parallel nesting (variable UIDs showed for clarity):
2738 #omp parallel shared(iD.1562) -> outer parallel
2739 iD.1562 = iD.1562 + 1;
2741 #omp parallel shared (iD.1562) -> inner parallel
2742 iD.1562 = iD.1562 - 1;
2744 Each parallel structure will create a distinct .omp_data_s structure
2745 for copying iD.1562 in/out of the directive:
2747 outer parallel .omp_data_s.1.i -> iD.1562
2748 inner parallel .omp_data_s.2.i -> iD.1562
2750 A shared variable mapping will produce a copy-out operation before
2751 the parallel directive and a copy-in operation after it. So, in
2752 this case we would have:
2755 .omp_data_o.1.i = iD.1562;
2756 #omp parallel shared(iD.1562) -> outer parallel
2757 .omp_data_i.1 = &.omp_data_o.1
2758 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2760 .omp_data_o.2.i = iD.1562; -> **
2761 #omp parallel shared(iD.1562) -> inner parallel
2762 .omp_data_i.2 = &.omp_data_o.2
2763 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2766 ** This is a problem. The symbol iD.1562 cannot be referenced
2767 inside the body of the outer parallel region. But since we are
2768 emitting this copy operation while expanding the inner parallel
2769 directive, we need to access the CTX structure of the outer
2770 parallel directive to get the correct mapping:
2772 .omp_data_o.2.i = .omp_data_i.1->i
2774 Since there may be other workshare or parallel directives enclosing
2775 the parallel directive, it may be necessary to walk up the context
2776 parent chain. This is not a problem in general because nested
2777 parallelism happens only rarely. */
2780 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2785 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2786 t
= maybe_lookup_decl (decl
, up
);
2788 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2790 return t
? t
: decl
;
2794 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2795 in outer contexts. */
2798 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2803 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2804 t
= maybe_lookup_decl (decl
, up
);
2806 return t
? t
: decl
;
2810 /* Construct the initialization value for reduction CLAUSE. */
2813 omp_reduction_init (tree clause
, tree type
)
2815 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2816 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2823 case TRUTH_ORIF_EXPR
:
2824 case TRUTH_XOR_EXPR
:
2826 return build_zero_cst (type
);
2829 case TRUTH_AND_EXPR
:
2830 case TRUTH_ANDIF_EXPR
:
2832 return fold_convert_loc (loc
, type
, integer_one_node
);
2835 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2838 if (SCALAR_FLOAT_TYPE_P (type
))
2840 REAL_VALUE_TYPE max
, min
;
2841 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2844 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2847 real_maxval (&min
, 1, TYPE_MODE (type
));
2848 return build_real (type
, min
);
2852 gcc_assert (INTEGRAL_TYPE_P (type
));
2853 return TYPE_MIN_VALUE (type
);
2857 if (SCALAR_FLOAT_TYPE_P (type
))
2859 REAL_VALUE_TYPE max
;
2860 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2863 real_maxval (&max
, 0, TYPE_MODE (type
));
2864 return build_real (type
, max
);
2868 gcc_assert (INTEGRAL_TYPE_P (type
));
2869 return TYPE_MAX_VALUE (type
);
2877 /* Return alignment to be assumed for var in CLAUSE, which should be
2878 OMP_CLAUSE_ALIGNED. */
2881 omp_clause_aligned_alignment (tree clause
)
2883 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2884 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2886 /* Otherwise return implementation defined alignment. */
2887 unsigned int al
= 1;
2888 enum machine_mode mode
, vmode
;
2889 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2891 vs
= 1 << floor_log2 (vs
);
2892 static enum mode_class classes
[]
2893 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2894 for (int i
= 0; i
< 4; i
+= 2)
2895 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2897 mode
= GET_MODE_WIDER_MODE (mode
))
2899 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2900 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2903 && GET_MODE_SIZE (vmode
) < vs
2904 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2905 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2907 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2908 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2910 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2911 / GET_MODE_SIZE (mode
));
2912 if (TYPE_MODE (type
) != vmode
)
2914 if (TYPE_ALIGN_UNIT (type
) > al
)
2915 al
= TYPE_ALIGN_UNIT (type
);
2917 return build_int_cst (integer_type_node
, al
);
2920 /* Return maximum possible vectorization factor for the target. */
2927 || (!flag_tree_loop_vectorize
2928 && (global_options_set
.x_flag_tree_loop_vectorize
2929 || global_options_set
.x_flag_tree_vectorize
)))
2932 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2935 vs
= 1 << floor_log2 (vs
);
2938 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2939 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2940 return GET_MODE_NUNITS (vqimode
);
2944 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2948 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2949 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2953 max_vf
= omp_max_vf ();
2956 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2957 OMP_CLAUSE_SAFELEN
);
2959 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
), max_vf
) == -1)
2960 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
2964 idx
= create_tmp_var (unsigned_type_node
, NULL
);
2965 lane
= create_tmp_var (unsigned_type_node
, NULL
);
2971 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
2972 tree avar
= create_tmp_var_raw (atype
, NULL
);
2973 if (TREE_ADDRESSABLE (new_var
))
2974 TREE_ADDRESSABLE (avar
) = 1;
2975 DECL_ATTRIBUTES (avar
)
2976 = tree_cons (get_identifier ("omp simd array"), NULL
,
2977 DECL_ATTRIBUTES (avar
));
2978 gimple_add_tmp_var (avar
);
2979 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
2980 NULL_TREE
, NULL_TREE
);
2981 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
2982 NULL_TREE
, NULL_TREE
);
2983 if (DECL_P (new_var
))
2985 SET_DECL_VALUE_EXPR (new_var
, lvar
);
2986 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
2991 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2992 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2993 private variables. Initialization statements go in ILIST, while calls
2994 to destructors go in DLIST. */
2997 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
2998 omp_context
*ctx
, struct omp_for_data
*fd
)
3000 tree c
, dtor
, copyin_seq
, x
, ptr
;
3001 bool copyin_by_ref
= false;
3002 bool lastprivate_firstprivate
= false;
3003 bool reduction_omp_orig_ref
= false;
3005 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3006 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
);
3008 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3009 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3010 gimple_seq llist
[2] = { NULL
, NULL
};
3014 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3015 with data sharing clauses referencing variable sized vars. That
3016 is unnecessarily hard to support and very unlikely to result in
3017 vectorized code anyway. */
3019 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3020 switch (OMP_CLAUSE_CODE (c
))
3022 case OMP_CLAUSE_REDUCTION
:
3023 case OMP_CLAUSE_PRIVATE
:
3024 case OMP_CLAUSE_FIRSTPRIVATE
:
3025 case OMP_CLAUSE_LASTPRIVATE
:
3026 case OMP_CLAUSE_LINEAR
:
3027 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3034 /* Do all the fixed sized types in the first pass, and the variable sized
3035 types in the second pass. This makes sure that the scalar arguments to
3036 the variable sized types are processed before we use them in the
3037 variable sized operations. */
3038 for (pass
= 0; pass
< 2; ++pass
)
3040 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3042 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3045 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3049 case OMP_CLAUSE_PRIVATE
:
3050 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3053 case OMP_CLAUSE_SHARED
:
3054 /* Ignore shared directives in teams construct. */
3055 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3057 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3059 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3062 case OMP_CLAUSE_FIRSTPRIVATE
:
3063 case OMP_CLAUSE_COPYIN
:
3064 case OMP_CLAUSE_LINEAR
:
3066 case OMP_CLAUSE_REDUCTION
:
3067 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3068 reduction_omp_orig_ref
= true;
3070 case OMP_CLAUSE__LOOPTEMP_
:
3071 /* Handle _looptemp_ clauses only on parallel. */
3075 case OMP_CLAUSE_LASTPRIVATE
:
3076 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3078 lastprivate_firstprivate
= true;
3083 case OMP_CLAUSE_ALIGNED
:
3086 var
= OMP_CLAUSE_DECL (c
);
3087 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3088 && !is_global_var (var
))
3090 new_var
= maybe_lookup_decl (var
, ctx
);
3091 if (new_var
== NULL_TREE
)
3092 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3093 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3094 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3095 omp_clause_aligned_alignment (c
));
3096 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3097 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3098 gimplify_and_add (x
, ilist
);
3100 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3101 && is_global_var (var
))
3103 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3104 new_var
= lookup_decl (var
, ctx
);
3105 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3106 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3107 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3108 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3109 omp_clause_aligned_alignment (c
));
3110 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3111 x
= create_tmp_var (ptype
, NULL
);
3112 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3113 gimplify_and_add (t
, ilist
);
3114 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3115 SET_DECL_VALUE_EXPR (new_var
, t
);
3116 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3123 new_var
= var
= OMP_CLAUSE_DECL (c
);
3124 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3125 new_var
= lookup_decl (var
, ctx
);
3127 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3132 else if (is_variable_sized (var
))
3134 /* For variable sized types, we need to allocate the
3135 actual storage here. Call alloca and store the
3136 result in the pointer decl that we created elsewhere. */
3140 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3145 ptr
= DECL_VALUE_EXPR (new_var
);
3146 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3147 ptr
= TREE_OPERAND (ptr
, 0);
3148 gcc_assert (DECL_P (ptr
));
3149 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3151 /* void *tmp = __builtin_alloca */
3152 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3153 stmt
= gimple_build_call (atmp
, 1, x
);
3154 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3155 gimple_add_tmp_var (tmp
);
3156 gimple_call_set_lhs (stmt
, tmp
);
3158 gimple_seq_add_stmt (ilist
, stmt
);
3160 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3161 gimplify_assign (ptr
, x
, ilist
);
3164 else if (is_reference (var
))
3166 /* For references that are being privatized for Fortran,
3167 allocate new backing storage for the new pointer
3168 variable. This allows us to avoid changing all the
3169 code that expects a pointer to something that expects
3170 a direct variable. */
3174 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3175 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3177 x
= build_receiver_ref (var
, false, ctx
);
3178 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3180 else if (TREE_CONSTANT (x
))
3182 const char *name
= NULL
;
3183 if (DECL_NAME (var
))
3184 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3186 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3188 gimple_add_tmp_var (x
);
3189 TREE_ADDRESSABLE (x
) = 1;
3190 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3194 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3195 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3198 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3199 gimplify_assign (new_var
, x
, ilist
);
3201 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3203 else if (c_kind
== OMP_CLAUSE_REDUCTION
3204 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3212 switch (OMP_CLAUSE_CODE (c
))
3214 case OMP_CLAUSE_SHARED
:
3215 /* Ignore shared directives in teams construct. */
3216 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3218 /* Shared global vars are just accessed directly. */
3219 if (is_global_var (new_var
))
3221 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3222 needs to be delayed until after fixup_child_record_type so
3223 that we get the correct type during the dereference. */
3224 by_ref
= use_pointer_for_field (var
, ctx
);
3225 x
= build_receiver_ref (var
, by_ref
, ctx
);
3226 SET_DECL_VALUE_EXPR (new_var
, x
);
3227 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3229 /* ??? If VAR is not passed by reference, and the variable
3230 hasn't been initialized yet, then we'll get a warning for
3231 the store into the omp_data_s structure. Ideally, we'd be
3232 able to notice this and not store anything at all, but
3233 we're generating code too early. Suppress the warning. */
3235 TREE_NO_WARNING (var
) = 1;
3238 case OMP_CLAUSE_LASTPRIVATE
:
3239 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3243 case OMP_CLAUSE_PRIVATE
:
3244 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3245 x
= build_outer_var_ref (var
, ctx
);
3246 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3248 if (is_task_ctx (ctx
))
3249 x
= build_receiver_ref (var
, false, ctx
);
3251 x
= build_outer_var_ref (var
, ctx
);
3257 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3260 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3261 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3262 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3263 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3264 idx
, lane
, ivar
, lvar
))
3267 x
= lang_hooks
.decls
.omp_clause_default_ctor
3268 (c
, unshare_expr (ivar
), x
);
3270 gimplify_and_add (x
, &llist
[0]);
3273 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3276 gimple_seq tseq
= NULL
;
3279 gimplify_stmt (&dtor
, &tseq
);
3280 gimple_seq_add_seq (&llist
[1], tseq
);
3287 gimplify_and_add (nx
, ilist
);
3291 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3294 gimple_seq tseq
= NULL
;
3297 gimplify_stmt (&dtor
, &tseq
);
3298 gimple_seq_add_seq (dlist
, tseq
);
3302 case OMP_CLAUSE_LINEAR
:
3303 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3304 goto do_firstprivate
;
3305 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3308 x
= build_outer_var_ref (var
, ctx
);
3311 case OMP_CLAUSE_FIRSTPRIVATE
:
3312 if (is_task_ctx (ctx
))
3314 if (is_reference (var
) || is_variable_sized (var
))
3316 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3318 || use_pointer_for_field (var
, NULL
))
3320 x
= build_receiver_ref (var
, false, ctx
);
3321 SET_DECL_VALUE_EXPR (new_var
, x
);
3322 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3327 x
= build_outer_var_ref (var
, ctx
);
3330 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3331 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3333 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3334 ? sizetype
: TREE_TYPE (x
);
3335 tree t
= fold_convert (stept
,
3336 OMP_CLAUSE_LINEAR_STEP (c
));
3337 tree c
= find_omp_clause (clauses
,
3338 OMP_CLAUSE__LOOPTEMP_
);
3340 tree l
= OMP_CLAUSE_DECL (c
);
3341 if (fd
->collapse
== 1)
3343 tree n1
= fd
->loop
.n1
;
3344 tree step
= fd
->loop
.step
;
3345 tree itype
= TREE_TYPE (l
);
3346 if (POINTER_TYPE_P (itype
))
3347 itype
= signed_type_for (itype
);
3348 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3349 if (TYPE_UNSIGNED (itype
)
3350 && fd
->loop
.cond_code
== GT_EXPR
)
3351 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3352 fold_build1 (NEGATE_EXPR
,
3354 fold_build1 (NEGATE_EXPR
,
3357 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3359 t
= fold_build2 (MULT_EXPR
, stept
,
3360 fold_convert (stept
, l
), t
);
3361 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3362 x
= fold_build2 (POINTER_PLUS_EXPR
,
3363 TREE_TYPE (x
), x
, t
);
3365 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3368 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3369 || TREE_ADDRESSABLE (new_var
))
3370 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3371 idx
, lane
, ivar
, lvar
))
3373 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3375 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3376 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3377 gimplify_and_add (x
, ilist
);
3378 gimple_stmt_iterator gsi
3379 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3381 = gimple_build_assign (unshare_expr (lvar
), iv
);
3382 gsi_insert_before_without_update (&gsi
, g
,
3384 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3385 ? sizetype
: TREE_TYPE (x
);
3386 tree t
= fold_convert (stept
,
3387 OMP_CLAUSE_LINEAR_STEP (c
));
3388 enum tree_code code
= PLUS_EXPR
;
3389 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3390 code
= POINTER_PLUS_EXPR
;
3391 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3392 gsi_insert_before_without_update (&gsi
, g
,
3396 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3397 (c
, unshare_expr (ivar
), x
);
3398 gimplify_and_add (x
, &llist
[0]);
3399 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3402 gimple_seq tseq
= NULL
;
3405 gimplify_stmt (&dtor
, &tseq
);
3406 gimple_seq_add_seq (&llist
[1], tseq
);
3411 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3412 gimplify_and_add (x
, ilist
);
3415 case OMP_CLAUSE__LOOPTEMP_
:
3416 gcc_assert (is_parallel_ctx (ctx
));
3417 x
= build_outer_var_ref (var
, ctx
);
3418 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3419 gimplify_and_add (x
, ilist
);
3422 case OMP_CLAUSE_COPYIN
:
3423 by_ref
= use_pointer_for_field (var
, NULL
);
3424 x
= build_receiver_ref (var
, by_ref
, ctx
);
3425 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3426 append_to_statement_list (x
, ©in_seq
);
3427 copyin_by_ref
|= by_ref
;
3430 case OMP_CLAUSE_REDUCTION
:
3431 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3433 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3435 x
= build_outer_var_ref (var
, ctx
);
3437 if (is_reference (var
)
3438 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3440 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3441 SET_DECL_VALUE_EXPR (placeholder
, x
);
3442 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3443 tree new_vard
= new_var
;
3444 if (is_reference (var
))
3446 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3447 new_vard
= TREE_OPERAND (new_var
, 0);
3448 gcc_assert (DECL_P (new_vard
));
3451 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3452 idx
, lane
, ivar
, lvar
))
3454 if (new_vard
== new_var
)
3456 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3457 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3461 SET_DECL_VALUE_EXPR (new_vard
,
3462 build_fold_addr_expr (ivar
));
3463 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3465 x
= lang_hooks
.decls
.omp_clause_default_ctor
3466 (c
, unshare_expr (ivar
),
3467 build_outer_var_ref (var
, ctx
));
3469 gimplify_and_add (x
, &llist
[0]);
3470 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3472 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3473 lower_omp (&tseq
, ctx
);
3474 gimple_seq_add_seq (&llist
[0], tseq
);
3476 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3477 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3478 lower_omp (&tseq
, ctx
);
3479 gimple_seq_add_seq (&llist
[1], tseq
);
3480 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3481 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3482 if (new_vard
== new_var
)
3483 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3485 SET_DECL_VALUE_EXPR (new_vard
,
3486 build_fold_addr_expr (lvar
));
3487 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3492 gimplify_stmt (&dtor
, &tseq
);
3493 gimple_seq_add_seq (&llist
[1], tseq
);
3497 x
= lang_hooks
.decls
.omp_clause_default_ctor
3498 (c
, new_var
, unshare_expr (x
));
3500 gimplify_and_add (x
, ilist
);
3501 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3503 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3504 lower_omp (&tseq
, ctx
);
3505 gimple_seq_add_seq (ilist
, tseq
);
3507 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3510 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3511 lower_omp (&tseq
, ctx
);
3512 gimple_seq_add_seq (dlist
, tseq
);
3513 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3515 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3520 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3521 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3523 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3524 idx
, lane
, ivar
, lvar
))
3526 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3527 tree ref
= build_outer_var_ref (var
, ctx
);
3529 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3531 /* reduction(-:var) sums up the partial results, so it
3532 acts identically to reduction(+:var). */
3533 if (code
== MINUS_EXPR
)
3536 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3537 ref
= build_outer_var_ref (var
, ctx
);
3538 gimplify_assign (ref
, x
, &llist
[1]);
3542 gimplify_assign (new_var
, x
, ilist
);
3544 gimplify_assign (build_outer_var_ref (var
, ctx
),
3558 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3559 /* Don't want uninit warnings on simduid, it is always uninitialized,
3560 but we use it not for the value, but for the DECL_UID only. */
3561 TREE_NO_WARNING (uid
) = 1;
3563 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3564 gimple_call_set_lhs (g
, lane
);
3565 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3566 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3567 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3568 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3569 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3570 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3571 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3572 build_int_cst (unsigned_type_node
, 0),
3574 gimple_seq_add_stmt (ilist
, g
);
3575 for (int i
= 0; i
< 2; i
++)
3578 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3579 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3580 gimple_call_set_lhs (g
, vf
);
3581 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3582 gimple_seq_add_stmt (seq
, g
);
3583 tree t
= build_int_cst (unsigned_type_node
, 0);
3584 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3585 gimple_seq_add_stmt (seq
, g
);
3586 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3587 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3588 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3589 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3590 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3591 gimple_seq_add_seq (seq
, llist
[i
]);
3592 t
= build_int_cst (unsigned_type_node
, 1);
3593 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3594 gimple_seq_add_stmt (seq
, g
);
3595 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3596 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3597 gimple_seq_add_stmt (seq
, g
);
3598 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3602 /* The copyin sequence is not to be executed by the main thread, since
3603 that would result in self-copies. Perhaps not visible to scalars,
3604 but it certainly is to C++ operator=. */
3607 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3609 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3610 build_int_cst (TREE_TYPE (x
), 0));
3611 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3612 gimplify_and_add (x
, ilist
);
3615 /* If any copyin variable is passed by reference, we must ensure the
3616 master thread doesn't modify it before it is copied over in all
3617 threads. Similarly for variables in both firstprivate and
3618 lastprivate clauses we need to ensure the lastprivate copying
3619 happens after firstprivate copying in all threads. And similarly
3620 for UDRs if initializer expression refers to omp_orig. */
3621 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3623 /* Don't add any barrier for #pragma omp simd or
3624 #pragma omp distribute. */
3625 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3626 || gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_FOR
)
3627 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3630 /* If max_vf is non-zero, then we can use only a vectorization factor
3631 up to the max_vf we chose. So stick it into the safelen clause. */
3634 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3635 OMP_CLAUSE_SAFELEN
);
3637 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3640 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3641 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3643 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3644 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3650 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3651 both parallel and workshare constructs. PREDICATE may be NULL if it's
3655 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3658 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3659 bool par_clauses
= false;
3660 tree simduid
= NULL
, lastlane
= NULL
;
3662 /* Early exit if there are no lastprivate or linear clauses. */
3663 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3664 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3665 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3666 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3668 if (clauses
== NULL
)
3670 /* If this was a workshare clause, see if it had been combined
3671 with its parallel. In that case, look for the clauses on the
3672 parallel statement itself. */
3673 if (is_parallel_ctx (ctx
))
3677 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3680 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3681 OMP_CLAUSE_LASTPRIVATE
);
3682 if (clauses
== NULL
)
3690 tree label_true
, arm1
, arm2
;
3692 label
= create_artificial_label (UNKNOWN_LOCATION
);
3693 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3694 arm1
= TREE_OPERAND (predicate
, 0);
3695 arm2
= TREE_OPERAND (predicate
, 1);
3696 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3697 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3698 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3700 gimple_seq_add_stmt (stmt_list
, stmt
);
3701 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3704 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3705 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3707 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3709 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3712 for (c
= clauses
; c
;)
3715 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3717 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3718 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3719 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3721 var
= OMP_CLAUSE_DECL (c
);
3722 new_var
= lookup_decl (var
, ctx
);
3724 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3726 tree val
= DECL_VALUE_EXPR (new_var
);
3727 if (TREE_CODE (val
) == ARRAY_REF
3728 && VAR_P (TREE_OPERAND (val
, 0))
3729 && lookup_attribute ("omp simd array",
3730 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3733 if (lastlane
== NULL
)
3735 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3737 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3739 TREE_OPERAND (val
, 1));
3740 gimple_call_set_lhs (g
, lastlane
);
3741 gimple_seq_add_stmt (stmt_list
, g
);
3743 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3744 TREE_OPERAND (val
, 0), lastlane
,
3745 NULL_TREE
, NULL_TREE
);
3749 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3750 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3752 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3753 gimple_seq_add_seq (stmt_list
,
3754 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3755 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3758 x
= build_outer_var_ref (var
, ctx
);
3759 if (is_reference (var
))
3760 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3761 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3762 gimplify_and_add (x
, stmt_list
);
3764 c
= OMP_CLAUSE_CHAIN (c
);
3765 if (c
== NULL
&& !par_clauses
)
3767 /* If this was a workshare clause, see if it had been combined
3768 with its parallel. In that case, continue looking for the
3769 clauses also on the parallel statement itself. */
3770 if (is_parallel_ctx (ctx
))
3774 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3777 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3778 OMP_CLAUSE_LASTPRIVATE
);
3784 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3788 /* Generate code to implement the REDUCTION clauses. */
3791 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3793 gimple_seq sub_seq
= NULL
;
3798 /* SIMD reductions are handled in lower_rec_input_clauses. */
3799 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3800 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3803 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3804 update in that case, otherwise use a lock. */
3805 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3806 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3808 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3810 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3820 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3822 tree var
, ref
, new_var
;
3823 enum tree_code code
;
3824 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3826 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3829 var
= OMP_CLAUSE_DECL (c
);
3830 new_var
= lookup_decl (var
, ctx
);
3831 if (is_reference (var
))
3832 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3833 ref
= build_outer_var_ref (var
, ctx
);
3834 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3836 /* reduction(-:var) sums up the partial results, so it acts
3837 identically to reduction(+:var). */
3838 if (code
== MINUS_EXPR
)
3843 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3845 addr
= save_expr (addr
);
3846 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3847 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3848 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3849 gimplify_and_add (x
, stmt_seqp
);
3853 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3855 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3857 if (is_reference (var
)
3858 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3860 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3861 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3862 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3863 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3864 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3865 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3866 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3870 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3871 ref
= build_outer_var_ref (var
, ctx
);
3872 gimplify_assign (ref
, x
, &sub_seq
);
3876 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
3878 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3880 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
3882 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
3884 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3888 /* Generate code to implement the COPYPRIVATE clauses. */
3891 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
3896 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3898 tree var
, new_var
, ref
, x
;
3900 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3902 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
3905 var
= OMP_CLAUSE_DECL (c
);
3906 by_ref
= use_pointer_for_field (var
, NULL
);
3908 ref
= build_sender_ref (var
, ctx
);
3909 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
3912 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
3913 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
3915 gimplify_assign (ref
, x
, slist
);
3917 ref
= build_receiver_ref (var
, false, ctx
);
3920 ref
= fold_convert_loc (clause_loc
,
3921 build_pointer_type (TREE_TYPE (new_var
)),
3923 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
3925 if (is_reference (var
))
3927 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
3928 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
3929 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3931 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
3932 gimplify_and_add (x
, rlist
);
3937 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3938 and REDUCTION from the sender (aka parent) side. */
3941 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
3946 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3948 tree val
, ref
, x
, var
;
3949 bool by_ref
, do_in
= false, do_out
= false;
3950 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3952 switch (OMP_CLAUSE_CODE (c
))
3954 case OMP_CLAUSE_PRIVATE
:
3955 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3958 case OMP_CLAUSE_FIRSTPRIVATE
:
3959 case OMP_CLAUSE_COPYIN
:
3960 case OMP_CLAUSE_LASTPRIVATE
:
3961 case OMP_CLAUSE_REDUCTION
:
3962 case OMP_CLAUSE__LOOPTEMP_
:
3968 val
= OMP_CLAUSE_DECL (c
);
3969 var
= lookup_decl_in_outer_ctx (val
, ctx
);
3971 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
3972 && is_global_var (var
))
3974 if (is_variable_sized (val
))
3976 by_ref
= use_pointer_for_field (val
, NULL
);
3978 switch (OMP_CLAUSE_CODE (c
))
3980 case OMP_CLAUSE_PRIVATE
:
3981 case OMP_CLAUSE_FIRSTPRIVATE
:
3982 case OMP_CLAUSE_COPYIN
:
3983 case OMP_CLAUSE__LOOPTEMP_
:
3987 case OMP_CLAUSE_LASTPRIVATE
:
3988 if (by_ref
|| is_reference (val
))
3990 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3997 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4002 case OMP_CLAUSE_REDUCTION
:
4004 do_out
= !(by_ref
|| is_reference (val
));
4013 ref
= build_sender_ref (val
, ctx
);
4014 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4015 gimplify_assign (ref
, x
, ilist
);
4016 if (is_task_ctx (ctx
))
4017 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4022 ref
= build_sender_ref (val
, ctx
);
4023 gimplify_assign (var
, ref
, olist
);
4028 /* Generate code to implement SHARED from the sender (aka parent)
4029 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4030 list things that got automatically shared. */
4033 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4035 tree var
, ovar
, nvar
, f
, x
, record_type
;
4037 if (ctx
->record_type
== NULL
)
4040 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4041 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4043 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4044 nvar
= maybe_lookup_decl (ovar
, ctx
);
4045 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4048 /* If CTX is a nested parallel directive. Find the immediately
4049 enclosing parallel or workshare construct that contains a
4050 mapping for OVAR. */
4051 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4053 if (use_pointer_for_field (ovar
, ctx
))
4055 x
= build_sender_ref (ovar
, ctx
);
4056 var
= build_fold_addr_expr (var
);
4057 gimplify_assign (x
, var
, ilist
);
4061 x
= build_sender_ref (ovar
, ctx
);
4062 gimplify_assign (x
, var
, ilist
);
4064 if (!TREE_READONLY (var
)
4065 /* We don't need to receive a new reference to a result
4066 or parm decl. In fact we may not store to it as we will
4067 invalidate any pending RSO and generate wrong gimple
4069 && !((TREE_CODE (var
) == RESULT_DECL
4070 || TREE_CODE (var
) == PARM_DECL
)
4071 && DECL_BY_REFERENCE (var
)))
4073 x
= build_sender_ref (ovar
, ctx
);
4074 gimplify_assign (var
, x
, olist
);
4081 /* A convenience function to build an empty GIMPLE_COND with just the
4085 gimple_build_cond_empty (tree cond
)
4087 enum tree_code pred_code
;
4090 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4091 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4095 /* Build the function calls to GOMP_parallel_start etc to actually
4096 generate the parallel operation. REGION is the parallel region
4097 being expanded. BB is the block where to insert the code. WS_ARGS
4098 will be set if this is a call to a combined parallel+workshare
4099 construct, it contains the list of additional arguments needed by
4100 the workshare construct. */
4103 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4104 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4106 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4107 gimple_stmt_iterator gsi
;
4109 enum built_in_function start_ix
;
4111 location_t clause_loc
;
4112 vec
<tree
, va_gc
> *args
;
4114 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4116 /* Determine what flavor of GOMP_parallel we will be
4118 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4119 if (is_combined_parallel (region
))
4121 switch (region
->inner
->type
)
4123 case GIMPLE_OMP_FOR
:
4124 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4125 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4126 + (region
->inner
->sched_kind
4127 == OMP_CLAUSE_SCHEDULE_RUNTIME
4128 ? 3 : region
->inner
->sched_kind
));
4129 start_ix
= (enum built_in_function
)start_ix2
;
4131 case GIMPLE_OMP_SECTIONS
:
4132 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4139 /* By default, the value of NUM_THREADS is zero (selected at run time)
4140 and there is no conditional. */
4142 val
= build_int_cst (unsigned_type_node
, 0);
4143 flags
= build_int_cst (unsigned_type_node
, 0);
4145 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4147 cond
= OMP_CLAUSE_IF_EXPR (c
);
4149 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4152 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4153 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4156 clause_loc
= gimple_location (entry_stmt
);
4158 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4160 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4162 /* Ensure 'val' is of the correct type. */
4163 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4165 /* If we found the clause 'if (cond)', build either
4166 (cond != 0) or (cond ? val : 1u). */
4169 gimple_stmt_iterator gsi
;
4171 cond
= gimple_boolify (cond
);
4173 if (integer_zerop (val
))
4174 val
= fold_build2_loc (clause_loc
,
4175 EQ_EXPR
, unsigned_type_node
, cond
,
4176 build_int_cst (TREE_TYPE (cond
), 0));
4179 basic_block cond_bb
, then_bb
, else_bb
;
4180 edge e
, e_then
, e_else
;
4181 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4183 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4184 if (gimple_in_ssa_p (cfun
))
4186 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4187 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4188 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4197 e
= split_block (bb
, NULL
);
4202 then_bb
= create_empty_bb (cond_bb
);
4203 else_bb
= create_empty_bb (then_bb
);
4204 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4205 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4207 stmt
= gimple_build_cond_empty (cond
);
4208 gsi
= gsi_start_bb (cond_bb
);
4209 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4211 gsi
= gsi_start_bb (then_bb
);
4212 stmt
= gimple_build_assign (tmp_then
, val
);
4213 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4215 gsi
= gsi_start_bb (else_bb
);
4216 stmt
= gimple_build_assign
4217 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4218 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4220 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4221 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4224 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4225 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4227 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4228 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4230 if (gimple_in_ssa_p (cfun
))
4232 gimple phi
= create_phi_node (tmp_join
, bb
);
4233 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4234 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4240 gsi
= gsi_start_bb (bb
);
4241 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4242 false, GSI_CONTINUE_LINKING
);
4245 gsi
= gsi_last_bb (bb
);
4246 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4248 t1
= null_pointer_node
;
4250 t1
= build_fold_addr_expr (t
);
4251 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4253 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4254 args
->quick_push (t2
);
4255 args
->quick_push (t1
);
4256 args
->quick_push (val
);
4258 args
->splice (*ws_args
);
4259 args
->quick_push (flags
);
4261 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4262 builtin_decl_explicit (start_ix
), args
);
4264 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4265 false, GSI_CONTINUE_LINKING
);
4269 /* Build the function call to GOMP_task to actually
4270 generate the task operation. BB is the block where to insert the code. */
4273 expand_task_call (basic_block bb
, gimple entry_stmt
)
4275 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4276 gimple_stmt_iterator gsi
;
4277 location_t loc
= gimple_location (entry_stmt
);
4279 clauses
= gimple_omp_task_clauses (entry_stmt
);
4281 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4283 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4285 cond
= boolean_true_node
;
4287 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4288 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4289 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4290 flags
= build_int_cst (unsigned_type_node
,
4291 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4293 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4296 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4297 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4298 build_int_cst (unsigned_type_node
, 2),
4299 build_int_cst (unsigned_type_node
, 0));
4300 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4303 depend
= OMP_CLAUSE_DECL (depend
);
4305 depend
= build_int_cst (ptr_type_node
, 0);
4307 gsi
= gsi_last_bb (bb
);
4308 t
= gimple_omp_task_data_arg (entry_stmt
);
4310 t2
= null_pointer_node
;
4312 t2
= build_fold_addr_expr_loc (loc
, t
);
4313 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4314 t
= gimple_omp_task_copy_fn (entry_stmt
);
4316 t3
= null_pointer_node
;
4318 t3
= build_fold_addr_expr_loc (loc
, t
);
4320 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4322 gimple_omp_task_arg_size (entry_stmt
),
4323 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4326 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4327 false, GSI_CONTINUE_LINKING
);
4331 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4332 catch handler and return it. This prevents programs from violating the
4333 structured block semantics with throws. */
4336 maybe_catch_exception (gimple_seq body
)
4341 if (!flag_exceptions
)
4344 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4345 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4347 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4349 g
= gimple_build_eh_must_not_throw (decl
);
4350 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4353 return gimple_seq_alloc_with_stmt (g
);
4356 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4359 vec2chain (vec
<tree
, va_gc
> *v
)
4361 tree chain
= NULL_TREE
, t
;
4364 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4366 DECL_CHAIN (t
) = chain
;
4374 /* Remove barriers in REGION->EXIT's block. Note that this is only
4375 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4376 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4377 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4381 remove_exit_barrier (struct omp_region
*region
)
4383 gimple_stmt_iterator gsi
;
4384 basic_block exit_bb
;
4388 int any_addressable_vars
= -1;
4390 exit_bb
= region
->exit
;
4392 /* If the parallel region doesn't return, we don't have REGION->EXIT
4397 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4398 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4399 statements that can appear in between are extremely limited -- no
4400 memory operations at all. Here, we allow nothing at all, so the
4401 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4402 gsi
= gsi_last_bb (exit_bb
);
4403 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4405 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4408 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4410 gsi
= gsi_last_bb (e
->src
);
4411 if (gsi_end_p (gsi
))
4413 stmt
= gsi_stmt (gsi
);
4414 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4415 && !gimple_omp_return_nowait_p (stmt
))
4417 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4418 in many cases. If there could be tasks queued, the barrier
4419 might be needed to let the tasks run before some local
4420 variable of the parallel that the task uses as shared
4421 runs out of scope. The task can be spawned either
4422 from within current function (this would be easy to check)
4423 or from some function it calls and gets passed an address
4424 of such a variable. */
4425 if (any_addressable_vars
< 0)
4427 gimple parallel_stmt
= last_stmt (region
->entry
);
4428 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4429 tree local_decls
, block
, decl
;
4432 any_addressable_vars
= 0;
4433 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4434 if (TREE_ADDRESSABLE (decl
))
4436 any_addressable_vars
= 1;
4439 for (block
= gimple_block (stmt
);
4440 !any_addressable_vars
4442 && TREE_CODE (block
) == BLOCK
;
4443 block
= BLOCK_SUPERCONTEXT (block
))
4445 for (local_decls
= BLOCK_VARS (block
);
4447 local_decls
= DECL_CHAIN (local_decls
))
4448 if (TREE_ADDRESSABLE (local_decls
))
4450 any_addressable_vars
= 1;
4453 if (block
== gimple_block (parallel_stmt
))
4457 if (!any_addressable_vars
)
4458 gimple_omp_return_set_nowait (stmt
);
4464 remove_exit_barriers (struct omp_region
*region
)
4466 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4467 remove_exit_barrier (region
);
4471 region
= region
->inner
;
4472 remove_exit_barriers (region
);
4473 while (region
->next
)
4475 region
= region
->next
;
4476 remove_exit_barriers (region
);
4481 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4482 calls. These can't be declared as const functions, but
4483 within one parallel body they are constant, so they can be
4484 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4485 which are declared const. Similarly for task body, except
4486 that in untied task omp_get_thread_num () can change at any task
4487 scheduling point. */
4490 optimize_omp_library_calls (gimple entry_stmt
)
4493 gimple_stmt_iterator gsi
;
4494 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4495 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4496 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4497 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4498 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4499 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4500 OMP_CLAUSE_UNTIED
) != NULL
);
4503 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4505 gimple call
= gsi_stmt (gsi
);
4508 if (is_gimple_call (call
)
4509 && (decl
= gimple_call_fndecl (call
))
4510 && DECL_EXTERNAL (decl
)
4511 && TREE_PUBLIC (decl
)
4512 && DECL_INITIAL (decl
) == NULL
)
4516 if (DECL_NAME (decl
) == thr_num_id
)
4518 /* In #pragma omp task untied omp_get_thread_num () can change
4519 during the execution of the task region. */
4522 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4524 else if (DECL_NAME (decl
) == num_thr_id
)
4525 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4529 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4530 || gimple_call_num_args (call
) != 0)
4533 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4536 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4537 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4538 TREE_TYPE (TREE_TYPE (built_in
))))
4541 gimple_call_set_fndecl (call
, built_in
);
4546 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4550 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4554 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4555 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4558 if (TREE_CODE (t
) == ADDR_EXPR
)
4559 recompute_tree_invariant_for_addr_expr (t
);
4561 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4565 /* Prepend TO = FROM assignment before *GSI_P. */
4568 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4570 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4571 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4572 true, GSI_SAME_STMT
);
4573 gimple stmt
= gimple_build_assign (to
, from
);
4574 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4575 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4576 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4578 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4579 gimple_regimplify_operands (stmt
, &gsi
);
4583 /* Expand the OpenMP parallel or task directive starting at REGION. */
4586 expand_omp_taskreg (struct omp_region
*region
)
4588 basic_block entry_bb
, exit_bb
, new_bb
;
4589 struct function
*child_cfun
;
4590 tree child_fn
, block
, t
;
4591 gimple_stmt_iterator gsi
;
4592 gimple entry_stmt
, stmt
;
4594 vec
<tree
, va_gc
> *ws_args
;
4596 entry_stmt
= last_stmt (region
->entry
);
4597 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4598 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4600 entry_bb
= region
->entry
;
4601 exit_bb
= region
->exit
;
4603 if (is_combined_parallel (region
))
4604 ws_args
= region
->ws_args
;
4608 if (child_cfun
->cfg
)
4610 /* Due to inlining, it may happen that we have already outlined
4611 the region, in which case all we need to do is make the
4612 sub-graph unreachable and emit the parallel call. */
4613 edge entry_succ_e
, exit_succ_e
;
4614 gimple_stmt_iterator gsi
;
4616 entry_succ_e
= single_succ_edge (entry_bb
);
4618 gsi
= gsi_last_bb (entry_bb
);
4619 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4620 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4621 gsi_remove (&gsi
, true);
4626 exit_succ_e
= single_succ_edge (exit_bb
);
4627 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4629 remove_edge_and_dominated_blocks (entry_succ_e
);
4633 unsigned srcidx
, dstidx
, num
;
4635 /* If the parallel region needs data sent from the parent
4636 function, then the very first statement (except possible
4637 tree profile counter updates) of the parallel body
4638 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4639 &.OMP_DATA_O is passed as an argument to the child function,
4640 we need to replace it with the argument as seen by the child
4643 In most cases, this will end up being the identity assignment
4644 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4645 a function call that has been inlined, the original PARM_DECL
4646 .OMP_DATA_I may have been converted into a different local
4647 variable. In which case, we need to keep the assignment. */
4648 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4650 basic_block entry_succ_bb
= single_succ (entry_bb
);
4651 gimple_stmt_iterator gsi
;
4653 gimple parcopy_stmt
= NULL
;
4655 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4659 gcc_assert (!gsi_end_p (gsi
));
4660 stmt
= gsi_stmt (gsi
);
4661 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4664 if (gimple_num_ops (stmt
) == 2)
4666 tree arg
= gimple_assign_rhs1 (stmt
);
4668 /* We're ignore the subcode because we're
4669 effectively doing a STRIP_NOPS. */
4671 if (TREE_CODE (arg
) == ADDR_EXPR
4672 && TREE_OPERAND (arg
, 0)
4673 == gimple_omp_taskreg_data_arg (entry_stmt
))
4675 parcopy_stmt
= stmt
;
4681 gcc_assert (parcopy_stmt
!= NULL
);
4682 arg
= DECL_ARGUMENTS (child_fn
);
4684 if (!gimple_in_ssa_p (cfun
))
4686 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4687 gsi_remove (&gsi
, true);
4690 /* ?? Is setting the subcode really necessary ?? */
4691 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4692 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4697 /* If we are in ssa form, we must load the value from the default
4698 definition of the argument. That should not be defined now,
4699 since the argument is not used uninitialized. */
4700 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4701 narg
= make_ssa_name (arg
, gimple_build_nop ());
4702 set_ssa_default_def (cfun
, arg
, narg
);
4703 /* ?? Is setting the subcode really necessary ?? */
4704 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4705 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4706 update_stmt (parcopy_stmt
);
4710 /* Declare local variables needed in CHILD_CFUN. */
4711 block
= DECL_INITIAL (child_fn
);
4712 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4713 /* The gimplifier could record temporaries in parallel/task block
4714 rather than in containing function's local_decls chain,
4715 which would mean cgraph missed finalizing them. Do it now. */
4716 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4717 if (TREE_CODE (t
) == VAR_DECL
4719 && !DECL_EXTERNAL (t
))
4720 varpool_finalize_decl (t
);
4721 DECL_SAVED_TREE (child_fn
) = NULL
;
4722 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4723 gimple_set_body (child_fn
, NULL
);
4724 TREE_USED (block
) = 1;
4726 /* Reset DECL_CONTEXT on function arguments. */
4727 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4728 DECL_CONTEXT (t
) = child_fn
;
4730 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4731 so that it can be moved to the child function. */
4732 gsi
= gsi_last_bb (entry_bb
);
4733 stmt
= gsi_stmt (gsi
);
4734 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4735 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4736 gsi_remove (&gsi
, true);
4737 e
= split_block (entry_bb
, stmt
);
4739 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4741 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4744 gsi
= gsi_last_bb (exit_bb
);
4745 gcc_assert (!gsi_end_p (gsi
)
4746 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4747 stmt
= gimple_build_return (NULL
);
4748 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4749 gsi_remove (&gsi
, true);
4752 /* Move the parallel region into CHILD_CFUN. */
4754 if (gimple_in_ssa_p (cfun
))
4756 init_tree_ssa (child_cfun
);
4757 init_ssa_operands (child_cfun
);
4758 child_cfun
->gimple_df
->in_ssa_p
= true;
4762 block
= gimple_block (entry_stmt
);
4764 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4766 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4767 /* When the OMP expansion process cannot guarantee an up-to-date
4768 loop tree arrange for the child function to fixup loops. */
4769 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4770 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4772 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4773 num
= vec_safe_length (child_cfun
->local_decls
);
4774 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4776 t
= (*child_cfun
->local_decls
)[srcidx
];
4777 if (DECL_CONTEXT (t
) == cfun
->decl
)
4779 if (srcidx
!= dstidx
)
4780 (*child_cfun
->local_decls
)[dstidx
] = t
;
4784 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4786 /* Inform the callgraph about the new function. */
4787 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4788 cgraph_add_new_function (child_fn
, true);
4790 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4791 fixed in a following pass. */
4792 push_cfun (child_cfun
);
4794 optimize_omp_library_calls (entry_stmt
);
4795 rebuild_cgraph_edges ();
4797 /* Some EH regions might become dead, see PR34608. If
4798 pass_cleanup_cfg isn't the first pass to happen with the
4799 new child, these dead EH edges might cause problems.
4800 Clean them up now. */
4801 if (flag_exceptions
)
4804 bool changed
= false;
4807 changed
|= gimple_purge_dead_eh_edges (bb
);
4809 cleanup_tree_cfg ();
4811 if (gimple_in_ssa_p (cfun
))
4812 update_ssa (TODO_update_ssa
);
4816 /* Emit a library call to launch the children threads. */
4817 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4818 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4820 expand_task_call (new_bb
, entry_stmt
);
4821 if (gimple_in_ssa_p (cfun
))
4822 update_ssa (TODO_update_ssa_only_virtuals
);
4826 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4827 of the combined collapse > 1 loop constructs, generate code like:
4828 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4833 count3 = (adj + N32 - N31) / STEP3;
4834 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4839 count2 = (adj + N22 - N21) / STEP2;
4840 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4845 count1 = (adj + N12 - N11) / STEP1;
4846 count = count1 * count2 * count3;
4847 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4849 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4850 of the combined loop constructs, just initialize COUNTS array
4851 from the _looptemp_ clauses. */
4853 /* NOTE: It *could* be better to moosh all of the BBs together,
4854 creating one larger BB with all the computation and the unexpected
4855 jump at the end. I.e.
4857 bool zero3, zero2, zero1, zero;
4860 count3 = (N32 - N31) /[cl] STEP3;
4862 count2 = (N22 - N21) /[cl] STEP2;
4864 count1 = (N12 - N11) /[cl] STEP1;
4865 zero = zero3 || zero2 || zero1;
4866 count = count1 * count2 * count3;
4867 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4869 After all, we expect the zero=false, and thus we expect to have to
4870 evaluate all of the comparison expressions, so short-circuiting
4871 oughtn't be a win. Since the condition isn't protecting a
4872 denominator, we're not concerned about divide-by-zero, so we can
4873 fully evaluate count even if a numerator turned out to be wrong.
4875 It seems like putting this all together would create much better
4876 scheduling opportunities, and less pressure on the chip's branch
4880 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4881 basic_block
&entry_bb
, tree
*counts
,
4882 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4883 basic_block
&l2_dom_bb
)
4885 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
4890 /* Collapsed loops need work for expansion into SSA form. */
4891 gcc_assert (!gimple_in_ssa_p (cfun
));
4893 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
4894 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
4896 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4897 isn't supposed to be handled, as the inner loop doesn't
4899 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
4900 OMP_CLAUSE__LOOPTEMP_
);
4901 gcc_assert (innerc
);
4902 for (i
= 0; i
< fd
->collapse
; i
++)
4904 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
4905 OMP_CLAUSE__LOOPTEMP_
);
4906 gcc_assert (innerc
);
4908 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
4910 counts
[0] = NULL_TREE
;
4915 for (i
= 0; i
< fd
->collapse
; i
++)
4917 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
4919 if (SSA_VAR_P (fd
->loop
.n2
)
4920 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
4921 fold_convert (itype
, fd
->loops
[i
].n1
),
4922 fold_convert (itype
, fd
->loops
[i
].n2
)))
4923 == NULL_TREE
|| !integer_onep (t
)))
4926 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
4927 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
4928 true, GSI_SAME_STMT
);
4929 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
4930 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
4931 true, GSI_SAME_STMT
);
4932 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
4933 NULL_TREE
, NULL_TREE
);
4934 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4935 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
4936 expand_omp_regimplify_p
, NULL
, NULL
)
4937 || walk_tree (gimple_cond_rhs_ptr (stmt
),
4938 expand_omp_regimplify_p
, NULL
, NULL
))
4940 *gsi
= gsi_for_stmt (stmt
);
4941 gimple_regimplify_operands (stmt
, gsi
);
4943 e
= split_block (entry_bb
, stmt
);
4944 if (zero_iter_bb
== NULL
)
4946 first_zero_iter
= i
;
4947 zero_iter_bb
= create_empty_bb (entry_bb
);
4949 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
4950 *gsi
= gsi_after_labels (zero_iter_bb
);
4951 stmt
= gimple_build_assign (fd
->loop
.n2
,
4952 build_zero_cst (type
));
4953 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4954 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
4957 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
4958 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
4959 e
->flags
= EDGE_TRUE_VALUE
;
4960 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
4961 if (l2_dom_bb
== NULL
)
4962 l2_dom_bb
= entry_bb
;
4964 *gsi
= gsi_last_bb (entry_bb
);
4967 if (POINTER_TYPE_P (itype
))
4968 itype
= signed_type_for (itype
);
4969 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
4971 t
= fold_build2 (PLUS_EXPR
, itype
,
4972 fold_convert (itype
, fd
->loops
[i
].step
), t
);
4973 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
4974 fold_convert (itype
, fd
->loops
[i
].n2
));
4975 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
4976 fold_convert (itype
, fd
->loops
[i
].n1
));
4977 /* ?? We could probably use CEIL_DIV_EXPR instead of
4978 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
4979 generate the same code in the end because generically we
4980 don't know that the values involved must be negative for
4982 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
4983 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4984 fold_build1 (NEGATE_EXPR
, itype
, t
),
4985 fold_build1 (NEGATE_EXPR
, itype
,
4986 fold_convert (itype
,
4987 fd
->loops
[i
].step
)));
4989 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
4990 fold_convert (itype
, fd
->loops
[i
].step
));
4991 t
= fold_convert (type
, t
);
4992 if (TREE_CODE (t
) == INTEGER_CST
)
4996 counts
[i
] = create_tmp_reg (type
, ".count");
4997 expand_omp_build_assign (gsi
, counts
[i
], t
);
4999 if (SSA_VAR_P (fd
->loop
.n2
))
5004 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5005 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5011 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5013 V3 = N31 + (T % count3) * STEP3;
5015 V2 = N21 + (T % count2) * STEP2;
5017 V1 = N11 + T * STEP1;
5018 if this loop doesn't have an inner loop construct combined with it.
5019 If it does have an inner loop construct combined with it and the
5020 iteration count isn't known constant, store values from counts array
5021 into its _looptemp_ temporaries instead. */
5024 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5025 tree
*counts
, gimple inner_stmt
, tree startvar
)
5028 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5030 /* If fd->loop.n2 is constant, then no propagation of the counts
5031 is needed, they are constant. */
5032 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5035 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5036 ? gimple_omp_parallel_clauses (inner_stmt
)
5037 : gimple_omp_for_clauses (inner_stmt
);
5038 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5039 isn't supposed to be handled, as the inner loop doesn't
5041 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5042 gcc_assert (innerc
);
5043 for (i
= 0; i
< fd
->collapse
; i
++)
5045 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5046 OMP_CLAUSE__LOOPTEMP_
);
5047 gcc_assert (innerc
);
5050 tree tem
= OMP_CLAUSE_DECL (innerc
);
5051 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5052 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5053 false, GSI_CONTINUE_LINKING
);
5054 gimple stmt
= gimple_build_assign (tem
, t
);
5055 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5061 tree type
= TREE_TYPE (fd
->loop
.v
);
5062 tree tem
= create_tmp_reg (type
, ".tem");
5063 gimple stmt
= gimple_build_assign (tem
, startvar
);
5064 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5066 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5068 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5070 if (POINTER_TYPE_P (vtype
))
5071 itype
= signed_type_for (vtype
);
5073 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5076 t
= fold_convert (itype
, t
);
5077 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5078 fold_convert (itype
, fd
->loops
[i
].step
));
5079 if (POINTER_TYPE_P (vtype
))
5080 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5082 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5083 t
= force_gimple_operand_gsi (gsi
, t
,
5084 DECL_P (fd
->loops
[i
].v
)
5085 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5087 GSI_CONTINUE_LINKING
);
5088 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5089 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5092 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5093 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5094 false, GSI_CONTINUE_LINKING
);
5095 stmt
= gimple_build_assign (tem
, t
);
5096 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5102 /* Helper function for expand_omp_for_*. Generate code like:
5105 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5109 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5116 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5117 basic_block body_bb
)
5119 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5121 gimple_stmt_iterator gsi
;
5127 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5129 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5131 bb
= create_empty_bb (last_bb
);
5133 add_bb_to_loop (bb
, last_bb
->loop_father
);
5134 gsi
= gsi_start_bb (bb
);
5136 if (i
< fd
->collapse
- 1)
5138 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5139 e
->probability
= REG_BR_PROB_BASE
/ 8;
5141 t
= fd
->loops
[i
+ 1].n1
;
5142 t
= force_gimple_operand_gsi (&gsi
, t
,
5143 DECL_P (fd
->loops
[i
+ 1].v
)
5144 && TREE_ADDRESSABLE (fd
->loops
[i
5147 GSI_CONTINUE_LINKING
);
5148 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5149 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5154 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5156 if (POINTER_TYPE_P (vtype
))
5157 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5159 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5160 t
= force_gimple_operand_gsi (&gsi
, t
,
5161 DECL_P (fd
->loops
[i
].v
)
5162 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5163 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5164 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5165 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5169 t
= fd
->loops
[i
].n2
;
5170 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5171 false, GSI_CONTINUE_LINKING
);
5172 tree v
= fd
->loops
[i
].v
;
5173 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5174 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5175 false, GSI_CONTINUE_LINKING
);
5176 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5177 stmt
= gimple_build_cond_empty (t
);
5178 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5179 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5180 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5183 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5191 /* A subroutine of expand_omp_for. Generate code for a parallel
5192 loop with any schedule. Given parameters:
5194 for (V = N1; V cond N2; V += STEP) BODY;
5196 where COND is "<" or ">", we generate pseudocode
5198 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5199 if (more) goto L0; else goto L3;
5206 if (V cond iend) goto L1; else goto L2;
5208 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5211 If this is a combined omp parallel loop, instead of the call to
5212 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5213 If this is gimple_omp_for_combined_p loop, then instead of assigning
5214 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5215 inner GIMPLE_OMP_FOR and V += STEP; and
5216 if (V cond iend) goto L1; else goto L2; are removed.
5218 For collapsed loops, given parameters:
5220 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5221 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5222 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5225 we generate pseudocode
5227 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5232 count3 = (adj + N32 - N31) / STEP3;
5233 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5238 count2 = (adj + N22 - N21) / STEP2;
5239 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5244 count1 = (adj + N12 - N11) / STEP1;
5245 count = count1 * count2 * count3;
5250 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5251 if (more) goto L0; else goto L3;
5255 V3 = N31 + (T % count3) * STEP3;
5257 V2 = N21 + (T % count2) * STEP2;
5259 V1 = N11 + T * STEP1;
5264 if (V < iend) goto L10; else goto L2;
5267 if (V3 cond3 N32) goto L1; else goto L11;
5271 if (V2 cond2 N22) goto L1; else goto L12;
5277 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5283 expand_omp_for_generic (struct omp_region
*region
,
5284 struct omp_for_data
*fd
,
5285 enum built_in_function start_fn
,
5286 enum built_in_function next_fn
,
5289 tree type
, istart0
, iend0
, iend
;
5290 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5291 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5292 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5293 gimple_stmt_iterator gsi
;
5295 bool in_combined_parallel
= is_combined_parallel (region
);
5296 bool broken_loop
= region
->cont
== NULL
;
5298 tree
*counts
= NULL
;
5301 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5302 gcc_assert (fd
->iter_type
== long_integer_type_node
5303 || !in_combined_parallel
);
5305 type
= TREE_TYPE (fd
->loop
.v
);
5306 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5307 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5308 TREE_ADDRESSABLE (istart0
) = 1;
5309 TREE_ADDRESSABLE (iend0
) = 1;
5311 /* See if we need to bias by LLONG_MIN. */
5312 if (fd
->iter_type
== long_long_unsigned_type_node
5313 && TREE_CODE (type
) == INTEGER_TYPE
5314 && !TYPE_UNSIGNED (type
))
5318 if (fd
->loop
.cond_code
== LT_EXPR
)
5321 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5325 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5328 if (TREE_CODE (n1
) != INTEGER_CST
5329 || TREE_CODE (n2
) != INTEGER_CST
5330 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5331 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5334 entry_bb
= region
->entry
;
5335 cont_bb
= region
->cont
;
5337 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5338 gcc_assert (broken_loop
5339 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5340 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5341 l1_bb
= single_succ (l0_bb
);
5344 l2_bb
= create_empty_bb (cont_bb
);
5345 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5346 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5350 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5351 exit_bb
= region
->exit
;
5353 gsi
= gsi_last_bb (entry_bb
);
5355 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5356 if (fd
->collapse
> 1)
5358 int first_zero_iter
= -1;
5359 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5361 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5362 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5363 zero_iter_bb
, first_zero_iter
,
5368 /* Some counts[i] vars might be uninitialized if
5369 some loop has zero iterations. But the body shouldn't
5370 be executed in that case, so just avoid uninit warnings. */
5371 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5372 if (SSA_VAR_P (counts
[i
]))
5373 TREE_NO_WARNING (counts
[i
]) = 1;
5375 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5377 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5378 gsi
= gsi_last_bb (entry_bb
);
5379 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5380 get_immediate_dominator (CDI_DOMINATORS
,
5384 if (in_combined_parallel
)
5386 /* In a combined parallel loop, emit a call to
5387 GOMP_loop_foo_next. */
5388 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5389 build_fold_addr_expr (istart0
),
5390 build_fold_addr_expr (iend0
));
5394 tree t0
, t1
, t2
, t3
, t4
;
5395 /* If this is not a combined parallel loop, emit a call to
5396 GOMP_loop_foo_start in ENTRY_BB. */
5397 t4
= build_fold_addr_expr (iend0
);
5398 t3
= build_fold_addr_expr (istart0
);
5399 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5402 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5404 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5405 OMP_CLAUSE__LOOPTEMP_
);
5406 gcc_assert (innerc
);
5407 t0
= OMP_CLAUSE_DECL (innerc
);
5408 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5409 OMP_CLAUSE__LOOPTEMP_
);
5410 gcc_assert (innerc
);
5411 t1
= OMP_CLAUSE_DECL (innerc
);
5413 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5414 && TYPE_PRECISION (TREE_TYPE (t0
))
5415 != TYPE_PRECISION (fd
->iter_type
))
5417 /* Avoid casting pointers to integer of a different size. */
5418 tree itype
= signed_type_for (type
);
5419 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5420 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5424 t1
= fold_convert (fd
->iter_type
, t1
);
5425 t0
= fold_convert (fd
->iter_type
, t0
);
5429 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5430 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5432 if (fd
->iter_type
== long_integer_type_node
)
5436 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5437 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5438 6, t0
, t1
, t2
, t
, t3
, t4
);
5441 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5442 5, t0
, t1
, t2
, t3
, t4
);
5450 /* The GOMP_loop_ull_*start functions have additional boolean
5451 argument, true for < loops and false for > loops.
5452 In Fortran, the C bool type can be different from
5453 boolean_type_node. */
5454 bfn_decl
= builtin_decl_explicit (start_fn
);
5455 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5456 t5
= build_int_cst (c_bool_type
,
5457 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5460 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5461 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5462 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5465 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5466 6, t5
, t0
, t1
, t2
, t3
, t4
);
5469 if (TREE_TYPE (t
) != boolean_type_node
)
5470 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5471 t
, build_int_cst (TREE_TYPE (t
), 0));
5472 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5473 true, GSI_SAME_STMT
);
5474 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5476 /* Remove the GIMPLE_OMP_FOR statement. */
5477 gsi_remove (&gsi
, true);
5479 /* Iteration setup for sequential loop goes in L0_BB. */
5480 tree startvar
= fd
->loop
.v
;
5481 tree endvar
= NULL_TREE
;
5483 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5485 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5486 && gimple_omp_for_kind (inner_stmt
)
5487 == GF_OMP_FOR_KIND_SIMD
);
5488 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5489 OMP_CLAUSE__LOOPTEMP_
);
5490 gcc_assert (innerc
);
5491 startvar
= OMP_CLAUSE_DECL (innerc
);
5492 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5493 OMP_CLAUSE__LOOPTEMP_
);
5494 gcc_assert (innerc
);
5495 endvar
= OMP_CLAUSE_DECL (innerc
);
5498 gsi
= gsi_start_bb (l0_bb
);
5501 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5502 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5503 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5504 t
= fold_convert (TREE_TYPE (startvar
), t
);
5505 t
= force_gimple_operand_gsi (&gsi
, t
,
5507 && TREE_ADDRESSABLE (startvar
),
5508 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5509 stmt
= gimple_build_assign (startvar
, t
);
5510 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5514 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5515 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5516 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5517 t
= fold_convert (TREE_TYPE (startvar
), t
);
5518 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5519 false, GSI_CONTINUE_LINKING
);
5522 stmt
= gimple_build_assign (endvar
, iend
);
5523 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5525 if (fd
->collapse
> 1)
5526 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5530 /* Code to control the increment and predicate for the sequential
5531 loop goes in the CONT_BB. */
5532 gsi
= gsi_last_bb (cont_bb
);
5533 stmt
= gsi_stmt (gsi
);
5534 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5535 vmain
= gimple_omp_continue_control_use (stmt
);
5536 vback
= gimple_omp_continue_control_def (stmt
);
5538 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5540 if (POINTER_TYPE_P (type
))
5541 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5543 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5544 t
= force_gimple_operand_gsi (&gsi
, t
,
5546 && TREE_ADDRESSABLE (vback
),
5547 NULL_TREE
, true, GSI_SAME_STMT
);
5548 stmt
= gimple_build_assign (vback
, t
);
5549 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5551 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5552 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5554 stmt
= gimple_build_cond_empty (t
);
5555 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5558 /* Remove GIMPLE_OMP_CONTINUE. */
5559 gsi_remove (&gsi
, true);
5561 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5562 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5564 /* Emit code to get the next parallel iteration in L2_BB. */
5565 gsi
= gsi_start_bb (l2_bb
);
5567 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5568 build_fold_addr_expr (istart0
),
5569 build_fold_addr_expr (iend0
));
5570 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5571 false, GSI_CONTINUE_LINKING
);
5572 if (TREE_TYPE (t
) != boolean_type_node
)
5573 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5574 t
, build_int_cst (TREE_TYPE (t
), 0));
5575 stmt
= gimple_build_cond_empty (t
);
5576 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5579 /* Add the loop cleanup function. */
5580 gsi
= gsi_last_bb (exit_bb
);
5581 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5582 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5583 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5584 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5586 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5587 stmt
= gimple_build_call (t
, 0);
5588 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5589 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5590 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5591 gsi_remove (&gsi
, true);
5593 /* Connect the new blocks. */
5594 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5595 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5601 e
= find_edge (cont_bb
, l3_bb
);
5602 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5604 phis
= phi_nodes (l3_bb
);
5605 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5607 gimple phi
= gsi_stmt (gsi
);
5608 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5609 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5613 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5615 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5616 e
= find_edge (cont_bb
, l1_bb
);
5617 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5622 else if (fd
->collapse
> 1)
5625 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5628 e
->flags
= EDGE_TRUE_VALUE
;
5631 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5632 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5636 e
= find_edge (cont_bb
, l2_bb
);
5637 e
->flags
= EDGE_FALLTHRU
;
5639 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5641 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5642 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5643 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5644 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5645 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5646 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5647 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5648 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5650 struct loop
*outer_loop
= alloc_loop ();
5651 outer_loop
->header
= l0_bb
;
5652 outer_loop
->latch
= l2_bb
;
5653 add_loop (outer_loop
, l0_bb
->loop_father
);
5655 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5657 struct loop
*loop
= alloc_loop ();
5658 loop
->header
= l1_bb
;
5659 /* The loop may have multiple latches. */
5660 add_loop (loop
, outer_loop
);
5666 /* A subroutine of expand_omp_for. Generate code for a parallel
5667 loop with static schedule and no specified chunk size. Given
5670 for (V = N1; V cond N2; V += STEP) BODY;
5672 where COND is "<" or ">", we generate pseudocode
5674 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5679 if ((__typeof (V)) -1 > 0 && cond is >)
5680 n = -(adj + N2 - N1) / -STEP;
5682 n = (adj + N2 - N1) / STEP;
5685 if (threadid < tt) goto L3; else goto L4;
5690 s0 = q * threadid + tt;
5693 if (s0 >= e0) goto L2; else goto L0;
5699 if (V cond e) goto L1;
5704 expand_omp_for_static_nochunk (struct omp_region
*region
,
5705 struct omp_for_data
*fd
,
5708 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5709 tree type
, itype
, vmain
, vback
;
5710 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5711 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5713 gimple_stmt_iterator gsi
;
5716 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5717 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5718 bool broken_loop
= region
->cont
== NULL
;
5719 tree
*counts
= NULL
;
5722 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5723 if (POINTER_TYPE_P (type
))
5724 itype
= signed_type_for (type
);
5726 entry_bb
= region
->entry
;
5727 cont_bb
= region
->cont
;
5728 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5729 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5730 gcc_assert (broken_loop
5731 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5732 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5733 body_bb
= single_succ (seq_start_bb
);
5736 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5737 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5739 exit_bb
= region
->exit
;
5741 /* Iteration space partitioning goes in ENTRY_BB. */
5742 gsi
= gsi_last_bb (entry_bb
);
5743 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5745 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5747 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5748 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5751 if (fd
->collapse
> 1)
5753 int first_zero_iter
= -1;
5754 basic_block l2_dom_bb
= NULL
;
5756 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5757 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5758 fin_bb
, first_zero_iter
,
5762 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5763 t
= integer_one_node
;
5765 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5766 fold_convert (type
, fd
->loop
.n1
),
5767 fold_convert (type
, fd
->loop
.n2
));
5768 if (fd
->collapse
== 1
5769 && TYPE_UNSIGNED (type
)
5770 && (t
== NULL_TREE
|| !integer_onep (t
)))
5772 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5773 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5774 true, GSI_SAME_STMT
);
5775 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5776 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5777 true, GSI_SAME_STMT
);
5778 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5779 NULL_TREE
, NULL_TREE
);
5780 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5781 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5782 expand_omp_regimplify_p
, NULL
, NULL
)
5783 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5784 expand_omp_regimplify_p
, NULL
, NULL
))
5786 gsi
= gsi_for_stmt (stmt
);
5787 gimple_regimplify_operands (stmt
, &gsi
);
5789 ep
= split_block (entry_bb
, stmt
);
5790 ep
->flags
= EDGE_TRUE_VALUE
;
5791 entry_bb
= ep
->dest
;
5792 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5793 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5794 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5795 if (gimple_in_ssa_p (cfun
))
5797 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5798 for (gsi
= gsi_start_phis (fin_bb
);
5799 !gsi_end_p (gsi
); gsi_next (&gsi
))
5801 gimple phi
= gsi_stmt (gsi
);
5802 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5803 ep
, UNKNOWN_LOCATION
);
5806 gsi
= gsi_last_bb (entry_bb
);
5809 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5810 t
= fold_convert (itype
, t
);
5811 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5812 true, GSI_SAME_STMT
);
5814 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5815 t
= fold_convert (itype
, t
);
5816 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5817 true, GSI_SAME_STMT
);
5821 step
= fd
->loop
.step
;
5822 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5824 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5825 OMP_CLAUSE__LOOPTEMP_
);
5826 gcc_assert (innerc
);
5827 n1
= OMP_CLAUSE_DECL (innerc
);
5828 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5829 OMP_CLAUSE__LOOPTEMP_
);
5830 gcc_assert (innerc
);
5831 n2
= OMP_CLAUSE_DECL (innerc
);
5833 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5834 true, NULL_TREE
, true, GSI_SAME_STMT
);
5835 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5836 true, NULL_TREE
, true, GSI_SAME_STMT
);
5837 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5838 true, NULL_TREE
, true, GSI_SAME_STMT
);
5840 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5841 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5842 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5843 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5844 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5845 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5846 fold_build1 (NEGATE_EXPR
, itype
, t
),
5847 fold_build1 (NEGATE_EXPR
, itype
, step
));
5849 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5850 t
= fold_convert (itype
, t
);
5851 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5853 q
= create_tmp_reg (itype
, "q");
5854 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5855 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5856 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5858 tt
= create_tmp_reg (itype
, "tt");
5859 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5860 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5861 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5863 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5864 stmt
= gimple_build_cond_empty (t
);
5865 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5867 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5868 gsi
= gsi_last_bb (second_bb
);
5869 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5871 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5873 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5874 build_int_cst (itype
, 1));
5875 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5877 third_bb
= split_block (second_bb
, stmt
)->dest
;
5878 gsi
= gsi_last_bb (third_bb
);
5879 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5881 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
5882 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
5883 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5885 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
5886 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5888 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
5889 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5891 /* Remove the GIMPLE_OMP_FOR statement. */
5892 gsi_remove (&gsi
, true);
5894 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5895 gsi
= gsi_start_bb (seq_start_bb
);
5897 tree startvar
= fd
->loop
.v
;
5898 tree endvar
= NULL_TREE
;
5900 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5902 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5903 ? gimple_omp_parallel_clauses (inner_stmt
)
5904 : gimple_omp_for_clauses (inner_stmt
);
5905 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5906 gcc_assert (innerc
);
5907 startvar
= OMP_CLAUSE_DECL (innerc
);
5908 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5909 OMP_CLAUSE__LOOPTEMP_
);
5910 gcc_assert (innerc
);
5911 endvar
= OMP_CLAUSE_DECL (innerc
);
5913 t
= fold_convert (itype
, s0
);
5914 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5915 if (POINTER_TYPE_P (type
))
5916 t
= fold_build_pointer_plus (n1
, t
);
5918 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5919 t
= fold_convert (TREE_TYPE (startvar
), t
);
5920 t
= force_gimple_operand_gsi (&gsi
, t
,
5922 && TREE_ADDRESSABLE (startvar
),
5923 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5924 stmt
= gimple_build_assign (startvar
, t
);
5925 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5927 t
= fold_convert (itype
, e0
);
5928 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5929 if (POINTER_TYPE_P (type
))
5930 t
= fold_build_pointer_plus (n1
, t
);
5932 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5933 t
= fold_convert (TREE_TYPE (startvar
), t
);
5934 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5935 false, GSI_CONTINUE_LINKING
);
5938 stmt
= gimple_build_assign (endvar
, e
);
5939 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5941 if (fd
->collapse
> 1)
5942 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5946 /* The code controlling the sequential loop replaces the
5947 GIMPLE_OMP_CONTINUE. */
5948 gsi
= gsi_last_bb (cont_bb
);
5949 stmt
= gsi_stmt (gsi
);
5950 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5951 vmain
= gimple_omp_continue_control_use (stmt
);
5952 vback
= gimple_omp_continue_control_def (stmt
);
5954 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5956 if (POINTER_TYPE_P (type
))
5957 t
= fold_build_pointer_plus (vmain
, step
);
5959 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
5960 t
= force_gimple_operand_gsi (&gsi
, t
,
5962 && TREE_ADDRESSABLE (vback
),
5963 NULL_TREE
, true, GSI_SAME_STMT
);
5964 stmt
= gimple_build_assign (vback
, t
);
5965 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5967 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5968 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
5970 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5973 /* Remove the GIMPLE_OMP_CONTINUE statement. */
5974 gsi_remove (&gsi
, true);
5976 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5977 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
5980 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5981 gsi
= gsi_last_bb (exit_bb
);
5982 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5984 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
5985 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
5987 gsi_remove (&gsi
, true);
5989 /* Connect all the blocks. */
5990 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
5991 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
5992 ep
= find_edge (entry_bb
, second_bb
);
5993 ep
->flags
= EDGE_TRUE_VALUE
;
5994 ep
->probability
= REG_BR_PROB_BASE
/ 4;
5995 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
5996 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6000 ep
= find_edge (cont_bb
, body_bb
);
6001 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6006 else if (fd
->collapse
> 1)
6009 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6012 ep
->flags
= EDGE_TRUE_VALUE
;
6013 find_edge (cont_bb
, fin_bb
)->flags
6014 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6017 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6018 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6019 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6021 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6022 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6023 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6024 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6026 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6028 struct loop
*loop
= alloc_loop ();
6029 loop
->header
= body_bb
;
6030 if (collapse_bb
== NULL
)
6031 loop
->latch
= cont_bb
;
6032 add_loop (loop
, body_bb
->loop_father
);
6037 /* A subroutine of expand_omp_for. Generate code for a parallel
6038 loop with static schedule and a specified chunk size. Given
6041 for (V = N1; V cond N2; V += STEP) BODY;
6043 where COND is "<" or ">", we generate pseudocode
6045 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6050 if ((__typeof (V)) -1 > 0 && cond is >)
6051 n = -(adj + N2 - N1) / -STEP;
6053 n = (adj + N2 - N1) / STEP;
6055 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6056 here so that V is defined
6057 if the loop is not entered
6059 s0 = (trip * nthreads + threadid) * CHUNK;
6060 e0 = min(s0 + CHUNK, n);
6061 if (s0 < n) goto L1; else goto L4;
6068 if (V cond e) goto L2; else goto L3;
6076 expand_omp_for_static_chunk (struct omp_region
*region
,
6077 struct omp_for_data
*fd
, gimple inner_stmt
)
6079 tree n
, s0
, e0
, e
, t
;
6080 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6081 tree type
, itype
, v_main
, v_back
, v_extra
;
6082 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6083 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6084 gimple_stmt_iterator si
;
6087 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6088 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6089 bool broken_loop
= region
->cont
== NULL
;
6090 tree
*counts
= NULL
;
6093 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6094 if (POINTER_TYPE_P (type
))
6095 itype
= signed_type_for (type
);
6097 entry_bb
= region
->entry
;
6098 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6100 iter_part_bb
= se
->dest
;
6101 cont_bb
= region
->cont
;
6102 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6103 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6104 gcc_assert (broken_loop
6105 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6106 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6107 body_bb
= single_succ (seq_start_bb
);
6110 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6111 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6112 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6114 exit_bb
= region
->exit
;
6116 /* Trip and adjustment setup goes in ENTRY_BB. */
6117 si
= gsi_last_bb (entry_bb
);
6118 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6120 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6122 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6123 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6126 if (fd
->collapse
> 1)
6128 int first_zero_iter
= -1;
6129 basic_block l2_dom_bb
= NULL
;
6131 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6132 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6133 fin_bb
, first_zero_iter
,
6137 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6138 t
= integer_one_node
;
6140 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6141 fold_convert (type
, fd
->loop
.n1
),
6142 fold_convert (type
, fd
->loop
.n2
));
6143 if (fd
->collapse
== 1
6144 && TYPE_UNSIGNED (type
)
6145 && (t
== NULL_TREE
|| !integer_onep (t
)))
6147 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6148 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6149 true, GSI_SAME_STMT
);
6150 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6151 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6152 true, GSI_SAME_STMT
);
6153 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6154 NULL_TREE
, NULL_TREE
);
6155 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6156 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6157 expand_omp_regimplify_p
, NULL
, NULL
)
6158 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6159 expand_omp_regimplify_p
, NULL
, NULL
))
6161 si
= gsi_for_stmt (stmt
);
6162 gimple_regimplify_operands (stmt
, &si
);
6164 se
= split_block (entry_bb
, stmt
);
6165 se
->flags
= EDGE_TRUE_VALUE
;
6166 entry_bb
= se
->dest
;
6167 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6168 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6169 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6170 if (gimple_in_ssa_p (cfun
))
6172 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6173 for (si
= gsi_start_phis (fin_bb
);
6174 !gsi_end_p (si
); gsi_next (&si
))
6176 gimple phi
= gsi_stmt (si
);
6177 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6178 se
, UNKNOWN_LOCATION
);
6181 si
= gsi_last_bb (entry_bb
);
6184 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6185 t
= fold_convert (itype
, t
);
6186 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6187 true, GSI_SAME_STMT
);
6189 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6190 t
= fold_convert (itype
, t
);
6191 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6192 true, GSI_SAME_STMT
);
6196 step
= fd
->loop
.step
;
6197 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6199 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6200 OMP_CLAUSE__LOOPTEMP_
);
6201 gcc_assert (innerc
);
6202 n1
= OMP_CLAUSE_DECL (innerc
);
6203 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6204 OMP_CLAUSE__LOOPTEMP_
);
6205 gcc_assert (innerc
);
6206 n2
= OMP_CLAUSE_DECL (innerc
);
6208 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6209 true, NULL_TREE
, true, GSI_SAME_STMT
);
6210 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6211 true, NULL_TREE
, true, GSI_SAME_STMT
);
6212 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6213 true, NULL_TREE
, true, GSI_SAME_STMT
);
6215 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6216 true, NULL_TREE
, true, GSI_SAME_STMT
);
6218 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6219 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6220 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6221 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6222 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6223 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6224 fold_build1 (NEGATE_EXPR
, itype
, t
),
6225 fold_build1 (NEGATE_EXPR
, itype
, step
));
6227 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6228 t
= fold_convert (itype
, t
);
6229 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6230 true, GSI_SAME_STMT
);
6232 trip_var
= create_tmp_reg (itype
, ".trip");
6233 if (gimple_in_ssa_p (cfun
))
6235 trip_init
= make_ssa_name (trip_var
, NULL
);
6236 trip_main
= make_ssa_name (trip_var
, NULL
);
6237 trip_back
= make_ssa_name (trip_var
, NULL
);
6241 trip_init
= trip_var
;
6242 trip_main
= trip_var
;
6243 trip_back
= trip_var
;
6246 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6247 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6249 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6250 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6251 if (POINTER_TYPE_P (type
))
6252 t
= fold_build_pointer_plus (n1
, t
);
6254 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6255 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6256 true, GSI_SAME_STMT
);
6258 /* Remove the GIMPLE_OMP_FOR. */
6259 gsi_remove (&si
, true);
6261 /* Iteration space partitioning goes in ITER_PART_BB. */
6262 si
= gsi_last_bb (iter_part_bb
);
6264 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6265 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6266 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6267 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6268 false, GSI_CONTINUE_LINKING
);
6270 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6271 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6272 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6273 false, GSI_CONTINUE_LINKING
);
6275 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6276 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6278 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6279 si
= gsi_start_bb (seq_start_bb
);
6281 tree startvar
= fd
->loop
.v
;
6282 tree endvar
= NULL_TREE
;
6284 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6286 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6287 ? gimple_omp_parallel_clauses (inner_stmt
)
6288 : gimple_omp_for_clauses (inner_stmt
);
6289 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6290 gcc_assert (innerc
);
6291 startvar
= OMP_CLAUSE_DECL (innerc
);
6292 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6293 OMP_CLAUSE__LOOPTEMP_
);
6294 gcc_assert (innerc
);
6295 endvar
= OMP_CLAUSE_DECL (innerc
);
6298 t
= fold_convert (itype
, s0
);
6299 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6300 if (POINTER_TYPE_P (type
))
6301 t
= fold_build_pointer_plus (n1
, t
);
6303 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6304 t
= fold_convert (TREE_TYPE (startvar
), t
);
6305 t
= force_gimple_operand_gsi (&si
, t
,
6307 && TREE_ADDRESSABLE (startvar
),
6308 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6309 stmt
= gimple_build_assign (startvar
, t
);
6310 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6312 t
= fold_convert (itype
, e0
);
6313 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6314 if (POINTER_TYPE_P (type
))
6315 t
= fold_build_pointer_plus (n1
, t
);
6317 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6318 t
= fold_convert (TREE_TYPE (startvar
), t
);
6319 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6320 false, GSI_CONTINUE_LINKING
);
6323 stmt
= gimple_build_assign (endvar
, e
);
6324 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6326 if (fd
->collapse
> 1)
6327 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6331 /* The code controlling the sequential loop goes in CONT_BB,
6332 replacing the GIMPLE_OMP_CONTINUE. */
6333 si
= gsi_last_bb (cont_bb
);
6334 stmt
= gsi_stmt (si
);
6335 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6336 v_main
= gimple_omp_continue_control_use (stmt
);
6337 v_back
= gimple_omp_continue_control_def (stmt
);
6339 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6341 if (POINTER_TYPE_P (type
))
6342 t
= fold_build_pointer_plus (v_main
, step
);
6344 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6345 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6346 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6347 true, GSI_SAME_STMT
);
6348 stmt
= gimple_build_assign (v_back
, t
);
6349 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6351 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6352 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6354 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6357 /* Remove GIMPLE_OMP_CONTINUE. */
6358 gsi_remove (&si
, true);
6360 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6361 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6363 /* Trip update code goes into TRIP_UPDATE_BB. */
6364 si
= gsi_start_bb (trip_update_bb
);
6366 t
= build_int_cst (itype
, 1);
6367 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6368 stmt
= gimple_build_assign (trip_back
, t
);
6369 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6372 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6373 si
= gsi_last_bb (exit_bb
);
6374 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6376 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6377 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6379 gsi_remove (&si
, true);
6381 /* Connect the new blocks. */
6382 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6383 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6387 se
= find_edge (cont_bb
, body_bb
);
6388 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6393 else if (fd
->collapse
> 1)
6396 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6399 se
->flags
= EDGE_TRUE_VALUE
;
6400 find_edge (cont_bb
, trip_update_bb
)->flags
6401 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6403 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6406 if (gimple_in_ssa_p (cfun
))
6408 gimple_stmt_iterator psi
;
6411 edge_var_map_vector
*head
;
6415 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6417 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6418 remove arguments of the phi nodes in fin_bb. We need to create
6419 appropriate phi nodes in iter_part_bb instead. */
6420 se
= single_pred_edge (fin_bb
);
6421 re
= single_succ_edge (trip_update_bb
);
6422 head
= redirect_edge_var_map_vector (re
);
6423 ene
= single_succ_edge (entry_bb
);
6425 psi
= gsi_start_phis (fin_bb
);
6426 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6427 gsi_next (&psi
), ++i
)
6430 source_location locus
;
6432 phi
= gsi_stmt (psi
);
6433 t
= gimple_phi_result (phi
);
6434 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6435 nphi
= create_phi_node (t
, iter_part_bb
);
6437 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6438 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6440 /* A special case -- fd->loop.v is not yet computed in
6441 iter_part_bb, we need to use v_extra instead. */
6442 if (t
== fd
->loop
.v
)
6444 add_phi_arg (nphi
, t
, ene
, locus
);
6445 locus
= redirect_edge_var_map_location (vm
);
6446 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6448 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6449 redirect_edge_var_map_clear (re
);
6452 psi
= gsi_start_phis (fin_bb
);
6453 if (gsi_end_p (psi
))
6455 remove_phi_node (&psi
, false);
6458 /* Make phi node for trip. */
6459 phi
= create_phi_node (trip_main
, iter_part_bb
);
6460 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6462 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6467 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6468 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6469 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6470 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6471 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6472 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6473 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6474 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6475 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6479 struct loop
*trip_loop
= alloc_loop ();
6480 trip_loop
->header
= iter_part_bb
;
6481 trip_loop
->latch
= trip_update_bb
;
6482 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6484 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6486 struct loop
*loop
= alloc_loop ();
6487 loop
->header
= body_bb
;
6488 loop
->latch
= cont_bb
;
6489 add_loop (loop
, trip_loop
);
6495 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6496 loop. Given parameters:
6498 for (V = N1; V cond N2; V += STEP) BODY;
6500 where COND is "<" or ">", we generate pseudocode
6508 if (V cond N2) goto L0; else goto L2;
6511 For collapsed loops, given parameters:
6513 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6514 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6515 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6518 we generate pseudocode
6524 count3 = (adj + N32 - N31) / STEP3;
6529 count2 = (adj + N22 - N21) / STEP2;
6534 count1 = (adj + N12 - N11) / STEP1;
6535 count = count1 * count2 * count3;
6545 V2 += (V3 cond3 N32) ? 0 : STEP2;
6546 V3 = (V3 cond3 N32) ? V3 : N31;
6547 V1 += (V2 cond2 N22) ? 0 : STEP1;
6548 V2 = (V2 cond2 N22) ? V2 : N21;
6550 if (V < count) goto L0; else goto L2;
6556 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6559 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6560 gimple_stmt_iterator gsi
;
6562 bool broken_loop
= region
->cont
== NULL
;
6564 tree
*counts
= NULL
;
6566 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6567 OMP_CLAUSE_SAFELEN
);
6568 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6569 OMP_CLAUSE__SIMDUID_
);
6572 type
= TREE_TYPE (fd
->loop
.v
);
6573 entry_bb
= region
->entry
;
6574 cont_bb
= region
->cont
;
6575 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6576 gcc_assert (broken_loop
6577 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6578 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6581 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6582 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6583 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6584 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6588 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6589 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6590 l2_bb
= single_succ (l1_bb
);
6592 exit_bb
= region
->exit
;
6595 gsi
= gsi_last_bb (entry_bb
);
6597 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6598 /* Not needed in SSA form right now. */
6599 gcc_assert (!gimple_in_ssa_p (cfun
));
6600 if (fd
->collapse
> 1)
6602 int first_zero_iter
= -1;
6603 basic_block zero_iter_bb
= l2_bb
;
6605 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6606 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6607 zero_iter_bb
, first_zero_iter
,
6610 if (l2_dom_bb
== NULL
)
6615 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6617 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6618 OMP_CLAUSE__LOOPTEMP_
);
6619 gcc_assert (innerc
);
6620 n1
= OMP_CLAUSE_DECL (innerc
);
6621 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6622 OMP_CLAUSE__LOOPTEMP_
);
6623 gcc_assert (innerc
);
6624 n2
= OMP_CLAUSE_DECL (innerc
);
6625 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6626 fold_convert (type
, n1
));
6627 if (fd
->collapse
> 1)
6630 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6636 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6637 fold_convert (type
, fd
->loop
.n1
));
6638 if (fd
->collapse
> 1)
6639 for (i
= 0; i
< fd
->collapse
; i
++)
6641 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6642 if (POINTER_TYPE_P (itype
))
6643 itype
= signed_type_for (itype
);
6644 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6645 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6649 /* Remove the GIMPLE_OMP_FOR statement. */
6650 gsi_remove (&gsi
, true);
6654 /* Code to control the increment goes in the CONT_BB. */
6655 gsi
= gsi_last_bb (cont_bb
);
6656 stmt
= gsi_stmt (gsi
);
6657 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6659 if (POINTER_TYPE_P (type
))
6660 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6662 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6663 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6665 if (fd
->collapse
> 1)
6667 i
= fd
->collapse
- 1;
6668 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6670 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6671 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6675 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6677 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6680 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6682 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6684 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6685 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6686 if (POINTER_TYPE_P (itype2
))
6687 itype2
= signed_type_for (itype2
);
6688 t
= build3 (COND_EXPR
, itype2
,
6689 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6691 fold_convert (itype
, fd
->loops
[i
].n2
)),
6692 build_int_cst (itype2
, 0),
6693 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6694 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6695 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6697 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6698 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6700 t
= build3 (COND_EXPR
, itype
,
6701 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6703 fold_convert (itype
, fd
->loops
[i
].n2
)),
6705 fold_convert (itype
, fd
->loops
[i
].n1
));
6706 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6710 /* Remove GIMPLE_OMP_CONTINUE. */
6711 gsi_remove (&gsi
, true);
6714 /* Emit the condition in L1_BB. */
6715 gsi
= gsi_start_bb (l1_bb
);
6717 t
= fold_convert (type
, n2
);
6718 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6719 false, GSI_CONTINUE_LINKING
);
6720 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6721 stmt
= gimple_build_cond_empty (t
);
6722 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6723 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6725 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6728 gsi
= gsi_for_stmt (stmt
);
6729 gimple_regimplify_operands (stmt
, &gsi
);
6732 /* Remove GIMPLE_OMP_RETURN. */
6733 gsi
= gsi_last_bb (exit_bb
);
6734 gsi_remove (&gsi
, true);
6736 /* Connect the new blocks. */
6737 remove_edge (FALLTHRU_EDGE (entry_bb
));
6741 remove_edge (BRANCH_EDGE (entry_bb
));
6742 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6744 e
= BRANCH_EDGE (l1_bb
);
6745 ne
= FALLTHRU_EDGE (l1_bb
);
6746 e
->flags
= EDGE_TRUE_VALUE
;
6750 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6752 ne
= single_succ_edge (l1_bb
);
6753 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6756 ne
->flags
= EDGE_FALSE_VALUE
;
6757 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6758 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6760 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6761 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6762 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6766 struct loop
*loop
= alloc_loop ();
6767 loop
->header
= l1_bb
;
6768 loop
->latch
= e
->dest
;
6769 add_loop (loop
, l1_bb
->loop_father
);
6770 if (safelen
== NULL_TREE
)
6771 loop
->safelen
= INT_MAX
;
6774 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6775 if (!tree_fits_uhwi_p (safelen
)
6776 || tree_to_uhwi (safelen
) > INT_MAX
)
6777 loop
->safelen
= INT_MAX
;
6779 loop
->safelen
= tree_to_uhwi (safelen
);
6780 if (loop
->safelen
== 1)
6785 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6786 cfun
->has_simduid_loops
= true;
6788 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6790 if ((flag_tree_loop_vectorize
6791 || (!global_options_set
.x_flag_tree_loop_vectorize
6792 && !global_options_set
.x_flag_tree_vectorize
))
6793 && loop
->safelen
> 1)
6795 loop
->force_vect
= true;
6796 cfun
->has_force_vect_loops
= true;
6802 /* Expand the OpenMP loop defined by REGION. */
6805 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6807 struct omp_for_data fd
;
6808 struct omp_for_data_loop
*loops
;
6811 = (struct omp_for_data_loop
*)
6812 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6813 * sizeof (struct omp_for_data_loop
));
6814 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6815 region
->sched_kind
= fd
.sched_kind
;
6817 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6818 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6819 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6822 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6823 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6824 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6827 /* If there isn't a continue then this is a degerate case where
6828 the introduction of abnormal edges during lowering will prevent
6829 original loops from being detected. Fix that up. */
6830 loops_state_set (LOOPS_NEED_FIXUP
);
6832 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_KIND_SIMD
)
6833 expand_omp_simd (region
, &fd
);
6834 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6835 && !fd
.have_ordered
)
6837 if (fd
.chunk_size
== NULL
)
6838 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6840 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6844 int fn_index
, start_ix
, next_ix
;
6846 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6847 == GF_OMP_FOR_KIND_FOR
);
6848 if (fd
.chunk_size
== NULL
6849 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6850 fd
.chunk_size
= integer_zero_node
;
6851 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6852 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6853 ? 3 : fd
.sched_kind
;
6854 fn_index
+= fd
.have_ordered
* 4;
6855 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6856 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6857 if (fd
.iter_type
== long_long_unsigned_type_node
)
6859 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6860 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6861 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6862 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6864 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6865 (enum built_in_function
) next_ix
, inner_stmt
);
6868 if (gimple_in_ssa_p (cfun
))
6869 update_ssa (TODO_update_ssa_only_virtuals
);
6873 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6875 v = GOMP_sections_start (n);
6892 v = GOMP_sections_next ();
6897 If this is a combined parallel sections, replace the call to
6898 GOMP_sections_start with call to GOMP_sections_next. */
6901 expand_omp_sections (struct omp_region
*region
)
6903 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
6904 vec
<tree
> label_vec
;
6906 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
6907 gimple_stmt_iterator si
, switch_si
;
6908 gimple sections_stmt
, stmt
, cont
;
6911 struct omp_region
*inner
;
6913 bool exit_reachable
= region
->cont
!= NULL
;
6915 gcc_assert (region
->exit
!= NULL
);
6916 entry_bb
= region
->entry
;
6917 l0_bb
= single_succ (entry_bb
);
6918 l1_bb
= region
->cont
;
6919 l2_bb
= region
->exit
;
6920 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
6921 l2
= gimple_block_label (l2_bb
);
6924 /* This can happen if there are reductions. */
6925 len
= EDGE_COUNT (l0_bb
->succs
);
6926 gcc_assert (len
> 0);
6927 e
= EDGE_SUCC (l0_bb
, len
- 1);
6928 si
= gsi_last_bb (e
->dest
);
6931 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6932 l2
= gimple_block_label (e
->dest
);
6934 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
6936 si
= gsi_last_bb (e
->dest
);
6938 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6940 l2
= gimple_block_label (e
->dest
);
6946 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
6948 default_bb
= create_empty_bb (l0_bb
);
6950 /* We will build a switch() with enough cases for all the
6951 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
6952 and a default case to abort if something goes wrong. */
6953 len
= EDGE_COUNT (l0_bb
->succs
);
6955 /* Use vec::quick_push on label_vec throughout, since we know the size
6957 label_vec
.create (len
);
6959 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
6960 GIMPLE_OMP_SECTIONS statement. */
6961 si
= gsi_last_bb (entry_bb
);
6962 sections_stmt
= gsi_stmt (si
);
6963 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
6964 vin
= gimple_omp_sections_control (sections_stmt
);
6965 if (!is_combined_parallel (region
))
6967 /* If we are not inside a combined parallel+sections region,
6968 call GOMP_sections_start. */
6969 t
= build_int_cst (unsigned_type_node
, len
- 1);
6970 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
6971 stmt
= gimple_build_call (u
, 1, t
);
6975 /* Otherwise, call GOMP_sections_next. */
6976 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
6977 stmt
= gimple_build_call (u
, 0);
6979 gimple_call_set_lhs (stmt
, vin
);
6980 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
6981 gsi_remove (&si
, true);
6983 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
6985 switch_si
= gsi_last_bb (l0_bb
);
6986 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
6989 cont
= last_stmt (l1_bb
);
6990 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
6991 vmain
= gimple_omp_continue_control_use (cont
);
6992 vnext
= gimple_omp_continue_control_def (cont
);
7000 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7001 label_vec
.quick_push (t
);
7004 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7005 for (inner
= region
->inner
, casei
= 1;
7007 inner
= inner
->next
, i
++, casei
++)
7009 basic_block s_entry_bb
, s_exit_bb
;
7011 /* Skip optional reduction region. */
7012 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7019 s_entry_bb
= inner
->entry
;
7020 s_exit_bb
= inner
->exit
;
7022 t
= gimple_block_label (s_entry_bb
);
7023 u
= build_int_cst (unsigned_type_node
, casei
);
7024 u
= build_case_label (u
, NULL
, t
);
7025 label_vec
.quick_push (u
);
7027 si
= gsi_last_bb (s_entry_bb
);
7028 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7029 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7030 gsi_remove (&si
, true);
7031 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7033 if (s_exit_bb
== NULL
)
7036 si
= gsi_last_bb (s_exit_bb
);
7037 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7038 gsi_remove (&si
, true);
7040 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7043 /* Error handling code goes in DEFAULT_BB. */
7044 t
= gimple_block_label (default_bb
);
7045 u
= build_case_label (NULL
, NULL
, t
);
7046 make_edge (l0_bb
, default_bb
, 0);
7048 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7050 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7051 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7052 gsi_remove (&switch_si
, true);
7053 label_vec
.release ();
7055 si
= gsi_start_bb (default_bb
);
7056 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7057 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7063 /* Code to get the next section goes in L1_BB. */
7064 si
= gsi_last_bb (l1_bb
);
7065 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7067 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7068 stmt
= gimple_build_call (bfn_decl
, 0);
7069 gimple_call_set_lhs (stmt
, vnext
);
7070 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7071 gsi_remove (&si
, true);
7073 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7076 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7077 si
= gsi_last_bb (l2_bb
);
7078 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7079 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7080 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7081 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7083 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7084 stmt
= gimple_build_call (t
, 0);
7085 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7086 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7087 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7088 gsi_remove (&si
, true);
7090 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7094 /* Expand code for an OpenMP single directive. We've already expanded
7095 much of the code, here we simply place the GOMP_barrier call. */
7098 expand_omp_single (struct omp_region
*region
)
7100 basic_block entry_bb
, exit_bb
;
7101 gimple_stmt_iterator si
;
7103 entry_bb
= region
->entry
;
7104 exit_bb
= region
->exit
;
7106 si
= gsi_last_bb (entry_bb
);
7107 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7108 gsi_remove (&si
, true);
7109 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7111 si
= gsi_last_bb (exit_bb
);
7112 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7114 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7115 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7117 gsi_remove (&si
, true);
7118 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7122 /* Generic expansion for OpenMP synchronization directives: master,
7123 ordered and critical. All we need to do here is remove the entry
7124 and exit markers for REGION. */
7127 expand_omp_synch (struct omp_region
*region
)
7129 basic_block entry_bb
, exit_bb
;
7130 gimple_stmt_iterator si
;
7132 entry_bb
= region
->entry
;
7133 exit_bb
= region
->exit
;
7135 si
= gsi_last_bb (entry_bb
);
7136 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7137 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7138 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7139 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7140 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7141 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7142 gsi_remove (&si
, true);
7143 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7147 si
= gsi_last_bb (exit_bb
);
7148 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7149 gsi_remove (&si
, true);
7150 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7154 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7155 operation as a normal volatile load. */
7158 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7159 tree loaded_val
, int index
)
7161 enum built_in_function tmpbase
;
7162 gimple_stmt_iterator gsi
;
7163 basic_block store_bb
;
7166 tree decl
, call
, type
, itype
;
7168 gsi
= gsi_last_bb (load_bb
);
7169 stmt
= gsi_stmt (gsi
);
7170 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7171 loc
= gimple_location (stmt
);
7173 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7174 is smaller than word size, then expand_atomic_load assumes that the load
7175 is atomic. We could avoid the builtin entirely in this case. */
7177 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7178 decl
= builtin_decl_explicit (tmpbase
);
7179 if (decl
== NULL_TREE
)
7182 type
= TREE_TYPE (loaded_val
);
7183 itype
= TREE_TYPE (TREE_TYPE (decl
));
7185 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7186 build_int_cst (NULL
,
7187 gimple_omp_atomic_seq_cst_p (stmt
)
7189 : MEMMODEL_RELAXED
));
7190 if (!useless_type_conversion_p (type
, itype
))
7191 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7192 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7194 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7195 gsi_remove (&gsi
, true);
7197 store_bb
= single_succ (load_bb
);
7198 gsi
= gsi_last_bb (store_bb
);
7199 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7200 gsi_remove (&gsi
, true);
7202 if (gimple_in_ssa_p (cfun
))
7203 update_ssa (TODO_update_ssa_no_phi
);
7208 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7209 operation as a normal volatile store. */
7212 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7213 tree loaded_val
, tree stored_val
, int index
)
7215 enum built_in_function tmpbase
;
7216 gimple_stmt_iterator gsi
;
7217 basic_block store_bb
= single_succ (load_bb
);
7220 tree decl
, call
, type
, itype
;
7221 enum machine_mode imode
;
7224 gsi
= gsi_last_bb (load_bb
);
7225 stmt
= gsi_stmt (gsi
);
7226 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7228 /* If the load value is needed, then this isn't a store but an exchange. */
7229 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7231 gsi
= gsi_last_bb (store_bb
);
7232 stmt
= gsi_stmt (gsi
);
7233 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7234 loc
= gimple_location (stmt
);
7236 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7237 is smaller than word size, then expand_atomic_store assumes that the store
7238 is atomic. We could avoid the builtin entirely in this case. */
7240 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7241 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7242 decl
= builtin_decl_explicit (tmpbase
);
7243 if (decl
== NULL_TREE
)
7246 type
= TREE_TYPE (stored_val
);
7248 /* Dig out the type of the function's second argument. */
7249 itype
= TREE_TYPE (decl
);
7250 itype
= TYPE_ARG_TYPES (itype
);
7251 itype
= TREE_CHAIN (itype
);
7252 itype
= TREE_VALUE (itype
);
7253 imode
= TYPE_MODE (itype
);
7255 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7258 if (!useless_type_conversion_p (itype
, type
))
7259 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7260 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7261 build_int_cst (NULL
,
7262 gimple_omp_atomic_seq_cst_p (stmt
)
7264 : MEMMODEL_RELAXED
));
7267 if (!useless_type_conversion_p (type
, itype
))
7268 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7269 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7272 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7273 gsi_remove (&gsi
, true);
7275 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7276 gsi
= gsi_last_bb (load_bb
);
7277 gsi_remove (&gsi
, true);
7279 if (gimple_in_ssa_p (cfun
))
7280 update_ssa (TODO_update_ssa_no_phi
);
7285 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7286 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7287 size of the data type, and thus usable to find the index of the builtin
7288 decl. Returns false if the expression is not of the proper form. */
7291 expand_omp_atomic_fetch_op (basic_block load_bb
,
7292 tree addr
, tree loaded_val
,
7293 tree stored_val
, int index
)
7295 enum built_in_function oldbase
, newbase
, tmpbase
;
7296 tree decl
, itype
, call
;
7298 basic_block store_bb
= single_succ (load_bb
);
7299 gimple_stmt_iterator gsi
;
7302 enum tree_code code
;
7303 bool need_old
, need_new
;
7304 enum machine_mode imode
;
7307 /* We expect to find the following sequences:
7310 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7313 val = tmp OP something; (or: something OP tmp)
7314 GIMPLE_OMP_STORE (val)
7316 ???FIXME: Allow a more flexible sequence.
7317 Perhaps use data flow to pick the statements.
7321 gsi
= gsi_after_labels (store_bb
);
7322 stmt
= gsi_stmt (gsi
);
7323 loc
= gimple_location (stmt
);
7324 if (!is_gimple_assign (stmt
))
7327 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7329 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7330 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7331 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7332 gcc_checking_assert (!need_old
|| !need_new
);
7334 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7337 /* Check for one of the supported fetch-op operations. */
7338 code
= gimple_assign_rhs_code (stmt
);
7342 case POINTER_PLUS_EXPR
:
7343 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7344 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7347 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7348 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7351 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7352 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7355 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7356 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7359 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7360 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7366 /* Make sure the expression is of the proper form. */
7367 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7368 rhs
= gimple_assign_rhs2 (stmt
);
7369 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7370 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7371 rhs
= gimple_assign_rhs1 (stmt
);
7375 tmpbase
= ((enum built_in_function
)
7376 ((need_new
? newbase
: oldbase
) + index
+ 1));
7377 decl
= builtin_decl_explicit (tmpbase
);
7378 if (decl
== NULL_TREE
)
7380 itype
= TREE_TYPE (TREE_TYPE (decl
));
7381 imode
= TYPE_MODE (itype
);
7383 /* We could test all of the various optabs involved, but the fact of the
7384 matter is that (with the exception of i486 vs i586 and xadd) all targets
7385 that support any atomic operaton optab also implements compare-and-swap.
7386 Let optabs.c take care of expanding any compare-and-swap loop. */
7387 if (!can_compare_and_swap_p (imode
, true))
7390 gsi
= gsi_last_bb (load_bb
);
7391 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7393 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7394 It only requires that the operation happen atomically. Thus we can
7395 use the RELAXED memory model. */
7396 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7397 fold_convert_loc (loc
, itype
, rhs
),
7398 build_int_cst (NULL
,
7399 seq_cst
? MEMMODEL_SEQ_CST
7400 : MEMMODEL_RELAXED
));
7402 if (need_old
|| need_new
)
7404 lhs
= need_old
? loaded_val
: stored_val
;
7405 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7406 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7409 call
= fold_convert_loc (loc
, void_type_node
, call
);
7410 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7411 gsi_remove (&gsi
, true);
7413 gsi
= gsi_last_bb (store_bb
);
7414 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7415 gsi_remove (&gsi
, true);
7416 gsi
= gsi_last_bb (store_bb
);
7417 gsi_remove (&gsi
, true);
7419 if (gimple_in_ssa_p (cfun
))
7420 update_ssa (TODO_update_ssa_no_phi
);
7425 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7429 newval = rhs; // with oldval replacing *addr in rhs
7430 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7431 if (oldval != newval)
7434 INDEX is log2 of the size of the data type, and thus usable to find the
7435 index of the builtin decl. */
7438 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7439 tree addr
, tree loaded_val
, tree stored_val
,
7442 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7443 tree type
, itype
, cmpxchg
, iaddr
;
7444 gimple_stmt_iterator si
;
7445 basic_block loop_header
= single_succ (load_bb
);
7448 enum built_in_function fncode
;
7450 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7451 order to use the RELAXED memory model effectively. */
7452 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7454 cmpxchg
= builtin_decl_explicit (fncode
);
7455 if (cmpxchg
== NULL_TREE
)
7457 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7458 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7460 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7463 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7464 si
= gsi_last_bb (load_bb
);
7465 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7467 /* For floating-point values, we'll need to view-convert them to integers
7468 so that we can perform the atomic compare and swap. Simplify the
7469 following code by always setting up the "i"ntegral variables. */
7470 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7474 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7477 = force_gimple_operand_gsi (&si
,
7478 fold_convert (TREE_TYPE (iaddr
), addr
),
7479 false, NULL_TREE
, true, GSI_SAME_STMT
);
7480 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7481 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7482 loadedi
= create_tmp_var (itype
, NULL
);
7483 if (gimple_in_ssa_p (cfun
))
7484 loadedi
= make_ssa_name (loadedi
, NULL
);
7489 loadedi
= loaded_val
;
7493 = force_gimple_operand_gsi (&si
,
7494 build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)),
7496 build_int_cst (TREE_TYPE (iaddr
), 0)),
7497 true, NULL_TREE
, true, GSI_SAME_STMT
);
7499 /* Move the value to the LOADEDI temporary. */
7500 if (gimple_in_ssa_p (cfun
))
7502 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7503 phi
= create_phi_node (loadedi
, loop_header
);
7504 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7508 gsi_insert_before (&si
,
7509 gimple_build_assign (loadedi
, initial
),
7511 if (loadedi
!= loaded_val
)
7513 gimple_stmt_iterator gsi2
;
7516 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7517 gsi2
= gsi_start_bb (loop_header
);
7518 if (gimple_in_ssa_p (cfun
))
7521 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7522 true, GSI_SAME_STMT
);
7523 stmt
= gimple_build_assign (loaded_val
, x
);
7524 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7528 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7529 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7530 true, GSI_SAME_STMT
);
7533 gsi_remove (&si
, true);
7535 si
= gsi_last_bb (store_bb
);
7536 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7539 storedi
= stored_val
;
7542 force_gimple_operand_gsi (&si
,
7543 build1 (VIEW_CONVERT_EXPR
, itype
,
7544 stored_val
), true, NULL_TREE
, true,
7547 /* Build the compare&swap statement. */
7548 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7549 new_storedi
= force_gimple_operand_gsi (&si
,
7550 fold_convert (TREE_TYPE (loadedi
),
7553 true, GSI_SAME_STMT
);
7555 if (gimple_in_ssa_p (cfun
))
7559 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7560 stmt
= gimple_build_assign (old_vali
, loadedi
);
7561 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7563 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7564 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7567 /* Note that we always perform the comparison as an integer, even for
7568 floating point. This allows the atomic operation to properly
7569 succeed even with NaNs and -0.0. */
7570 stmt
= gimple_build_cond_empty
7571 (build2 (NE_EXPR
, boolean_type_node
,
7572 new_storedi
, old_vali
));
7573 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7576 e
= single_succ_edge (store_bb
);
7577 e
->flags
&= ~EDGE_FALLTHRU
;
7578 e
->flags
|= EDGE_FALSE_VALUE
;
7580 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7582 /* Copy the new value to loadedi (we already did that before the condition
7583 if we are not in SSA). */
7584 if (gimple_in_ssa_p (cfun
))
7586 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7587 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7590 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7591 gsi_remove (&si
, true);
7593 struct loop
*loop
= alloc_loop ();
7594 loop
->header
= loop_header
;
7595 loop
->latch
= store_bb
;
7596 add_loop (loop
, loop_header
->loop_father
);
7598 if (gimple_in_ssa_p (cfun
))
7599 update_ssa (TODO_update_ssa_no_phi
);
7604 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7606 GOMP_atomic_start ();
7610 The result is not globally atomic, but works so long as all parallel
7611 references are within #pragma omp atomic directives. According to
7612 responses received from omp@openmp.org, appears to be within spec.
7613 Which makes sense, since that's how several other compilers handle
7614 this situation as well.
7615 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7616 expanding. STORED_VAL is the operand of the matching
7617 GIMPLE_OMP_ATOMIC_STORE.
7620 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7624 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7629 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7630 tree addr
, tree loaded_val
, tree stored_val
)
7632 gimple_stmt_iterator si
;
7636 si
= gsi_last_bb (load_bb
);
7637 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7639 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7640 t
= build_call_expr (t
, 0);
7641 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7643 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7644 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7645 gsi_remove (&si
, true);
7647 si
= gsi_last_bb (store_bb
);
7648 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7650 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7652 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7654 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7655 t
= build_call_expr (t
, 0);
7656 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7657 gsi_remove (&si
, true);
7659 if (gimple_in_ssa_p (cfun
))
7660 update_ssa (TODO_update_ssa_no_phi
);
7664 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7665 using expand_omp_atomic_fetch_op. If it failed, we try to
7666 call expand_omp_atomic_pipeline, and if it fails too, the
7667 ultimate fallback is wrapping the operation in a mutex
7668 (expand_omp_atomic_mutex). REGION is the atomic region built
7669 by build_omp_regions_1(). */
7672 expand_omp_atomic (struct omp_region
*region
)
7674 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7675 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7676 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7677 tree addr
= gimple_omp_atomic_load_rhs (load
);
7678 tree stored_val
= gimple_omp_atomic_store_val (store
);
7679 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7680 HOST_WIDE_INT index
;
7682 /* Make sure the type is one of the supported sizes. */
7683 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7684 index
= exact_log2 (index
);
7685 if (index
>= 0 && index
<= 4)
7687 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7689 /* __sync builtins require strict data alignment. */
7690 if (exact_log2 (align
) >= index
)
7693 if (loaded_val
== stored_val
7694 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7695 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7696 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7697 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7701 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7702 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7703 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7704 && store_bb
== single_succ (load_bb
)
7705 && first_stmt (store_bb
) == store
7706 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7710 /* When possible, use specialized atomic update functions. */
7711 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7712 && store_bb
== single_succ (load_bb
)
7713 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7714 loaded_val
, stored_val
, index
))
7717 /* If we don't have specialized __sync builtins, try and implement
7718 as a compare and swap loop. */
7719 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7720 loaded_val
, stored_val
, index
))
7725 /* The ultimate fallback is wrapping the operation in a mutex. */
7726 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7730 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7733 expand_omp_target (struct omp_region
*region
)
7735 basic_block entry_bb
, exit_bb
, new_bb
;
7736 struct function
*child_cfun
= NULL
;
7737 tree child_fn
= NULL_TREE
, block
, t
;
7738 gimple_stmt_iterator gsi
;
7739 gimple entry_stmt
, stmt
;
7742 entry_stmt
= last_stmt (region
->entry
);
7743 new_bb
= region
->entry
;
7744 int kind
= gimple_omp_target_kind (entry_stmt
);
7745 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7747 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7748 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7751 entry_bb
= region
->entry
;
7752 exit_bb
= region
->exit
;
7754 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7756 unsigned srcidx
, dstidx
, num
;
7758 /* If the target region needs data sent from the parent
7759 function, then the very first statement (except possible
7760 tree profile counter updates) of the parallel body
7761 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7762 &.OMP_DATA_O is passed as an argument to the child function,
7763 we need to replace it with the argument as seen by the child
7766 In most cases, this will end up being the identity assignment
7767 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7768 a function call that has been inlined, the original PARM_DECL
7769 .OMP_DATA_I may have been converted into a different local
7770 variable. In which case, we need to keep the assignment. */
7771 if (gimple_omp_target_data_arg (entry_stmt
))
7773 basic_block entry_succ_bb
= single_succ (entry_bb
);
7774 gimple_stmt_iterator gsi
;
7776 gimple tgtcopy_stmt
= NULL
;
7778 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7780 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7782 gcc_assert (!gsi_end_p (gsi
));
7783 stmt
= gsi_stmt (gsi
);
7784 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7787 if (gimple_num_ops (stmt
) == 2)
7789 tree arg
= gimple_assign_rhs1 (stmt
);
7791 /* We're ignoring the subcode because we're
7792 effectively doing a STRIP_NOPS. */
7794 if (TREE_CODE (arg
) == ADDR_EXPR
7795 && TREE_OPERAND (arg
, 0) == sender
)
7797 tgtcopy_stmt
= stmt
;
7803 gcc_assert (tgtcopy_stmt
!= NULL
);
7804 arg
= DECL_ARGUMENTS (child_fn
);
7806 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7807 gsi_remove (&gsi
, true);
7810 /* Declare local variables needed in CHILD_CFUN. */
7811 block
= DECL_INITIAL (child_fn
);
7812 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7813 /* The gimplifier could record temporaries in target block
7814 rather than in containing function's local_decls chain,
7815 which would mean cgraph missed finalizing them. Do it now. */
7816 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7817 if (TREE_CODE (t
) == VAR_DECL
7819 && !DECL_EXTERNAL (t
))
7820 varpool_finalize_decl (t
);
7821 DECL_SAVED_TREE (child_fn
) = NULL
;
7822 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7823 gimple_set_body (child_fn
, NULL
);
7824 TREE_USED (block
) = 1;
7826 /* Reset DECL_CONTEXT on function arguments. */
7827 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7828 DECL_CONTEXT (t
) = child_fn
;
7830 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7831 so that it can be moved to the child function. */
7832 gsi
= gsi_last_bb (entry_bb
);
7833 stmt
= gsi_stmt (gsi
);
7834 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7835 && gimple_omp_target_kind (stmt
)
7836 == GF_OMP_TARGET_KIND_REGION
);
7837 gsi_remove (&gsi
, true);
7838 e
= split_block (entry_bb
, stmt
);
7840 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7842 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7845 gsi
= gsi_last_bb (exit_bb
);
7846 gcc_assert (!gsi_end_p (gsi
)
7847 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7848 stmt
= gimple_build_return (NULL
);
7849 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7850 gsi_remove (&gsi
, true);
7853 /* Move the target region into CHILD_CFUN. */
7855 block
= gimple_block (entry_stmt
);
7857 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7859 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7860 /* When the OMP expansion process cannot guarantee an up-to-date
7861 loop tree arrange for the child function to fixup loops. */
7862 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7863 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7865 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7866 num
= vec_safe_length (child_cfun
->local_decls
);
7867 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7869 t
= (*child_cfun
->local_decls
)[srcidx
];
7870 if (DECL_CONTEXT (t
) == cfun
->decl
)
7872 if (srcidx
!= dstidx
)
7873 (*child_cfun
->local_decls
)[dstidx
] = t
;
7877 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7879 /* Inform the callgraph about the new function. */
7880 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
7881 cgraph_add_new_function (child_fn
, true);
7883 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7884 fixed in a following pass. */
7885 push_cfun (child_cfun
);
7886 rebuild_cgraph_edges ();
7888 /* Some EH regions might become dead, see PR34608. If
7889 pass_cleanup_cfg isn't the first pass to happen with the
7890 new child, these dead EH edges might cause problems.
7891 Clean them up now. */
7892 if (flag_exceptions
)
7895 bool changed
= false;
7898 changed
|= gimple_purge_dead_eh_edges (bb
);
7900 cleanup_tree_cfg ();
7905 /* Emit a library call to launch the target region, or do data
7907 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
7908 enum built_in_function start_ix
;
7909 location_t clause_loc
;
7911 clauses
= gimple_omp_target_clauses (entry_stmt
);
7913 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7914 start_ix
= BUILT_IN_GOMP_TARGET
;
7915 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
7916 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
7918 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
7920 /* By default, the value of DEVICE is -1 (let runtime library choose)
7921 and there is no conditional. */
7923 device
= build_int_cst (integer_type_node
, -1);
7925 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
7927 cond
= OMP_CLAUSE_IF_EXPR (c
);
7929 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
7932 device
= OMP_CLAUSE_DEVICE_ID (c
);
7933 clause_loc
= OMP_CLAUSE_LOCATION (c
);
7936 clause_loc
= gimple_location (entry_stmt
);
7938 /* Ensure 'device' is of the correct type. */
7939 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
7941 /* If we found the clause 'if (cond)', build
7942 (cond ? device : -2). */
7945 cond
= gimple_boolify (cond
);
7947 basic_block cond_bb
, then_bb
, else_bb
;
7951 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
7952 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
7954 gsi
= gsi_last_bb (new_bb
);
7956 e
= split_block (new_bb
, gsi_stmt (gsi
));
7959 e
= split_block (new_bb
, NULL
);
7964 then_bb
= create_empty_bb (cond_bb
);
7965 else_bb
= create_empty_bb (then_bb
);
7966 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
7967 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
7969 stmt
= gimple_build_cond_empty (cond
);
7970 gsi
= gsi_last_bb (cond_bb
);
7971 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7973 gsi
= gsi_start_bb (then_bb
);
7974 stmt
= gimple_build_assign (tmp_var
, device
);
7975 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7977 gsi
= gsi_start_bb (else_bb
);
7978 stmt
= gimple_build_assign (tmp_var
,
7979 build_int_cst (integer_type_node
, -2));
7980 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7982 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
7983 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
7986 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
7987 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
7989 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
7990 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
7995 gsi
= gsi_last_bb (new_bb
);
7996 t
= gimple_omp_target_data_arg (entry_stmt
);
7999 t1
= size_zero_node
;
8000 t2
= build_zero_cst (ptr_type_node
);
8006 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8007 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8008 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8009 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8010 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8014 /* FIXME: This will be address of
8015 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8016 symbol, as soon as the linker plugin is able to create it for us. */
8017 tree openmp_target
= build_zero_cst (ptr_type_node
);
8018 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8020 tree fnaddr
= build_fold_addr_expr (child_fn
);
8021 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8022 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8025 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8026 device
, openmp_target
, t1
, t2
, t3
, t4
);
8027 gimple_set_location (g
, gimple_location (entry_stmt
));
8028 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8029 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8032 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8033 gsi_remove (&gsi
, true);
8035 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8037 gsi
= gsi_last_bb (region
->exit
);
8039 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8040 gsi_remove (&gsi
, true);
8045 /* Expand the parallel region tree rooted at REGION. Expansion
8046 proceeds in depth-first order. Innermost regions are expanded
8047 first. This way, parallel regions that require a new function to
8048 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8049 internal dependencies in their body. */
8052 expand_omp (struct omp_region
*region
)
8056 location_t saved_location
;
8057 gimple inner_stmt
= NULL
;
8059 /* First, determine whether this is a combined parallel+workshare
8061 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8062 determine_parallel_type (region
);
8064 if (region
->type
== GIMPLE_OMP_FOR
8065 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8066 inner_stmt
= last_stmt (region
->inner
->entry
);
8069 expand_omp (region
->inner
);
8071 saved_location
= input_location
;
8072 if (gimple_has_location (last_stmt (region
->entry
)))
8073 input_location
= gimple_location (last_stmt (region
->entry
));
8075 switch (region
->type
)
8077 case GIMPLE_OMP_PARALLEL
:
8078 case GIMPLE_OMP_TASK
:
8079 expand_omp_taskreg (region
);
8082 case GIMPLE_OMP_FOR
:
8083 expand_omp_for (region
, inner_stmt
);
8086 case GIMPLE_OMP_SECTIONS
:
8087 expand_omp_sections (region
);
8090 case GIMPLE_OMP_SECTION
:
8091 /* Individual omp sections are handled together with their
8092 parent GIMPLE_OMP_SECTIONS region. */
8095 case GIMPLE_OMP_SINGLE
:
8096 expand_omp_single (region
);
8099 case GIMPLE_OMP_MASTER
:
8100 case GIMPLE_OMP_TASKGROUP
:
8101 case GIMPLE_OMP_ORDERED
:
8102 case GIMPLE_OMP_CRITICAL
:
8103 case GIMPLE_OMP_TEAMS
:
8104 expand_omp_synch (region
);
8107 case GIMPLE_OMP_ATOMIC_LOAD
:
8108 expand_omp_atomic (region
);
8111 case GIMPLE_OMP_TARGET
:
8112 expand_omp_target (region
);
8119 input_location
= saved_location
;
8120 region
= region
->next
;
8125 /* Helper for build_omp_regions. Scan the dominator tree starting at
8126 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8127 true, the function ends once a single tree is built (otherwise, whole
8128 forest of OMP constructs may be built). */
8131 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8134 gimple_stmt_iterator gsi
;
8138 gsi
= gsi_last_bb (bb
);
8139 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8141 struct omp_region
*region
;
8142 enum gimple_code code
;
8144 stmt
= gsi_stmt (gsi
);
8145 code
= gimple_code (stmt
);
8146 if (code
== GIMPLE_OMP_RETURN
)
8148 /* STMT is the return point out of region PARENT. Mark it
8149 as the exit point and make PARENT the immediately
8150 enclosing region. */
8151 gcc_assert (parent
);
8154 parent
= parent
->outer
;
8156 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8158 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8159 GIMPLE_OMP_RETURN, but matches with
8160 GIMPLE_OMP_ATOMIC_LOAD. */
8161 gcc_assert (parent
);
8162 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8165 parent
= parent
->outer
;
8168 else if (code
== GIMPLE_OMP_CONTINUE
)
8170 gcc_assert (parent
);
8173 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8175 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8176 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8179 else if (code
== GIMPLE_OMP_TARGET
8180 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8181 new_omp_region (bb
, code
, parent
);
8184 /* Otherwise, this directive becomes the parent for a new
8186 region
= new_omp_region (bb
, code
, parent
);
8191 if (single_tree
&& !parent
)
8194 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8196 son
= next_dom_son (CDI_DOMINATORS
, son
))
8197 build_omp_regions_1 (son
, parent
, single_tree
);
8200 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8204 build_omp_regions_root (basic_block root
)
8206 gcc_assert (root_omp_region
== NULL
);
8207 build_omp_regions_1 (root
, NULL
, true);
8208 gcc_assert (root_omp_region
!= NULL
);
8211 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8214 omp_expand_local (basic_block head
)
8216 build_omp_regions_root (head
);
8217 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8219 fprintf (dump_file
, "\nOMP region tree\n\n");
8220 dump_omp_region (dump_file
, root_omp_region
, 0);
8221 fprintf (dump_file
, "\n");
8224 remove_exit_barriers (root_omp_region
);
8225 expand_omp (root_omp_region
);
8227 free_omp_regions ();
8230 /* Scan the CFG and build a tree of OMP regions. Return the root of
8231 the OMP region tree. */
8234 build_omp_regions (void)
8236 gcc_assert (root_omp_region
== NULL
);
8237 calculate_dominance_info (CDI_DOMINATORS
);
8238 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8241 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8244 execute_expand_omp (void)
8246 build_omp_regions ();
8248 if (!root_omp_region
)
8253 fprintf (dump_file
, "\nOMP region tree\n\n");
8254 dump_omp_region (dump_file
, root_omp_region
, 0);
8255 fprintf (dump_file
, "\n");
8258 remove_exit_barriers (root_omp_region
);
8260 expand_omp (root_omp_region
);
8262 cleanup_tree_cfg ();
8264 free_omp_regions ();
8269 /* OMP expansion -- the default pass, run before creation of SSA form. */
8272 gate_expand_omp (void)
8274 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8275 || flag_enable_cilkplus
!= 0) && !seen_error ());
8280 const pass_data pass_data_expand_omp
=
8282 GIMPLE_PASS
, /* type */
8283 "ompexp", /* name */
8284 OPTGROUP_NONE
, /* optinfo_flags */
8285 true, /* has_gate */
8286 true, /* has_execute */
8287 TV_NONE
, /* tv_id */
8288 PROP_gimple_any
, /* properties_required */
8289 0, /* properties_provided */
8290 0, /* properties_destroyed */
8291 0, /* todo_flags_start */
8292 0, /* todo_flags_finish */
8295 class pass_expand_omp
: public gimple_opt_pass
8298 pass_expand_omp (gcc::context
*ctxt
)
8299 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8302 /* opt_pass methods: */
8303 bool gate () { return gate_expand_omp (); }
8304 unsigned int execute () { return execute_expand_omp (); }
8306 }; // class pass_expand_omp
8311 make_pass_expand_omp (gcc::context
*ctxt
)
8313 return new pass_expand_omp (ctxt
);
8316 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8318 /* If ctx is a worksharing context inside of a cancellable parallel
8319 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8320 and conditional branch to parallel's cancel_label to handle
8321 cancellation in the implicit barrier. */
8324 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8326 gimple omp_return
= gimple_seq_last_stmt (*body
);
8327 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8328 if (gimple_omp_return_nowait_p (omp_return
))
8331 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8332 && ctx
->outer
->cancellable
)
8334 tree lhs
= create_tmp_var (boolean_type_node
, NULL
);
8335 gimple_omp_return_set_lhs (omp_return
, lhs
);
8336 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8337 gimple g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
8338 ctx
->outer
->cancel_label
, fallthru_label
);
8339 gimple_seq_add_stmt (body
, g
);
8340 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8344 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8345 CTX is the enclosing OMP context for the current statement. */
8348 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8350 tree block
, control
;
8351 gimple_stmt_iterator tgsi
;
8352 gimple stmt
, new_stmt
, bind
, t
;
8353 gimple_seq ilist
, dlist
, olist
, new_body
;
8354 struct gimplify_ctx gctx
;
8356 stmt
= gsi_stmt (*gsi_p
);
8358 push_gimplify_context (&gctx
);
8362 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8363 &ilist
, &dlist
, ctx
, NULL
);
8365 new_body
= gimple_omp_body (stmt
);
8366 gimple_omp_set_body (stmt
, NULL
);
8367 tgsi
= gsi_start (new_body
);
8368 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8373 sec_start
= gsi_stmt (tgsi
);
8374 sctx
= maybe_lookup_ctx (sec_start
);
8377 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8378 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8379 GSI_CONTINUE_LINKING
);
8380 gimple_omp_set_body (sec_start
, NULL
);
8382 if (gsi_one_before_end_p (tgsi
))
8384 gimple_seq l
= NULL
;
8385 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8387 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8388 gimple_omp_section_set_last (sec_start
);
8391 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8392 GSI_CONTINUE_LINKING
);
8395 block
= make_node (BLOCK
);
8396 bind
= gimple_build_bind (NULL
, new_body
, block
);
8399 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8401 block
= make_node (BLOCK
);
8402 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8403 gsi_replace (gsi_p
, new_stmt
, true);
8405 pop_gimplify_context (new_stmt
);
8406 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8407 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8408 if (BLOCK_VARS (block
))
8409 TREE_USED (block
) = 1;
8412 gimple_seq_add_seq (&new_body
, ilist
);
8413 gimple_seq_add_stmt (&new_body
, stmt
);
8414 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8415 gimple_seq_add_stmt (&new_body
, bind
);
8417 control
= create_tmp_var (unsigned_type_node
, ".section");
8418 t
= gimple_build_omp_continue (control
, control
);
8419 gimple_omp_sections_set_control (stmt
, control
);
8420 gimple_seq_add_stmt (&new_body
, t
);
8422 gimple_seq_add_seq (&new_body
, olist
);
8423 if (ctx
->cancellable
)
8424 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8425 gimple_seq_add_seq (&new_body
, dlist
);
8427 new_body
= maybe_catch_exception (new_body
);
8429 t
= gimple_build_omp_return
8430 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8431 OMP_CLAUSE_NOWAIT
));
8432 gimple_seq_add_stmt (&new_body
, t
);
8433 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8435 gimple_bind_set_body (new_stmt
, new_body
);
8439 /* A subroutine of lower_omp_single. Expand the simple form of
8440 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8442 if (GOMP_single_start ())
8444 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8446 FIXME. It may be better to delay expanding the logic of this until
8447 pass_expand_omp. The expanded logic may make the job more difficult
8448 to a synchronization analysis pass. */
8451 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8453 location_t loc
= gimple_location (single_stmt
);
8454 tree tlabel
= create_artificial_label (loc
);
8455 tree flabel
= create_artificial_label (loc
);
8459 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8460 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8461 call
= gimple_build_call (decl
, 0);
8462 gimple_call_set_lhs (call
, lhs
);
8463 gimple_seq_add_stmt (pre_p
, call
);
8465 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8466 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8469 gimple_seq_add_stmt (pre_p
, cond
);
8470 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8471 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8472 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8476 /* A subroutine of lower_omp_single. Expand the simple form of
8477 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8479 #pragma omp single copyprivate (a, b, c)
8481 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8484 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8490 GOMP_single_copy_end (©out);
8501 FIXME. It may be better to delay expanding the logic of this until
8502 pass_expand_omp. The expanded logic may make the job more difficult
8503 to a synchronization analysis pass. */
8506 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8508 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8509 gimple_seq copyin_seq
;
8510 location_t loc
= gimple_location (single_stmt
);
8512 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8514 ptr_type
= build_pointer_type (ctx
->record_type
);
8515 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8517 l0
= create_artificial_label (loc
);
8518 l1
= create_artificial_label (loc
);
8519 l2
= create_artificial_label (loc
);
8521 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8522 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8523 t
= fold_convert_loc (loc
, ptr_type
, t
);
8524 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8526 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8527 build_int_cst (ptr_type
, 0));
8528 t
= build3 (COND_EXPR
, void_type_node
, t
,
8529 build_and_jump (&l0
), build_and_jump (&l1
));
8530 gimplify_and_add (t
, pre_p
);
8532 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8534 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8537 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8540 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8541 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8542 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8543 gimplify_and_add (t
, pre_p
);
8545 t
= build_and_jump (&l2
);
8546 gimplify_and_add (t
, pre_p
);
8548 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8550 gimple_seq_add_seq (pre_p
, copyin_seq
);
8552 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8556 /* Expand code for an OpenMP single directive. */
8559 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8562 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8563 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8564 struct gimplify_ctx gctx
;
8566 push_gimplify_context (&gctx
);
8568 block
= make_node (BLOCK
);
8569 bind
= gimple_build_bind (NULL
, NULL
, block
);
8570 gsi_replace (gsi_p
, bind
, true);
8573 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8574 &bind_body
, &dlist
, ctx
, NULL
);
8575 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8577 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8579 if (ctx
->record_type
)
8580 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8582 lower_omp_single_simple (single_stmt
, &bind_body
);
8584 gimple_omp_set_body (single_stmt
, NULL
);
8586 gimple_seq_add_seq (&bind_body
, dlist
);
8588 bind_body
= maybe_catch_exception (bind_body
);
8590 t
= gimple_build_omp_return
8591 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8592 OMP_CLAUSE_NOWAIT
));
8593 gimple_seq_add_stmt (&bind_body_tail
, t
);
8594 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8595 if (ctx
->record_type
)
8597 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8598 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8599 TREE_THIS_VOLATILE (clobber
) = 1;
8600 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8601 clobber
), GSI_SAME_STMT
);
8603 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8604 gimple_bind_set_body (bind
, bind_body
);
8606 pop_gimplify_context (bind
);
8608 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8609 BLOCK_VARS (block
) = ctx
->block_vars
;
8610 if (BLOCK_VARS (block
))
8611 TREE_USED (block
) = 1;
8615 /* Expand code for an OpenMP master directive. */
8618 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8620 tree block
, lab
= NULL
, x
, bfn_decl
;
8621 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8622 location_t loc
= gimple_location (stmt
);
8624 struct gimplify_ctx gctx
;
8626 push_gimplify_context (&gctx
);
8628 block
= make_node (BLOCK
);
8629 bind
= gimple_build_bind (NULL
, NULL
, block
);
8630 gsi_replace (gsi_p
, bind
, true);
8631 gimple_bind_add_stmt (bind
, stmt
);
8633 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8634 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8635 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8636 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8638 gimplify_and_add (x
, &tseq
);
8639 gimple_bind_add_seq (bind
, tseq
);
8641 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8642 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8643 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8644 gimple_omp_set_body (stmt
, NULL
);
8646 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8648 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8650 pop_gimplify_context (bind
);
8652 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8653 BLOCK_VARS (block
) = ctx
->block_vars
;
8657 /* Expand code for an OpenMP taskgroup directive. */
8660 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8662 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8663 tree block
= make_node (BLOCK
);
8665 bind
= gimple_build_bind (NULL
, NULL
, block
);
8666 gsi_replace (gsi_p
, bind
, true);
8667 gimple_bind_add_stmt (bind
, stmt
);
8669 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8671 gimple_bind_add_stmt (bind
, x
);
8673 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8674 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8675 gimple_omp_set_body (stmt
, NULL
);
8677 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8679 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8680 BLOCK_VARS (block
) = ctx
->block_vars
;
8684 /* Expand code for an OpenMP ordered directive. */
8687 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8690 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8691 struct gimplify_ctx gctx
;
8693 push_gimplify_context (&gctx
);
8695 block
= make_node (BLOCK
);
8696 bind
= gimple_build_bind (NULL
, NULL
, block
);
8697 gsi_replace (gsi_p
, bind
, true);
8698 gimple_bind_add_stmt (bind
, stmt
);
8700 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8702 gimple_bind_add_stmt (bind
, x
);
8704 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8705 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8706 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8707 gimple_omp_set_body (stmt
, NULL
);
8709 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8710 gimple_bind_add_stmt (bind
, x
);
8712 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8714 pop_gimplify_context (bind
);
8716 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8717 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8721 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8722 substitution of a couple of function calls. But in the NAMED case,
8723 requires that languages coordinate a symbol name. It is therefore
8724 best put here in common code. */
8726 static GTY((param1_is (tree
), param2_is (tree
)))
8727 splay_tree critical_name_mutexes
;
8730 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8733 tree name
, lock
, unlock
;
8734 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8735 location_t loc
= gimple_location (stmt
);
8737 struct gimplify_ctx gctx
;
8739 name
= gimple_omp_critical_name (stmt
);
8745 if (!critical_name_mutexes
)
8746 critical_name_mutexes
8747 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8748 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8749 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8751 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8756 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8758 new_str
= ACONCAT ((".gomp_critical_user_",
8759 IDENTIFIER_POINTER (name
), NULL
));
8760 DECL_NAME (decl
) = get_identifier (new_str
);
8761 TREE_PUBLIC (decl
) = 1;
8762 TREE_STATIC (decl
) = 1;
8763 DECL_COMMON (decl
) = 1;
8764 DECL_ARTIFICIAL (decl
) = 1;
8765 DECL_IGNORED_P (decl
) = 1;
8766 varpool_finalize_decl (decl
);
8768 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8769 (splay_tree_value
) decl
);
8772 decl
= (tree
) n
->value
;
8774 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8775 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8777 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8778 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8779 build_fold_addr_expr_loc (loc
, decl
));
8783 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8784 lock
= build_call_expr_loc (loc
, lock
, 0);
8786 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8787 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8790 push_gimplify_context (&gctx
);
8792 block
= make_node (BLOCK
);
8793 bind
= gimple_build_bind (NULL
, NULL
, block
);
8794 gsi_replace (gsi_p
, bind
, true);
8795 gimple_bind_add_stmt (bind
, stmt
);
8797 tbody
= gimple_bind_body (bind
);
8798 gimplify_and_add (lock
, &tbody
);
8799 gimple_bind_set_body (bind
, tbody
);
8801 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8802 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8803 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8804 gimple_omp_set_body (stmt
, NULL
);
8806 tbody
= gimple_bind_body (bind
);
8807 gimplify_and_add (unlock
, &tbody
);
8808 gimple_bind_set_body (bind
, tbody
);
8810 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8812 pop_gimplify_context (bind
);
8813 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8814 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8818 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8819 for a lastprivate clause. Given a loop control predicate of (V
8820 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8821 is appended to *DLIST, iterator initialization is appended to
8825 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8826 gimple_seq
*dlist
, struct omp_context
*ctx
)
8828 tree clauses
, cond
, vinit
;
8829 enum tree_code cond_code
;
8832 cond_code
= fd
->loop
.cond_code
;
8833 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8835 /* When possible, use a strict equality expression. This can let VRP
8836 type optimizations deduce the value and remove a copy. */
8837 if (tree_fits_shwi_p (fd
->loop
.step
))
8839 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
8840 if (step
== 1 || step
== -1)
8841 cond_code
= EQ_EXPR
;
8844 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8846 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8848 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8849 if (!gimple_seq_empty_p (stmts
))
8851 gimple_seq_add_seq (&stmts
, *dlist
);
8854 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8855 vinit
= fd
->loop
.n1
;
8856 if (cond_code
== EQ_EXPR
8857 && tree_fits_shwi_p (fd
->loop
.n2
)
8858 && ! integer_zerop (fd
->loop
.n2
))
8859 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8861 vinit
= unshare_expr (vinit
);
8863 /* Initialize the iterator variable, so that threads that don't execute
8864 any iterations don't execute the lastprivate clauses by accident. */
8865 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
8870 /* Lower code for an OpenMP loop directive. */
8873 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8876 struct omp_for_data fd
, *fdp
= NULL
;
8877 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
8878 gimple_seq omp_for_body
, body
, dlist
;
8880 struct gimplify_ctx gctx
;
8882 push_gimplify_context (&gctx
);
8884 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
8886 block
= make_node (BLOCK
);
8887 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8888 /* Replace at gsi right away, so that 'stmt' is no member
8889 of a sequence anymore as we're going to add to to a different
8891 gsi_replace (gsi_p
, new_stmt
, true);
8893 /* Move declaration of temporaries in the loop body before we make
8895 omp_for_body
= gimple_omp_body (stmt
);
8896 if (!gimple_seq_empty_p (omp_for_body
)
8897 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
8899 tree vars
= gimple_bind_vars (gimple_seq_first_stmt (omp_for_body
));
8900 gimple_bind_append_vars (new_stmt
, vars
);
8903 if (gimple_omp_for_combined_into_p (stmt
))
8905 extract_omp_for_data (stmt
, &fd
, NULL
);
8908 /* We need two temporaries with fd.loop.v type (istart/iend)
8909 and then (fd.collapse - 1) temporaries with the same
8910 type for count2 ... countN-1 vars if not constant. */
8912 tree type
= fd
.iter_type
;
8914 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
8915 count
+= fd
.collapse
- 1;
8916 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
8917 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
8921 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
8922 OMP_CLAUSE__LOOPTEMP_
);
8923 for (i
= 0; i
< count
; i
++)
8928 gcc_assert (outerc
);
8929 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
8930 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
8931 OMP_CLAUSE__LOOPTEMP_
);
8934 temp
= create_tmp_var (type
, NULL
);
8935 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
8936 OMP_CLAUSE_DECL (*pc
) = temp
;
8937 pc
= &OMP_CLAUSE_CHAIN (*pc
);
8942 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
8945 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
8947 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
8949 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8951 /* Lower the header expressions. At this point, we can assume that
8952 the header is of the form:
8954 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
8956 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
8957 using the .omp_data_s mapping, if needed. */
8958 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
8960 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
8961 if (!is_gimple_min_invariant (*rhs_p
))
8962 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8964 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
8965 if (!is_gimple_min_invariant (*rhs_p
))
8966 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8968 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
8969 if (!is_gimple_min_invariant (*rhs_p
))
8970 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8973 /* Once lowered, extract the bounds and clauses. */
8974 extract_omp_for_data (stmt
, &fd
, NULL
);
8976 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
8978 gimple_seq_add_stmt (&body
, stmt
);
8979 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
8981 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
8984 /* After the loop, add exit clauses. */
8985 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
8987 if (ctx
->cancellable
)
8988 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
8990 gimple_seq_add_seq (&body
, dlist
);
8992 body
= maybe_catch_exception (body
);
8994 /* Region exit marker goes at the end of the loop body. */
8995 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
8996 maybe_add_implicit_barrier_cancel (ctx
, &body
);
8997 pop_gimplify_context (new_stmt
);
8999 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9000 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9001 if (BLOCK_VARS (block
))
9002 TREE_USED (block
) = 1;
9004 gimple_bind_set_body (new_stmt
, body
);
9005 gimple_omp_set_body (stmt
, NULL
);
9006 gimple_omp_for_set_pre_body (stmt
, NULL
);
9009 /* Callback for walk_stmts. Check if the current statement only contains
9010 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
9013 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9014 bool *handled_ops_p
,
9015 struct walk_stmt_info
*wi
)
9017 int *info
= (int *) wi
->info
;
9018 gimple stmt
= gsi_stmt (*gsi_p
);
9020 *handled_ops_p
= true;
9021 switch (gimple_code (stmt
))
9025 case GIMPLE_OMP_FOR
:
9026 case GIMPLE_OMP_SECTIONS
:
9027 *info
= *info
== 0 ? 1 : -1;
9036 struct omp_taskcopy_context
9038 /* This field must be at the beginning, as we do "inheritance": Some
9039 callback functions for tree-inline.c (e.g., omp_copy_decl)
9040 receive a copy_body_data pointer that is up-casted to an
9041 omp_context pointer. */
9047 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9049 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9051 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9052 return create_tmp_var (TREE_TYPE (var
), NULL
);
9058 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9060 tree name
, new_fields
= NULL
, type
, f
;
9062 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9063 name
= DECL_NAME (TYPE_NAME (orig_type
));
9064 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9065 TYPE_DECL
, name
, type
);
9066 TYPE_NAME (type
) = name
;
9068 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9070 tree new_f
= copy_node (f
);
9071 DECL_CONTEXT (new_f
) = type
;
9072 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9073 TREE_CHAIN (new_f
) = new_fields
;
9074 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9075 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9076 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9079 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9081 TYPE_FIELDS (type
) = nreverse (new_fields
);
9086 /* Create task copyfn. */
9089 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9091 struct function
*child_cfun
;
9092 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9093 tree record_type
, srecord_type
, bind
, list
;
9094 bool record_needs_remap
= false, srecord_needs_remap
= false;
9096 struct omp_taskcopy_context tcctx
;
9097 struct gimplify_ctx gctx
;
9098 location_t loc
= gimple_location (task_stmt
);
9100 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9101 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9102 gcc_assert (child_cfun
->cfg
== NULL
);
9103 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9105 /* Reset DECL_CONTEXT on function arguments. */
9106 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9107 DECL_CONTEXT (t
) = child_fn
;
9109 /* Populate the function. */
9110 push_gimplify_context (&gctx
);
9111 push_cfun (child_cfun
);
9113 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9114 TREE_SIDE_EFFECTS (bind
) = 1;
9116 DECL_SAVED_TREE (child_fn
) = bind
;
9117 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9119 /* Remap src and dst argument types if needed. */
9120 record_type
= ctx
->record_type
;
9121 srecord_type
= ctx
->srecord_type
;
9122 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9123 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9125 record_needs_remap
= true;
9128 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9129 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9131 srecord_needs_remap
= true;
9135 if (record_needs_remap
|| srecord_needs_remap
)
9137 memset (&tcctx
, '\0', sizeof (tcctx
));
9138 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9139 tcctx
.cb
.dst_fn
= child_fn
;
9140 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9141 gcc_checking_assert (tcctx
.cb
.src_node
);
9142 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9143 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9144 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9145 tcctx
.cb
.eh_lp_nr
= 0;
9146 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9147 tcctx
.cb
.decl_map
= pointer_map_create ();
9150 if (record_needs_remap
)
9151 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9152 if (srecord_needs_remap
)
9153 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9156 tcctx
.cb
.decl_map
= NULL
;
9158 arg
= DECL_ARGUMENTS (child_fn
);
9159 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9160 sarg
= DECL_CHAIN (arg
);
9161 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9163 /* First pass: initialize temporaries used in record_type and srecord_type
9164 sizes and field offsets. */
9165 if (tcctx
.cb
.decl_map
)
9166 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9167 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9171 decl
= OMP_CLAUSE_DECL (c
);
9172 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9175 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9176 sf
= (tree
) n
->value
;
9177 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9178 src
= build_simple_mem_ref_loc (loc
, sarg
);
9179 src
= omp_build_component_ref (src
, sf
);
9180 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9181 append_to_statement_list (t
, &list
);
9184 /* Second pass: copy shared var pointers and copy construct non-VLA
9185 firstprivate vars. */
9186 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9187 switch (OMP_CLAUSE_CODE (c
))
9189 case OMP_CLAUSE_SHARED
:
9190 decl
= OMP_CLAUSE_DECL (c
);
9191 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9194 f
= (tree
) n
->value
;
9195 if (tcctx
.cb
.decl_map
)
9196 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9197 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9198 sf
= (tree
) n
->value
;
9199 if (tcctx
.cb
.decl_map
)
9200 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9201 src
= build_simple_mem_ref_loc (loc
, sarg
);
9202 src
= omp_build_component_ref (src
, sf
);
9203 dst
= build_simple_mem_ref_loc (loc
, arg
);
9204 dst
= omp_build_component_ref (dst
, f
);
9205 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9206 append_to_statement_list (t
, &list
);
9208 case OMP_CLAUSE_FIRSTPRIVATE
:
9209 decl
= OMP_CLAUSE_DECL (c
);
9210 if (is_variable_sized (decl
))
9212 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9215 f
= (tree
) n
->value
;
9216 if (tcctx
.cb
.decl_map
)
9217 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9218 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9221 sf
= (tree
) n
->value
;
9222 if (tcctx
.cb
.decl_map
)
9223 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9224 src
= build_simple_mem_ref_loc (loc
, sarg
);
9225 src
= omp_build_component_ref (src
, sf
);
9226 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9227 src
= build_simple_mem_ref_loc (loc
, src
);
9231 dst
= build_simple_mem_ref_loc (loc
, arg
);
9232 dst
= omp_build_component_ref (dst
, f
);
9233 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9234 append_to_statement_list (t
, &list
);
9236 case OMP_CLAUSE_PRIVATE
:
9237 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9239 decl
= OMP_CLAUSE_DECL (c
);
9240 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9241 f
= (tree
) n
->value
;
9242 if (tcctx
.cb
.decl_map
)
9243 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9244 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9247 sf
= (tree
) n
->value
;
9248 if (tcctx
.cb
.decl_map
)
9249 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9250 src
= build_simple_mem_ref_loc (loc
, sarg
);
9251 src
= omp_build_component_ref (src
, sf
);
9252 if (use_pointer_for_field (decl
, NULL
))
9253 src
= build_simple_mem_ref_loc (loc
, src
);
9257 dst
= build_simple_mem_ref_loc (loc
, arg
);
9258 dst
= omp_build_component_ref (dst
, f
);
9259 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9260 append_to_statement_list (t
, &list
);
9266 /* Last pass: handle VLA firstprivates. */
9267 if (tcctx
.cb
.decl_map
)
9268 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9269 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9273 decl
= OMP_CLAUSE_DECL (c
);
9274 if (!is_variable_sized (decl
))
9276 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9279 f
= (tree
) n
->value
;
9280 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9281 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9282 ind
= DECL_VALUE_EXPR (decl
);
9283 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9284 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9285 n
= splay_tree_lookup (ctx
->sfield_map
,
9286 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9287 sf
= (tree
) n
->value
;
9288 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9289 src
= build_simple_mem_ref_loc (loc
, sarg
);
9290 src
= omp_build_component_ref (src
, sf
);
9291 src
= build_simple_mem_ref_loc (loc
, src
);
9292 dst
= build_simple_mem_ref_loc (loc
, arg
);
9293 dst
= omp_build_component_ref (dst
, f
);
9294 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9295 append_to_statement_list (t
, &list
);
9296 n
= splay_tree_lookup (ctx
->field_map
,
9297 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9298 df
= (tree
) n
->value
;
9299 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9300 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9301 ptr
= omp_build_component_ref (ptr
, df
);
9302 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9303 build_fold_addr_expr_loc (loc
, dst
));
9304 append_to_statement_list (t
, &list
);
9307 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9308 append_to_statement_list (t
, &list
);
9310 if (tcctx
.cb
.decl_map
)
9311 pointer_map_destroy (tcctx
.cb
.decl_map
);
9312 pop_gimplify_context (NULL
);
9313 BIND_EXPR_BODY (bind
) = list
;
9318 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9322 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9324 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9326 gcc_assert (clauses
);
9327 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9328 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9329 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9331 case OMP_CLAUSE_DEPEND_IN
:
9334 case OMP_CLAUSE_DEPEND_OUT
:
9335 case OMP_CLAUSE_DEPEND_INOUT
:
9341 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9342 tree array
= create_tmp_var (type
, NULL
);
9343 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9345 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9346 gimple_seq_add_stmt (iseq
, g
);
9347 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9349 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9350 gimple_seq_add_stmt (iseq
, g
);
9351 for (i
= 0; i
< 2; i
++)
9353 if ((i
? n_in
: n_out
) == 0)
9355 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9356 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9357 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9359 tree t
= OMP_CLAUSE_DECL (c
);
9360 t
= fold_convert (ptr_type_node
, t
);
9361 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9362 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9363 NULL_TREE
, NULL_TREE
);
9364 g
= gimple_build_assign (r
, t
);
9365 gimple_seq_add_stmt (iseq
, g
);
9368 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9369 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9370 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9371 OMP_CLAUSE_CHAIN (c
) = *p
;
9373 tree clobber
= build_constructor (type
, NULL
);
9374 TREE_THIS_VOLATILE (clobber
) = 1;
9375 g
= gimple_build_assign (array
, clobber
);
9376 gimple_seq_add_stmt (oseq
, g
);
9379 /* Lower the OpenMP parallel or task directive in the current statement
9380 in GSI_P. CTX holds context information for the directive. */
9383 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9387 gimple stmt
= gsi_stmt (*gsi_p
);
9388 gimple par_bind
, bind
, dep_bind
= NULL
;
9389 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9390 struct gimplify_ctx gctx
, dep_gctx
;
9391 location_t loc
= gimple_location (stmt
);
9393 clauses
= gimple_omp_taskreg_clauses (stmt
);
9394 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9395 par_body
= gimple_bind_body (par_bind
);
9396 child_fn
= ctx
->cb
.dst_fn
;
9397 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9398 && !gimple_omp_parallel_combined_p (stmt
))
9400 struct walk_stmt_info wi
;
9403 memset (&wi
, 0, sizeof (wi
));
9406 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9408 gimple_omp_parallel_set_combined_p (stmt
, true);
9410 gimple_seq dep_ilist
= NULL
;
9411 gimple_seq dep_olist
= NULL
;
9412 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9413 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9415 push_gimplify_context (&dep_gctx
);
9416 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9417 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9420 if (ctx
->srecord_type
)
9421 create_task_copyfn (stmt
, ctx
);
9423 push_gimplify_context (&gctx
);
9428 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9429 lower_omp (&par_body
, ctx
);
9430 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9431 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9433 /* Declare all the variables created by mapping and the variables
9434 declared in the scope of the parallel body. */
9435 record_vars_into (ctx
->block_vars
, child_fn
);
9436 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9438 if (ctx
->record_type
)
9441 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9442 : ctx
->record_type
, ".omp_data_o");
9443 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9444 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9445 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9450 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9451 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9453 if (ctx
->record_type
)
9455 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9456 TREE_THIS_VOLATILE (clobber
) = 1;
9457 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9461 /* Once all the expansions are done, sequence all the different
9462 fragments inside gimple_omp_body. */
9466 if (ctx
->record_type
)
9468 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9469 /* fixup_child_record_type might have changed receiver_decl's type. */
9470 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9471 gimple_seq_add_stmt (&new_body
,
9472 gimple_build_assign (ctx
->receiver_decl
, t
));
9475 gimple_seq_add_seq (&new_body
, par_ilist
);
9476 gimple_seq_add_seq (&new_body
, par_body
);
9477 gimple_seq_add_seq (&new_body
, par_rlist
);
9478 if (ctx
->cancellable
)
9479 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9480 gimple_seq_add_seq (&new_body
, par_olist
);
9481 new_body
= maybe_catch_exception (new_body
);
9482 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9483 gimple_omp_set_body (stmt
, new_body
);
9485 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9486 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9487 gimple_bind_add_seq (bind
, ilist
);
9488 gimple_bind_add_stmt (bind
, stmt
);
9489 gimple_bind_add_seq (bind
, olist
);
9491 pop_gimplify_context (NULL
);
9495 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9496 gimple_bind_add_stmt (dep_bind
, bind
);
9497 gimple_bind_add_seq (dep_bind
, dep_olist
);
9498 pop_gimplify_context (dep_bind
);
9502 /* Lower the OpenMP target directive in the current statement
9503 in GSI_P. CTX holds context information for the directive. */
9506 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9509 tree child_fn
, t
, c
;
9510 gimple stmt
= gsi_stmt (*gsi_p
);
9511 gimple tgt_bind
= NULL
, bind
;
9512 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9513 struct gimplify_ctx gctx
;
9514 location_t loc
= gimple_location (stmt
);
9515 int kind
= gimple_omp_target_kind (stmt
);
9516 unsigned int map_cnt
= 0;
9518 clauses
= gimple_omp_target_clauses (stmt
);
9519 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9521 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9522 tgt_body
= gimple_bind_body (tgt_bind
);
9524 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9525 tgt_body
= gimple_omp_body (stmt
);
9526 child_fn
= ctx
->cb
.dst_fn
;
9528 push_gimplify_context (&gctx
);
9530 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9531 switch (OMP_CLAUSE_CODE (c
))
9537 case OMP_CLAUSE_MAP
:
9539 case OMP_CLAUSE_FROM
:
9540 var
= OMP_CLAUSE_DECL (c
);
9543 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9544 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9550 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9552 tree var2
= DECL_VALUE_EXPR (var
);
9553 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9554 var2
= TREE_OPERAND (var2
, 0);
9555 gcc_assert (DECL_P (var2
));
9559 if (!maybe_lookup_field (var
, ctx
))
9562 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9564 x
= build_receiver_ref (var
, true, ctx
);
9565 tree new_var
= lookup_decl (var
, ctx
);
9566 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9567 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9568 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9569 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9570 x
= build_simple_mem_ref (x
);
9571 SET_DECL_VALUE_EXPR (new_var
, x
);
9572 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9577 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9579 target_nesting_level
++;
9580 lower_omp (&tgt_body
, ctx
);
9581 target_nesting_level
--;
9583 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9584 lower_omp (&tgt_body
, ctx
);
9586 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9588 /* Declare all the variables created by mapping and the variables
9589 declared in the scope of the target body. */
9590 record_vars_into (ctx
->block_vars
, child_fn
);
9591 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9596 if (ctx
->record_type
)
9599 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9600 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9601 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9602 t
= make_tree_vec (3);
9603 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9605 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9607 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9608 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9609 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9611 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9614 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9615 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9616 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9617 gimple_omp_target_set_data_arg (stmt
, t
);
9619 vec
<constructor_elt
, va_gc
> *vsize
;
9620 vec
<constructor_elt
, va_gc
> *vkind
;
9621 vec_alloc (vsize
, map_cnt
);
9622 vec_alloc (vkind
, map_cnt
);
9623 unsigned int map_idx
= 0;
9625 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9626 switch (OMP_CLAUSE_CODE (c
))
9632 case OMP_CLAUSE_MAP
:
9634 case OMP_CLAUSE_FROM
:
9636 ovar
= OMP_CLAUSE_DECL (c
);
9639 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9640 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9642 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9643 == get_base_address (ovar
));
9644 nc
= OMP_CLAUSE_CHAIN (c
);
9645 ovar
= OMP_CLAUSE_DECL (nc
);
9649 tree x
= build_sender_ref (ovar
, ctx
);
9651 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9652 gimplify_assign (x
, v
, &ilist
);
9658 if (DECL_SIZE (ovar
)
9659 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9661 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9662 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9663 ovar2
= TREE_OPERAND (ovar2
, 0);
9664 gcc_assert (DECL_P (ovar2
));
9667 if (!maybe_lookup_field (ovar
, ctx
))
9673 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9674 tree x
= build_sender_ref (ovar
, ctx
);
9675 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9676 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9677 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9678 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9680 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9682 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9683 mark_addressable (avar
);
9684 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9685 avar
= build_fold_addr_expr (avar
);
9686 gimplify_assign (x
, avar
, &ilist
);
9688 else if (is_gimple_reg (var
))
9690 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9691 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9692 mark_addressable (avar
);
9693 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9694 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9695 gimplify_assign (avar
, var
, &ilist
);
9696 avar
= build_fold_addr_expr (avar
);
9697 gimplify_assign (x
, avar
, &ilist
);
9698 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9699 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9700 && !TYPE_READONLY (TREE_TYPE (var
)))
9702 x
= build_sender_ref (ovar
, ctx
);
9703 x
= build_simple_mem_ref (x
);
9704 gimplify_assign (var
, x
, &olist
);
9709 var
= build_fold_addr_expr (var
);
9710 gimplify_assign (x
, var
, &ilist
);
9713 tree s
= OMP_CLAUSE_SIZE (c
);
9715 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9716 s
= fold_convert (size_type_node
, s
);
9717 tree purpose
= size_int (map_idx
++);
9718 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9719 if (TREE_CODE (s
) != INTEGER_CST
)
9720 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9722 unsigned char tkind
= 0;
9723 switch (OMP_CLAUSE_CODE (c
))
9725 case OMP_CLAUSE_MAP
:
9726 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9729 tkind
= OMP_CLAUSE_MAP_TO
;
9731 case OMP_CLAUSE_FROM
:
9732 tkind
= OMP_CLAUSE_MAP_FROM
;
9737 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9738 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9739 talign
= DECL_ALIGN_UNIT (ovar
);
9740 talign
= ceil_log2 (talign
);
9741 tkind
|= talign
<< 3;
9742 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9743 build_int_cst (unsigned_char_type_node
,
9749 gcc_assert (map_idx
== map_cnt
);
9751 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9752 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9753 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9754 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9755 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9757 gimple_seq initlist
= NULL
;
9758 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9759 TREE_VEC_ELT (t
, 1)),
9760 &initlist
, true, NULL_TREE
);
9761 gimple_seq_add_seq (&ilist
, initlist
);
9764 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9765 TREE_THIS_VOLATILE (clobber
) = 1;
9766 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9770 /* Once all the expansions are done, sequence all the different
9771 fragments inside gimple_omp_body. */
9775 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9777 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9778 /* fixup_child_record_type might have changed receiver_decl's type. */
9779 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9780 gimple_seq_add_stmt (&new_body
,
9781 gimple_build_assign (ctx
->receiver_decl
, t
));
9784 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9786 gimple_seq_add_seq (&new_body
, tgt_body
);
9787 new_body
= maybe_catch_exception (new_body
);
9789 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9790 new_body
= tgt_body
;
9791 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9793 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9794 gimple_omp_set_body (stmt
, new_body
);
9797 bind
= gimple_build_bind (NULL
, NULL
,
9798 tgt_bind
? gimple_bind_block (tgt_bind
)
9800 gsi_replace (gsi_p
, bind
, true);
9801 gimple_bind_add_seq (bind
, ilist
);
9802 gimple_bind_add_stmt (bind
, stmt
);
9803 gimple_bind_add_seq (bind
, olist
);
9805 pop_gimplify_context (NULL
);
9808 /* Expand code for an OpenMP teams directive. */
9811 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9813 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9814 struct gimplify_ctx gctx
;
9815 push_gimplify_context (&gctx
);
9817 tree block
= make_node (BLOCK
);
9818 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9819 gsi_replace (gsi_p
, bind
, true);
9820 gimple_seq bind_body
= NULL
;
9821 gimple_seq dlist
= NULL
;
9822 gimple_seq olist
= NULL
;
9824 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9825 OMP_CLAUSE_NUM_TEAMS
);
9826 if (num_teams
== NULL_TREE
)
9827 num_teams
= build_int_cst (unsigned_type_node
, 0);
9830 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9831 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9832 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9834 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9835 OMP_CLAUSE_THREAD_LIMIT
);
9836 if (thread_limit
== NULL_TREE
)
9837 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9840 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9841 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9842 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9846 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9847 &bind_body
, &dlist
, ctx
, NULL
);
9848 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9849 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9850 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9852 location_t loc
= gimple_location (teams_stmt
);
9853 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9854 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9855 gimple_set_location (call
, loc
);
9856 gimple_seq_add_stmt (&bind_body
, call
);
9858 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
9859 gimple_omp_set_body (teams_stmt
, NULL
);
9860 gimple_seq_add_seq (&bind_body
, olist
);
9861 gimple_seq_add_seq (&bind_body
, dlist
);
9862 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
9863 gimple_bind_set_body (bind
, bind_body
);
9865 pop_gimplify_context (bind
);
9867 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9868 BLOCK_VARS (block
) = ctx
->block_vars
;
9869 if (BLOCK_VARS (block
))
9870 TREE_USED (block
) = 1;
9874 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9875 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9876 of OpenMP context, but with task_shared_vars set. */
9879 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
9884 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9885 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
9888 if (task_shared_vars
9890 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
9893 /* If a global variable has been privatized, TREE_CONSTANT on
9894 ADDR_EXPR might be wrong. */
9895 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
9896 recompute_tree_invariant_for_addr_expr (t
);
9898 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
9903 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9905 gimple stmt
= gsi_stmt (*gsi_p
);
9906 struct walk_stmt_info wi
;
9908 if (gimple_has_location (stmt
))
9909 input_location
= gimple_location (stmt
);
9911 if (task_shared_vars
)
9912 memset (&wi
, '\0', sizeof (wi
));
9914 /* If we have issued syntax errors, avoid doing any heavy lifting.
9915 Just replace the OpenMP directives with a NOP to avoid
9916 confusing RTL expansion. */
9917 if (seen_error () && is_gimple_omp (stmt
))
9919 gsi_replace (gsi_p
, gimple_build_nop (), true);
9923 switch (gimple_code (stmt
))
9926 if ((ctx
|| task_shared_vars
)
9927 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
9928 ctx
? NULL
: &wi
, NULL
)
9929 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
9930 ctx
? NULL
: &wi
, NULL
)))
9931 gimple_regimplify_operands (stmt
, gsi_p
);
9934 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
9936 case GIMPLE_EH_FILTER
:
9937 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
9940 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
9941 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
9943 case GIMPLE_TRANSACTION
:
9944 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
9947 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
9949 case GIMPLE_OMP_PARALLEL
:
9950 case GIMPLE_OMP_TASK
:
9951 ctx
= maybe_lookup_ctx (stmt
);
9953 if (ctx
->cancellable
)
9954 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9955 lower_omp_taskreg (gsi_p
, ctx
);
9957 case GIMPLE_OMP_FOR
:
9958 ctx
= maybe_lookup_ctx (stmt
);
9960 if (ctx
->cancellable
)
9961 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9962 lower_omp_for (gsi_p
, ctx
);
9964 case GIMPLE_OMP_SECTIONS
:
9965 ctx
= maybe_lookup_ctx (stmt
);
9967 if (ctx
->cancellable
)
9968 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9969 lower_omp_sections (gsi_p
, ctx
);
9971 case GIMPLE_OMP_SINGLE
:
9972 ctx
= maybe_lookup_ctx (stmt
);
9974 lower_omp_single (gsi_p
, ctx
);
9976 case GIMPLE_OMP_MASTER
:
9977 ctx
= maybe_lookup_ctx (stmt
);
9979 lower_omp_master (gsi_p
, ctx
);
9981 case GIMPLE_OMP_TASKGROUP
:
9982 ctx
= maybe_lookup_ctx (stmt
);
9984 lower_omp_taskgroup (gsi_p
, ctx
);
9986 case GIMPLE_OMP_ORDERED
:
9987 ctx
= maybe_lookup_ctx (stmt
);
9989 lower_omp_ordered (gsi_p
, ctx
);
9991 case GIMPLE_OMP_CRITICAL
:
9992 ctx
= maybe_lookup_ctx (stmt
);
9994 lower_omp_critical (gsi_p
, ctx
);
9996 case GIMPLE_OMP_ATOMIC_LOAD
:
9997 if ((ctx
|| task_shared_vars
)
9998 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
9999 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10000 gimple_regimplify_operands (stmt
, gsi_p
);
10002 case GIMPLE_OMP_TARGET
:
10003 ctx
= maybe_lookup_ctx (stmt
);
10005 lower_omp_target (gsi_p
, ctx
);
10007 case GIMPLE_OMP_TEAMS
:
10008 ctx
= maybe_lookup_ctx (stmt
);
10010 lower_omp_teams (gsi_p
, ctx
);
10014 fndecl
= gimple_call_fndecl (stmt
);
10016 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10017 switch (DECL_FUNCTION_CODE (fndecl
))
10019 case BUILT_IN_GOMP_BARRIER
:
10023 case BUILT_IN_GOMP_CANCEL
:
10024 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10027 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10028 cctx
= cctx
->outer
;
10029 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10030 if (!cctx
->cancellable
)
10032 if (DECL_FUNCTION_CODE (fndecl
)
10033 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10035 stmt
= gimple_build_nop ();
10036 gsi_replace (gsi_p
, stmt
, false);
10041 lhs
= create_tmp_var (boolean_type_node
, NULL
);
10042 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10044 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10045 gimple_call_set_fndecl (stmt
, fndecl
);
10046 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10048 gimple_call_set_lhs (stmt
, lhs
);
10049 tree fallthru_label
;
10050 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10052 g
= gimple_build_label (fallthru_label
);
10053 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10054 g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
10055 cctx
->cancel_label
, fallthru_label
);
10056 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10063 if ((ctx
|| task_shared_vars
)
10064 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10066 gimple_regimplify_operands (stmt
, gsi_p
);
10072 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10074 location_t saved_location
= input_location
;
10075 gimple_stmt_iterator gsi
;
10076 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10077 lower_omp_1 (&gsi
, ctx
);
10078 /* Inside target region we haven't called fold_stmt during gimplification,
10079 because it can break code by adding decl references that weren't in the
10080 source. Call fold_stmt now. */
10081 if (target_nesting_level
)
10082 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10084 input_location
= saved_location
;
10087 /* Main entry point. */
10089 static unsigned int
10090 execute_lower_omp (void)
10094 /* This pass always runs, to provide PROP_gimple_lomp.
10095 But there is nothing to do unless -fopenmp is given. */
10096 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_enable_cilkplus
== 0)
10099 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10100 delete_omp_context
);
10102 body
= gimple_body (current_function_decl
);
10103 scan_omp (&body
, NULL
);
10104 gcc_assert (taskreg_nesting_level
== 0);
10106 if (all_contexts
->root
)
10108 struct gimplify_ctx gctx
;
10110 if (task_shared_vars
)
10111 push_gimplify_context (&gctx
);
10112 lower_omp (&body
, NULL
);
10113 if (task_shared_vars
)
10114 pop_gimplify_context (NULL
);
10119 splay_tree_delete (all_contexts
);
10120 all_contexts
= NULL
;
10122 BITMAP_FREE (task_shared_vars
);
10128 const pass_data pass_data_lower_omp
=
10130 GIMPLE_PASS
, /* type */
10131 "omplower", /* name */
10132 OPTGROUP_NONE
, /* optinfo_flags */
10133 false, /* has_gate */
10134 true, /* has_execute */
10135 TV_NONE
, /* tv_id */
10136 PROP_gimple_any
, /* properties_required */
10137 PROP_gimple_lomp
, /* properties_provided */
10138 0, /* properties_destroyed */
10139 0, /* todo_flags_start */
10140 0, /* todo_flags_finish */
10143 class pass_lower_omp
: public gimple_opt_pass
10146 pass_lower_omp (gcc::context
*ctxt
)
10147 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10150 /* opt_pass methods: */
10151 unsigned int execute () { return execute_lower_omp (); }
10153 }; // class pass_lower_omp
10155 } // anon namespace
10158 make_pass_lower_omp (gcc::context
*ctxt
)
10160 return new pass_lower_omp (ctxt
);
10163 /* The following is a utility to diagnose OpenMP structured block violations.
10164 It is not part of the "omplower" pass, as that's invoked too late. It
10165 should be invoked by the respective front ends after gimplification. */
10167 static splay_tree all_labels
;
10169 /* Check for mismatched contexts and generate an error if needed. Return
10170 true if an error is detected. */
10173 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10174 gimple branch_ctx
, gimple label_ctx
)
10176 if (label_ctx
== branch_ctx
)
10181 Previously we kept track of the label's entire context in diagnose_sb_[12]
10182 so we could traverse it and issue a correct "exit" or "enter" error
10183 message upon a structured block violation.
10185 We built the context by building a list with tree_cons'ing, but there is
10186 no easy counterpart in gimple tuples. It seems like far too much work
10187 for issuing exit/enter error messages. If someone really misses the
10188 distinct error message... patches welcome.
10192 /* Try to avoid confusing the user by producing and error message
10193 with correct "exit" or "enter" verbiage. We prefer "exit"
10194 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10195 if (branch_ctx
== NULL
)
10201 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10206 label_ctx
= TREE_CHAIN (label_ctx
);
10211 error ("invalid exit from OpenMP structured block");
10213 error ("invalid entry to OpenMP structured block");
10216 bool cilkplus_block
= false;
10217 if (flag_enable_cilkplus
)
10220 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10221 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10222 || (gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10223 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10224 cilkplus_block
= true;
10227 /* If it's obvious we have an invalid entry, be specific about the error. */
10228 if (branch_ctx
== NULL
)
10230 if (cilkplus_block
)
10231 error ("invalid entry to Cilk Plus structured block");
10233 error ("invalid entry to OpenMP structured block");
10237 /* Otherwise, be vague and lazy, but efficient. */
10238 if (cilkplus_block
)
10239 error ("invalid branch to/from a Cilk Plus structured block");
10241 error ("invalid branch to/from an OpenMP structured block");
10244 gsi_replace (gsi_p
, gimple_build_nop (), false);
10248 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10249 where each label is found. */
10252 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10253 struct walk_stmt_info
*wi
)
10255 gimple context
= (gimple
) wi
->info
;
10256 gimple inner_context
;
10257 gimple stmt
= gsi_stmt (*gsi_p
);
10259 *handled_ops_p
= true;
10261 switch (gimple_code (stmt
))
10265 case GIMPLE_OMP_PARALLEL
:
10266 case GIMPLE_OMP_TASK
:
10267 case GIMPLE_OMP_SECTIONS
:
10268 case GIMPLE_OMP_SINGLE
:
10269 case GIMPLE_OMP_SECTION
:
10270 case GIMPLE_OMP_MASTER
:
10271 case GIMPLE_OMP_ORDERED
:
10272 case GIMPLE_OMP_CRITICAL
:
10273 case GIMPLE_OMP_TARGET
:
10274 case GIMPLE_OMP_TEAMS
:
10275 case GIMPLE_OMP_TASKGROUP
:
10276 /* The minimal context here is just the current OMP construct. */
10277 inner_context
= stmt
;
10278 wi
->info
= inner_context
;
10279 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10280 wi
->info
= context
;
10283 case GIMPLE_OMP_FOR
:
10284 inner_context
= stmt
;
10285 wi
->info
= inner_context
;
10286 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10288 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10289 diagnose_sb_1
, NULL
, wi
);
10290 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10291 wi
->info
= context
;
10295 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10296 (splay_tree_value
) context
);
10306 /* Pass 2: Check each branch and see if its context differs from that of
10307 the destination label's context. */
10310 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10311 struct walk_stmt_info
*wi
)
10313 gimple context
= (gimple
) wi
->info
;
10315 gimple stmt
= gsi_stmt (*gsi_p
);
10317 *handled_ops_p
= true;
10319 switch (gimple_code (stmt
))
10323 case GIMPLE_OMP_PARALLEL
:
10324 case GIMPLE_OMP_TASK
:
10325 case GIMPLE_OMP_SECTIONS
:
10326 case GIMPLE_OMP_SINGLE
:
10327 case GIMPLE_OMP_SECTION
:
10328 case GIMPLE_OMP_MASTER
:
10329 case GIMPLE_OMP_ORDERED
:
10330 case GIMPLE_OMP_CRITICAL
:
10331 case GIMPLE_OMP_TARGET
:
10332 case GIMPLE_OMP_TEAMS
:
10333 case GIMPLE_OMP_TASKGROUP
:
10335 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10336 wi
->info
= context
;
10339 case GIMPLE_OMP_FOR
:
10341 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10343 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10344 diagnose_sb_2
, NULL
, wi
);
10345 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10346 wi
->info
= context
;
10351 tree lab
= gimple_cond_true_label (stmt
);
10354 n
= splay_tree_lookup (all_labels
,
10355 (splay_tree_key
) lab
);
10356 diagnose_sb_0 (gsi_p
, context
,
10357 n
? (gimple
) n
->value
: NULL
);
10359 lab
= gimple_cond_false_label (stmt
);
10362 n
= splay_tree_lookup (all_labels
,
10363 (splay_tree_key
) lab
);
10364 diagnose_sb_0 (gsi_p
, context
,
10365 n
? (gimple
) n
->value
: NULL
);
10372 tree lab
= gimple_goto_dest (stmt
);
10373 if (TREE_CODE (lab
) != LABEL_DECL
)
10376 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10377 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10381 case GIMPLE_SWITCH
:
10384 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10386 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10387 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10388 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10394 case GIMPLE_RETURN
:
10395 diagnose_sb_0 (gsi_p
, context
, NULL
);
10405 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10408 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
)
10410 gimple last
= last_stmt (bb
);
10411 enum gimple_code code
= gimple_code (last
);
10412 struct omp_region
*cur_region
= *region
;
10413 bool fallthru
= false;
10417 case GIMPLE_OMP_PARALLEL
:
10418 case GIMPLE_OMP_TASK
:
10419 case GIMPLE_OMP_FOR
:
10420 case GIMPLE_OMP_SINGLE
:
10421 case GIMPLE_OMP_TEAMS
:
10422 case GIMPLE_OMP_MASTER
:
10423 case GIMPLE_OMP_TASKGROUP
:
10424 case GIMPLE_OMP_ORDERED
:
10425 case GIMPLE_OMP_CRITICAL
:
10426 case GIMPLE_OMP_SECTION
:
10427 cur_region
= new_omp_region (bb
, code
, cur_region
);
10431 case GIMPLE_OMP_TARGET
:
10432 cur_region
= new_omp_region (bb
, code
, cur_region
);
10434 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10435 cur_region
= cur_region
->outer
;
10438 case GIMPLE_OMP_SECTIONS
:
10439 cur_region
= new_omp_region (bb
, code
, cur_region
);
10443 case GIMPLE_OMP_SECTIONS_SWITCH
:
10447 case GIMPLE_OMP_ATOMIC_LOAD
:
10448 case GIMPLE_OMP_ATOMIC_STORE
:
10452 case GIMPLE_OMP_RETURN
:
10453 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10454 somewhere other than the next block. This will be
10456 cur_region
->exit
= bb
;
10457 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10458 cur_region
= cur_region
->outer
;
10461 case GIMPLE_OMP_CONTINUE
:
10462 cur_region
->cont
= bb
;
10463 switch (cur_region
->type
)
10465 case GIMPLE_OMP_FOR
:
10466 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10467 succs edges as abnormal to prevent splitting
10469 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10470 /* Make the loopback edge. */
10471 make_edge (bb
, single_succ (cur_region
->entry
),
10474 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10475 corresponds to the case that the body of the loop
10476 is not executed at all. */
10477 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10478 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10482 case GIMPLE_OMP_SECTIONS
:
10483 /* Wire up the edges into and out of the nested sections. */
10485 basic_block switch_bb
= single_succ (cur_region
->entry
);
10487 struct omp_region
*i
;
10488 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10490 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10491 make_edge (switch_bb
, i
->entry
, 0);
10492 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10495 /* Make the loopback edge to the block with
10496 GIMPLE_OMP_SECTIONS_SWITCH. */
10497 make_edge (bb
, switch_bb
, 0);
10499 /* Make the edge from the switch to exit. */
10500 make_edge (switch_bb
, bb
->next_bb
, 0);
10506 gcc_unreachable ();
10511 gcc_unreachable ();
10514 if (*region
!= cur_region
)
10515 *region
= cur_region
;
10520 static unsigned int
10521 diagnose_omp_structured_block_errors (void)
10523 struct walk_stmt_info wi
;
10524 gimple_seq body
= gimple_body (current_function_decl
);
10526 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10528 memset (&wi
, 0, sizeof (wi
));
10529 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10531 memset (&wi
, 0, sizeof (wi
));
10532 wi
.want_locations
= true;
10533 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10535 gimple_set_body (current_function_decl
, body
);
10537 splay_tree_delete (all_labels
);
10544 gate_diagnose_omp_blocks (void)
10546 return flag_openmp
|| flag_enable_cilkplus
;
10551 const pass_data pass_data_diagnose_omp_blocks
=
10553 GIMPLE_PASS
, /* type */
10554 "*diagnose_omp_blocks", /* name */
10555 OPTGROUP_NONE
, /* optinfo_flags */
10556 true, /* has_gate */
10557 true, /* has_execute */
10558 TV_NONE
, /* tv_id */
10559 PROP_gimple_any
, /* properties_required */
10560 0, /* properties_provided */
10561 0, /* properties_destroyed */
10562 0, /* todo_flags_start */
10563 0, /* todo_flags_finish */
10566 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10569 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10570 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10573 /* opt_pass methods: */
10574 bool gate () { return gate_diagnose_omp_blocks (); }
10575 unsigned int execute () {
10576 return diagnose_omp_structured_block_errors ();
10579 }; // class pass_diagnose_omp_blocks
10581 } // anon namespace
10584 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10586 return new pass_diagnose_omp_blocks (ctxt
);
10589 #include "gt-omp-low.h"