1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
84 /* Lowering of OMP parallel and workshare constructs proceeds in two
85 phases. The first phase scans the function looking for OMP statements
86 and then for variables that must be replaced to satisfy data sharing
87 clauses. The second phase expands code for the constructs, as well as
88 re-gimplifying things when variables have been replaced with complex
91 Final code generation is done by pass_expand_omp. The flowgraph is
92 scanned for regions which are then moved to a new
93 function, to be invoked by the thread library, or offloaded. */
95 /* OMP region information. Every parallel and workshare
96 directive is enclosed between two markers, the OMP_* directive
97 and a corresponding GIMPLE_OMP_RETURN statement. */
101 /* The enclosing region. */
102 struct omp_region
*outer
;
104 /* First child region. */
105 struct omp_region
*inner
;
107 /* Next peer region. */
108 struct omp_region
*next
;
110 /* Block containing the omp directive as its last stmt. */
113 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
116 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
119 /* If this is a combined parallel+workshare region, this is a list
120 of additional arguments needed by the combined parallel+workshare
122 vec
<tree
, va_gc
> *ws_args
;
124 /* The code for the omp directive of this region. */
125 enum gimple_code type
;
127 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
128 enum omp_clause_schedule_kind sched_kind
;
130 /* Schedule modifiers. */
131 unsigned char sched_modifiers
;
133 /* True if this is a combined parallel+workshare region. */
134 bool is_combined_parallel
;
136 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
138 gomp_ordered
*ord_stmt
;
141 /* Context structure. Used to store information about each parallel
142 directive in the code. */
146 /* This field must be at the beginning, as we do "inheritance": Some
147 callback functions for tree-inline.c (e.g., omp_copy_decl)
148 receive a copy_body_data pointer that is up-casted to an
149 omp_context pointer. */
152 /* The tree of contexts corresponding to the encountered constructs. */
153 struct omp_context
*outer
;
156 /* Map variables to fields in a structure that allows communication
157 between sending and receiving threads. */
158 splay_tree field_map
;
163 /* These are used just by task contexts, if task firstprivate fn is
164 needed. srecord_type is used to communicate from the thread
165 that encountered the task construct to task firstprivate fn,
166 record_type is allocated by GOMP_task, initialized by task firstprivate
167 fn and passed to the task body fn. */
168 splay_tree sfield_map
;
171 /* A chain of variables to add to the top-level block surrounding the
172 construct. In the case of a parallel, this is in the child function. */
175 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
176 barriers should jump to during omplower pass. */
179 /* What to do with variables with implicitly determined sharing
181 enum omp_clause_default_kind default_kind
;
183 /* Nesting depth of this context. Used to beautify error messages re
184 invalid gotos. The outermost ctx is depth 1, with depth 0 being
185 reserved for the main body of the function. */
188 /* True if this parallel directive is nested within another. */
191 /* True if this construct can be cancelled. */
195 /* A structure holding the elements of:
196 for (V = N1; V cond N2; V += STEP) [...] */
198 struct omp_for_data_loop
200 tree v
, n1
, n2
, step
;
201 enum tree_code cond_code
;
204 /* A structure describing the main elements of a parallel loop. */
208 struct omp_for_data_loop loop
;
214 bool have_nowait
, have_ordered
, simd_schedule
;
215 unsigned char sched_modifiers
;
216 enum omp_clause_schedule_kind sched_kind
;
217 struct omp_for_data_loop
*loops
;
220 /* Describe the OpenACC looping structure of a function. The entire
221 function is held in a 'NULL' loop. */
225 oacc_loop
*parent
; /* Containing loop. */
227 oacc_loop
*child
; /* First inner loop. */
229 oacc_loop
*sibling
; /* Next loop within same parent. */
231 location_t loc
; /* Location of the loop start. */
233 gcall
*marker
; /* Initial head marker. */
235 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
236 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
238 tree routine
; /* Pseudo-loop enclosing a routine. */
240 unsigned mask
; /* Partitioning mask. */
241 unsigned flags
; /* Partitioning flags. */
242 tree chunk_size
; /* Chunk size. */
243 gcall
*head_end
; /* Final marker of head sequence. */
246 /* Flags for an OpenACC loop. */
248 enum oacc_loop_flags
{
249 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
250 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
251 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
252 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
254 /* Explicitly specified loop axes. */
256 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
257 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
258 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
260 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
264 static splay_tree all_contexts
;
265 static int taskreg_nesting_level
;
266 static int target_nesting_level
;
267 static struct omp_region
*root_omp_region
;
268 static bitmap task_shared_vars
;
269 static vec
<omp_context
*> taskreg_contexts
;
270 static bool omp_any_child_fn_dumped
;
272 static void scan_omp (gimple_seq
*, omp_context
*);
273 static tree
scan_omp_1_op (tree
*, int *, void *);
274 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
276 #define WALK_SUBSTMTS \
280 case GIMPLE_EH_FILTER: \
281 case GIMPLE_TRANSACTION: \
282 /* The sub-statements for these should be walked. */ \
283 *handled_ops_p = false; \
286 /* Return true if CTX corresponds to an oacc parallel region. */
289 is_oacc_parallel (omp_context
*ctx
)
291 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
292 return ((outer_type
== GIMPLE_OMP_TARGET
)
293 && (gimple_omp_target_kind (ctx
->stmt
)
294 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
297 /* Return true if CTX corresponds to an oacc kernels region. */
300 is_oacc_kernels (omp_context
*ctx
)
302 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
303 return ((outer_type
== GIMPLE_OMP_TARGET
)
304 && (gimple_omp_target_kind (ctx
->stmt
)
305 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
308 /* If DECL is the artificial dummy VAR_DECL created for non-static
309 data member privatization, return the underlying "this" parameter,
310 otherwise return NULL. */
313 omp_member_access_dummy_var (tree decl
)
316 || !DECL_ARTIFICIAL (decl
)
317 || !DECL_IGNORED_P (decl
)
318 || !DECL_HAS_VALUE_EXPR_P (decl
)
319 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
322 tree v
= DECL_VALUE_EXPR (decl
);
323 if (TREE_CODE (v
) != COMPONENT_REF
)
327 switch (TREE_CODE (v
))
333 case POINTER_PLUS_EXPR
:
334 v
= TREE_OPERAND (v
, 0);
337 if (DECL_CONTEXT (v
) == current_function_decl
338 && DECL_ARTIFICIAL (v
)
339 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
347 /* Helper for unshare_and_remap, called through walk_tree. */
350 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
352 tree
*pair
= (tree
*) data
;
355 *tp
= unshare_expr (pair
[1]);
358 else if (IS_TYPE_OR_DECL_P (*tp
))
363 /* Return unshare_expr (X) with all occurrences of FROM
367 unshare_and_remap (tree x
, tree from
, tree to
)
369 tree pair
[2] = { from
, to
};
370 x
= unshare_expr (x
);
371 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
375 /* Holds offload tables with decls. */
376 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
378 /* Convenience function for calling scan_omp_1_op on tree operands. */
381 scan_omp_op (tree
*tp
, omp_context
*ctx
)
383 struct walk_stmt_info wi
;
385 memset (&wi
, 0, sizeof (wi
));
387 wi
.want_locations
= true;
389 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
392 static void lower_omp (gimple_seq
*, omp_context
*);
393 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
394 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
396 /* Find an OMP clause of type KIND within CLAUSES. */
399 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
401 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
402 if (OMP_CLAUSE_CODE (clauses
) == kind
)
408 /* Return true if CTX is for an omp parallel. */
411 is_parallel_ctx (omp_context
*ctx
)
413 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
417 /* Return true if CTX is for an omp task. */
420 is_task_ctx (omp_context
*ctx
)
422 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
426 /* Return true if CTX is for an omp taskloop. */
429 is_taskloop_ctx (omp_context
*ctx
)
431 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
432 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
436 /* Return true if CTX is for an omp parallel or omp task. */
439 is_taskreg_ctx (omp_context
*ctx
)
441 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
445 /* Return true if REGION is a combined parallel+workshare region. */
448 is_combined_parallel (struct omp_region
*region
)
450 return region
->is_combined_parallel
;
454 /* Extract the header elements of parallel loop FOR_STMT and store
458 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
459 struct omp_for_data_loop
*loops
)
461 tree t
, var
, *collapse_iter
, *collapse_count
;
462 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
463 struct omp_for_data_loop
*loop
;
465 struct omp_for_data_loop dummy_loop
;
466 location_t loc
= gimple_location (for_stmt
);
467 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
468 bool distribute
= gimple_omp_for_kind (for_stmt
)
469 == GF_OMP_FOR_KIND_DISTRIBUTE
;
470 bool taskloop
= gimple_omp_for_kind (for_stmt
)
471 == GF_OMP_FOR_KIND_TASKLOOP
;
474 fd
->for_stmt
= for_stmt
;
476 if (gimple_omp_for_collapse (for_stmt
) > 1)
479 fd
->loops
= &fd
->loop
;
481 fd
->have_nowait
= distribute
|| simd
;
482 fd
->have_ordered
= false;
485 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
486 fd
->sched_modifiers
= 0;
487 fd
->chunk_size
= NULL_TREE
;
488 fd
->simd_schedule
= false;
489 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
490 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
491 collapse_iter
= NULL
;
492 collapse_count
= NULL
;
494 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
495 switch (OMP_CLAUSE_CODE (t
))
497 case OMP_CLAUSE_NOWAIT
:
498 fd
->have_nowait
= true;
500 case OMP_CLAUSE_ORDERED
:
501 fd
->have_ordered
= true;
502 if (OMP_CLAUSE_ORDERED_EXPR (t
))
503 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
505 case OMP_CLAUSE_SCHEDULE
:
506 gcc_assert (!distribute
&& !taskloop
);
508 = (enum omp_clause_schedule_kind
)
509 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
510 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
511 & ~OMP_CLAUSE_SCHEDULE_MASK
);
512 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
513 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
515 case OMP_CLAUSE_DIST_SCHEDULE
:
516 gcc_assert (distribute
);
517 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
519 case OMP_CLAUSE_COLLAPSE
:
520 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
521 if (fd
->collapse
> 1)
523 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
524 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
530 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
535 collapse_iter
= &iterv
;
536 collapse_count
= &countv
;
539 /* FIXME: for now map schedule(auto) to schedule(static).
540 There should be analysis to determine whether all iterations
541 are approximately the same amount of work (then schedule(static)
542 is best) or if it varies (then schedule(dynamic,N) is better). */
543 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
545 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
546 gcc_assert (fd
->chunk_size
== NULL
);
548 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
550 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
551 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
552 gcc_assert (fd
->chunk_size
== NULL
);
553 else if (fd
->chunk_size
== NULL
)
555 /* We only need to compute a default chunk size for ordered
556 static loops and dynamic loops. */
557 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
559 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
560 ? integer_zero_node
: integer_one_node
;
563 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
564 for (i
= 0; i
< cnt
; i
++)
566 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
568 else if (loops
!= NULL
)
573 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
574 gcc_assert (SSA_VAR_P (loop
->v
));
575 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
576 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
577 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
578 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
580 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
581 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
582 switch (loop
->cond_code
)
588 gcc_assert (gimple_omp_for_kind (for_stmt
)
589 == GF_OMP_FOR_KIND_CILKSIMD
590 || (gimple_omp_for_kind (for_stmt
)
591 == GF_OMP_FOR_KIND_CILKFOR
));
594 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
595 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
597 loop
->n2
= fold_build2_loc (loc
,
598 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
599 build_int_cst (TREE_TYPE (loop
->n2
), 1));
600 loop
->cond_code
= LT_EXPR
;
603 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
604 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
606 loop
->n2
= fold_build2_loc (loc
,
607 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
608 build_int_cst (TREE_TYPE (loop
->n2
), 1));
609 loop
->cond_code
= GT_EXPR
;
615 t
= gimple_omp_for_incr (for_stmt
, i
);
616 gcc_assert (TREE_OPERAND (t
, 0) == var
);
617 switch (TREE_CODE (t
))
620 loop
->step
= TREE_OPERAND (t
, 1);
622 case POINTER_PLUS_EXPR
:
623 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
626 loop
->step
= TREE_OPERAND (t
, 1);
627 loop
->step
= fold_build1_loc (loc
,
628 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
636 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
637 && !fd
->have_ordered
))
639 if (fd
->collapse
== 1)
640 iter_type
= TREE_TYPE (loop
->v
);
642 || TYPE_PRECISION (iter_type
)
643 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
645 = build_nonstandard_integer_type
646 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
648 else if (iter_type
!= long_long_unsigned_type_node
)
650 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
651 iter_type
= long_long_unsigned_type_node
;
652 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
653 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
654 >= TYPE_PRECISION (iter_type
))
658 if (loop
->cond_code
== LT_EXPR
)
659 n
= fold_build2_loc (loc
,
660 PLUS_EXPR
, TREE_TYPE (loop
->v
),
661 loop
->n2
, loop
->step
);
664 if (TREE_CODE (n
) != INTEGER_CST
665 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
666 iter_type
= long_long_unsigned_type_node
;
668 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
669 > TYPE_PRECISION (iter_type
))
673 if (loop
->cond_code
== LT_EXPR
)
676 n2
= fold_build2_loc (loc
,
677 PLUS_EXPR
, TREE_TYPE (loop
->v
),
678 loop
->n2
, loop
->step
);
682 n1
= fold_build2_loc (loc
,
683 MINUS_EXPR
, TREE_TYPE (loop
->v
),
684 loop
->n2
, loop
->step
);
687 if (TREE_CODE (n1
) != INTEGER_CST
688 || TREE_CODE (n2
) != INTEGER_CST
689 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
690 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
691 iter_type
= long_long_unsigned_type_node
;
695 if (i
>= fd
->collapse
)
698 if (collapse_count
&& *collapse_count
== NULL
)
700 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
701 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
702 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
703 if (t
&& integer_zerop (t
))
704 count
= build_zero_cst (long_long_unsigned_type_node
);
705 else if ((i
== 0 || count
!= NULL_TREE
)
706 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
707 && TREE_CONSTANT (loop
->n1
)
708 && TREE_CONSTANT (loop
->n2
)
709 && TREE_CODE (loop
->step
) == INTEGER_CST
)
711 tree itype
= TREE_TYPE (loop
->v
);
713 if (POINTER_TYPE_P (itype
))
714 itype
= signed_type_for (itype
);
715 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
716 t
= fold_build2_loc (loc
,
718 fold_convert_loc (loc
, itype
, loop
->step
), t
);
719 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
720 fold_convert_loc (loc
, itype
, loop
->n2
));
721 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
722 fold_convert_loc (loc
, itype
, loop
->n1
));
723 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
724 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
725 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
726 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
727 fold_convert_loc (loc
, itype
,
730 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
731 fold_convert_loc (loc
, itype
, loop
->step
));
732 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
733 if (count
!= NULL_TREE
)
734 count
= fold_build2_loc (loc
,
735 MULT_EXPR
, long_long_unsigned_type_node
,
739 if (TREE_CODE (count
) != INTEGER_CST
)
742 else if (count
&& !integer_zerop (count
))
749 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
750 || fd
->have_ordered
))
752 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
753 iter_type
= long_long_unsigned_type_node
;
755 iter_type
= long_integer_type_node
;
757 else if (collapse_iter
&& *collapse_iter
!= NULL
)
758 iter_type
= TREE_TYPE (*collapse_iter
);
759 fd
->iter_type
= iter_type
;
760 if (collapse_iter
&& *collapse_iter
== NULL
)
761 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
762 if (collapse_count
&& *collapse_count
== NULL
)
765 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
767 *collapse_count
= create_tmp_var (iter_type
, ".count");
770 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
772 fd
->loop
.v
= *collapse_iter
;
773 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
774 fd
->loop
.n2
= *collapse_count
;
775 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
776 fd
->loop
.cond_code
= LT_EXPR
;
783 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
784 is the immediate dominator of PAR_ENTRY_BB, return true if there
785 are no data dependencies that would prevent expanding the parallel
786 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
788 When expanding a combined parallel+workshare region, the call to
789 the child function may need additional arguments in the case of
790 GIMPLE_OMP_FOR regions. In some cases, these arguments are
791 computed out of variables passed in from the parent to the child
792 via 'struct .omp_data_s'. For instance:
794 #pragma omp parallel for schedule (guided, i * 4)
799 # BLOCK 2 (PAR_ENTRY_BB)
801 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
803 # BLOCK 3 (WS_ENTRY_BB)
804 .omp_data_i = &.omp_data_o;
805 D.1667 = .omp_data_i->i;
807 #pragma omp for schedule (guided, D.1598)
809 When we outline the parallel region, the call to the child function
810 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
811 that value is computed *after* the call site. So, in principle we
812 cannot do the transformation.
814 To see whether the code in WS_ENTRY_BB blocks the combined
815 parallel+workshare call, we collect all the variables used in the
816 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
817 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
820 FIXME. If we had the SSA form built at this point, we could merely
821 hoist the code in block 3 into block 2 and be done with it. But at
822 this point we don't have dataflow information and though we could
823 hack something up here, it is really not worth the aggravation. */
826 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
828 struct omp_for_data fd
;
829 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
831 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
834 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
836 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
838 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
840 if (fd
.iter_type
!= long_integer_type_node
)
843 /* FIXME. We give up too easily here. If any of these arguments
844 are not constants, they will likely involve variables that have
845 been mapped into fields of .omp_data_s for sharing with the child
846 function. With appropriate data flow, it would be possible to
848 if (!is_gimple_min_invariant (fd
.loop
.n1
)
849 || !is_gimple_min_invariant (fd
.loop
.n2
)
850 || !is_gimple_min_invariant (fd
.loop
.step
)
851 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
858 static int omp_max_vf (void);
860 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
861 presence (SIMD_SCHEDULE). */
864 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
869 int vf
= omp_max_vf ();
873 tree type
= TREE_TYPE (chunk_size
);
874 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
875 build_int_cst (type
, vf
- 1));
876 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
877 build_int_cst (type
, -vf
));
881 /* Collect additional arguments needed to emit a combined
882 parallel+workshare call. WS_STMT is the workshare directive being
885 static vec
<tree
, va_gc
> *
886 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
889 location_t loc
= gimple_location (ws_stmt
);
890 vec
<tree
, va_gc
> *ws_args
;
892 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
894 struct omp_for_data fd
;
897 extract_omp_for_data (for_stmt
, &fd
, NULL
);
901 if (gimple_omp_for_combined_into_p (for_stmt
))
904 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
905 OMP_CLAUSE__LOOPTEMP_
);
907 n1
= OMP_CLAUSE_DECL (innerc
);
908 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
909 OMP_CLAUSE__LOOPTEMP_
);
911 n2
= OMP_CLAUSE_DECL (innerc
);
914 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
916 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
917 ws_args
->quick_push (t
);
919 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
920 ws_args
->quick_push (t
);
922 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
923 ws_args
->quick_push (t
);
927 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
928 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
929 ws_args
->quick_push (t
);
934 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
936 /* Number of sections is equal to the number of edges from the
937 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
938 the exit of the sections region. */
939 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
940 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
941 vec_alloc (ws_args
, 1);
942 ws_args
->quick_push (t
);
950 /* Discover whether REGION is a combined parallel+workshare region. */
953 determine_parallel_type (struct omp_region
*region
)
955 basic_block par_entry_bb
, par_exit_bb
;
956 basic_block ws_entry_bb
, ws_exit_bb
;
958 if (region
== NULL
|| region
->inner
== NULL
959 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
960 || region
->inner
->cont
== NULL
)
963 /* We only support parallel+for and parallel+sections. */
964 if (region
->type
!= GIMPLE_OMP_PARALLEL
965 || (region
->inner
->type
!= GIMPLE_OMP_FOR
966 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
969 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
970 WS_EXIT_BB -> PAR_EXIT_BB. */
971 par_entry_bb
= region
->entry
;
972 par_exit_bb
= region
->exit
;
973 ws_entry_bb
= region
->inner
->entry
;
974 ws_exit_bb
= region
->inner
->exit
;
976 if (single_succ (par_entry_bb
) == ws_entry_bb
977 && single_succ (ws_exit_bb
) == par_exit_bb
978 && workshare_safe_to_combine_p (ws_entry_bb
)
979 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
980 || (last_and_only_stmt (ws_entry_bb
)
981 && last_and_only_stmt (par_exit_bb
))))
983 gimple
*par_stmt
= last_stmt (par_entry_bb
);
984 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
986 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
988 /* If this is a combined parallel loop, we need to determine
989 whether or not to use the combined library calls. There
990 are two cases where we do not apply the transformation:
991 static loops and any kind of ordered loop. In the first
992 case, we already open code the loop so there is no need
993 to do anything else. In the latter case, the combined
994 parallel loop call would still need extra synchronization
995 to implement ordered semantics, so there would not be any
996 gain in using the combined call. */
997 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
998 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1000 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1001 == OMP_CLAUSE_SCHEDULE_STATIC
)
1002 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1004 region
->is_combined_parallel
= false;
1005 region
->inner
->is_combined_parallel
= false;
1010 region
->is_combined_parallel
= true;
1011 region
->inner
->is_combined_parallel
= true;
1012 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1017 /* Return true if EXPR is variable sized. */
1020 is_variable_sized (const_tree expr
)
1022 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1025 /* Return true if DECL is a reference type. */
1028 is_reference (tree decl
)
1030 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1033 /* Return the type of a decl. If the decl is reference type,
1034 return its base type. */
1036 get_base_type (tree decl
)
1038 tree type
= TREE_TYPE (decl
);
1039 if (is_reference (decl
))
1040 type
= TREE_TYPE (type
);
1044 /* Lookup variables. The "maybe" form
1045 allows for the variable form to not have been entered, otherwise we
1046 assert that the variable must have been entered. */
1049 lookup_decl (tree var
, omp_context
*ctx
)
1051 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1056 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1058 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1059 return n
? *n
: NULL_TREE
;
1063 lookup_field (tree var
, omp_context
*ctx
)
1066 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1067 return (tree
) n
->value
;
1071 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1074 n
= splay_tree_lookup (ctx
->sfield_map
1075 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1076 return (tree
) n
->value
;
1080 lookup_sfield (tree var
, omp_context
*ctx
)
1082 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1086 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1089 n
= splay_tree_lookup (ctx
->field_map
, key
);
1090 return n
? (tree
) n
->value
: NULL_TREE
;
1094 maybe_lookup_field (tree var
, omp_context
*ctx
)
1096 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1099 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1100 the parallel context if DECL is to be shared. */
1103 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1105 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1108 /* We can only use copy-in/copy-out semantics for shared variables
1109 when we know the value is not accessible from an outer scope. */
1112 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1114 /* ??? Trivially accessible from anywhere. But why would we even
1115 be passing an address in this case? Should we simply assert
1116 this to be false, or should we have a cleanup pass that removes
1117 these from the list of mappings? */
1118 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1121 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1122 without analyzing the expression whether or not its location
1123 is accessible to anyone else. In the case of nested parallel
1124 regions it certainly may be. */
1125 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1128 /* Do not use copy-in/copy-out for variables that have their
1130 if (TREE_ADDRESSABLE (decl
))
1133 /* lower_send_shared_vars only uses copy-in, but not copy-out
1135 if (TREE_READONLY (decl
)
1136 || ((TREE_CODE (decl
) == RESULT_DECL
1137 || TREE_CODE (decl
) == PARM_DECL
)
1138 && DECL_BY_REFERENCE (decl
)))
1141 /* Disallow copy-in/out in nested parallel if
1142 decl is shared in outer parallel, otherwise
1143 each thread could store the shared variable
1144 in its own copy-in location, making the
1145 variable no longer really shared. */
1146 if (shared_ctx
->is_nested
)
1150 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1151 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1158 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1159 c
; c
= OMP_CLAUSE_CHAIN (c
))
1160 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1161 && OMP_CLAUSE_DECL (c
) == decl
)
1165 goto maybe_mark_addressable_and_ret
;
1169 /* For tasks avoid using copy-in/out. As tasks can be
1170 deferred or executed in different thread, when GOMP_task
1171 returns, the task hasn't necessarily terminated. */
1172 if (is_task_ctx (shared_ctx
))
1175 maybe_mark_addressable_and_ret
:
1176 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1177 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1179 /* Taking address of OUTER in lower_send_shared_vars
1180 might need regimplification of everything that uses the
1182 if (!task_shared_vars
)
1183 task_shared_vars
= BITMAP_ALLOC (NULL
);
1184 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1185 TREE_ADDRESSABLE (outer
) = 1;
1194 /* Construct a new automatic decl similar to VAR. */
1197 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1199 tree copy
= copy_var_decl (var
, name
, type
);
1201 DECL_CONTEXT (copy
) = current_function_decl
;
1202 DECL_CHAIN (copy
) = ctx
->block_vars
;
1203 /* If VAR is listed in task_shared_vars, it means it wasn't
1204 originally addressable and is just because task needs to take
1205 it's address. But we don't need to take address of privatizations
1207 if (TREE_ADDRESSABLE (var
)
1209 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1210 TREE_ADDRESSABLE (copy
) = 0;
1211 ctx
->block_vars
= copy
;
1217 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1219 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1222 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1225 omp_build_component_ref (tree obj
, tree field
)
1227 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1228 if (TREE_THIS_VOLATILE (field
))
1229 TREE_THIS_VOLATILE (ret
) |= 1;
1230 if (TREE_READONLY (field
))
1231 TREE_READONLY (ret
) |= 1;
1235 /* Build tree nodes to access the field for VAR on the receiver side. */
1238 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1240 tree x
, field
= lookup_field (var
, ctx
);
1242 /* If the receiver record type was remapped in the child function,
1243 remap the field into the new record type. */
1244 x
= maybe_lookup_field (field
, ctx
);
1248 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1249 TREE_THIS_NOTRAP (x
) = 1;
1250 x
= omp_build_component_ref (x
, field
);
1252 x
= build_simple_mem_ref (x
);
1257 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1258 of a parallel, this is a component reference; for workshare constructs
1259 this is some variable. */
1262 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1266 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1268 else if (is_variable_sized (var
))
1270 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1271 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1272 x
= build_simple_mem_ref (x
);
1274 else if (is_taskreg_ctx (ctx
))
1276 bool by_ref
= use_pointer_for_field (var
, NULL
);
1277 x
= build_receiver_ref (var
, by_ref
, ctx
);
1279 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1280 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1282 /* #pragma omp simd isn't a worksharing construct, and can reference even
1283 private vars in its linear etc. clauses. */
1285 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1286 x
= lookup_decl (var
, ctx
->outer
);
1287 else if (ctx
->outer
)
1288 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1292 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1294 gcc_assert (ctx
->outer
);
1296 = splay_tree_lookup (ctx
->outer
->field_map
,
1297 (splay_tree_key
) &DECL_UID (var
));
1300 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1303 x
= lookup_decl (var
, ctx
->outer
);
1307 tree field
= (tree
) n
->value
;
1308 /* If the receiver record type was remapped in the child function,
1309 remap the field into the new record type. */
1310 x
= maybe_lookup_field (field
, ctx
->outer
);
1314 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1315 x
= omp_build_component_ref (x
, field
);
1316 if (use_pointer_for_field (var
, ctx
->outer
))
1317 x
= build_simple_mem_ref (x
);
1320 else if (ctx
->outer
)
1321 x
= lookup_decl (var
, ctx
->outer
);
1322 else if (is_reference (var
))
1323 /* This can happen with orphaned constructs. If var is reference, it is
1324 possible it is shared and as such valid. */
1326 else if (omp_member_access_dummy_var (var
))
1333 tree t
= omp_member_access_dummy_var (var
);
1336 x
= DECL_VALUE_EXPR (var
);
1337 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1339 x
= unshare_and_remap (x
, t
, o
);
1341 x
= unshare_expr (x
);
1345 if (is_reference (var
))
1346 x
= build_simple_mem_ref (x
);
1351 /* Build tree nodes to access the field for VAR on the sender side. */
1354 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1356 tree field
= lookup_sfield (key
, ctx
);
1357 return omp_build_component_ref (ctx
->sender_decl
, field
);
1361 build_sender_ref (tree var
, omp_context
*ctx
)
1363 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1366 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1369 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1371 tree field
, type
, sfield
= NULL_TREE
;
1372 splay_tree_key key
= (splay_tree_key
) var
;
1374 if ((mask
& 8) != 0)
1376 key
= (splay_tree_key
) &DECL_UID (var
);
1377 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1379 gcc_assert ((mask
& 1) == 0
1380 || !splay_tree_lookup (ctx
->field_map
, key
));
1381 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1382 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1383 gcc_assert ((mask
& 3) == 3
1384 || !is_gimple_omp_oacc (ctx
->stmt
));
1386 type
= TREE_TYPE (var
);
1389 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1390 type
= build_pointer_type (build_pointer_type (type
));
1393 type
= build_pointer_type (type
);
1394 else if ((mask
& 3) == 1 && is_reference (var
))
1395 type
= TREE_TYPE (type
);
1397 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1398 FIELD_DECL
, DECL_NAME (var
), type
);
1400 /* Remember what variable this field was created for. This does have a
1401 side effect of making dwarf2out ignore this member, so for helpful
1402 debugging we clear it later in delete_omp_context. */
1403 DECL_ABSTRACT_ORIGIN (field
) = var
;
1404 if (type
== TREE_TYPE (var
))
1406 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1407 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1408 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1411 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1413 if ((mask
& 3) == 3)
1415 insert_field_into_struct (ctx
->record_type
, field
);
1416 if (ctx
->srecord_type
)
1418 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1419 FIELD_DECL
, DECL_NAME (var
), type
);
1420 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1421 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1422 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1423 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1424 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1429 if (ctx
->srecord_type
== NULL_TREE
)
1433 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1434 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1435 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1437 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1438 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1439 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1440 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1441 splay_tree_insert (ctx
->sfield_map
,
1442 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1443 (splay_tree_value
) sfield
);
1447 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1448 : ctx
->srecord_type
, field
);
1452 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1453 if ((mask
& 2) && ctx
->sfield_map
)
1454 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1458 install_var_local (tree var
, omp_context
*ctx
)
1460 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1461 insert_decl_map (&ctx
->cb
, var
, new_var
);
1465 /* Adjust the replacement for DECL in CTX for the new context. This means
1466 copying the DECL_VALUE_EXPR, and fixing up the type. */
1469 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1471 tree new_decl
, size
;
1473 new_decl
= lookup_decl (decl
, ctx
);
1475 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1477 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1478 && DECL_HAS_VALUE_EXPR_P (decl
))
1480 tree ve
= DECL_VALUE_EXPR (decl
);
1481 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1482 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1483 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1486 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1488 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1489 if (size
== error_mark_node
)
1490 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1491 DECL_SIZE (new_decl
) = size
;
1493 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1494 if (size
== error_mark_node
)
1495 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1496 DECL_SIZE_UNIT (new_decl
) = size
;
1500 /* The callback for remap_decl. Search all containing contexts for a
1501 mapping of the variable; this avoids having to duplicate the splay
1502 tree ahead of time. We know a mapping doesn't already exist in the
1503 given context. Create new mappings to implement default semantics. */
1506 omp_copy_decl (tree var
, copy_body_data
*cb
)
1508 omp_context
*ctx
= (omp_context
*) cb
;
1511 if (TREE_CODE (var
) == LABEL_DECL
)
1513 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1514 DECL_CONTEXT (new_var
) = current_function_decl
;
1515 insert_decl_map (&ctx
->cb
, var
, new_var
);
1519 while (!is_taskreg_ctx (ctx
))
1524 new_var
= maybe_lookup_decl (var
, ctx
);
1529 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1532 return error_mark_node
;
1536 /* Debugging dumps for parallel regions. */
1537 void dump_omp_region (FILE *, struct omp_region
*, int);
1538 void debug_omp_region (struct omp_region
*);
1539 void debug_all_omp_regions (void);
1541 /* Dump the parallel region tree rooted at REGION. */
1544 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1546 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1547 gimple_code_name
[region
->type
]);
1550 dump_omp_region (file
, region
->inner
, indent
+ 4);
1554 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1555 region
->cont
->index
);
1559 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1560 region
->exit
->index
);
1562 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1565 dump_omp_region (file
, region
->next
, indent
);
1569 debug_omp_region (struct omp_region
*region
)
1571 dump_omp_region (stderr
, region
, 0);
1575 debug_all_omp_regions (void)
1577 dump_omp_region (stderr
, root_omp_region
, 0);
1581 /* Create a new parallel region starting at STMT inside region PARENT. */
1583 static struct omp_region
*
1584 new_omp_region (basic_block bb
, enum gimple_code type
,
1585 struct omp_region
*parent
)
1587 struct omp_region
*region
= XCNEW (struct omp_region
);
1589 region
->outer
= parent
;
1591 region
->type
= type
;
1595 /* This is a nested region. Add it to the list of inner
1596 regions in PARENT. */
1597 region
->next
= parent
->inner
;
1598 parent
->inner
= region
;
1602 /* This is a toplevel region. Add it to the list of toplevel
1603 regions in ROOT_OMP_REGION. */
1604 region
->next
= root_omp_region
;
1605 root_omp_region
= region
;
1611 /* Release the memory associated with the region tree rooted at REGION. */
1614 free_omp_region_1 (struct omp_region
*region
)
1616 struct omp_region
*i
, *n
;
1618 for (i
= region
->inner
; i
; i
= n
)
1621 free_omp_region_1 (i
);
1627 /* Release the memory for the entire omp region tree. */
1630 free_omp_regions (void)
1632 struct omp_region
*r
, *n
;
1633 for (r
= root_omp_region
; r
; r
= n
)
1636 free_omp_region_1 (r
);
1638 root_omp_region
= NULL
;
1642 /* Create a new context, with OUTER_CTX being the surrounding context. */
1644 static omp_context
*
1645 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1647 omp_context
*ctx
= XCNEW (omp_context
);
1649 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1650 (splay_tree_value
) ctx
);
1655 ctx
->outer
= outer_ctx
;
1656 ctx
->cb
= outer_ctx
->cb
;
1657 ctx
->cb
.block
= NULL
;
1658 ctx
->depth
= outer_ctx
->depth
+ 1;
1662 ctx
->cb
.src_fn
= current_function_decl
;
1663 ctx
->cb
.dst_fn
= current_function_decl
;
1664 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1665 gcc_checking_assert (ctx
->cb
.src_node
);
1666 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1667 ctx
->cb
.src_cfun
= cfun
;
1668 ctx
->cb
.copy_decl
= omp_copy_decl
;
1669 ctx
->cb
.eh_lp_nr
= 0;
1670 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1674 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1679 static gimple_seq
maybe_catch_exception (gimple_seq
);
1681 /* Finalize task copyfn. */
1684 finalize_task_copyfn (gomp_task
*task_stmt
)
1686 struct function
*child_cfun
;
1688 gimple_seq seq
= NULL
, new_seq
;
1691 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1692 if (child_fn
== NULL_TREE
)
1695 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1696 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1698 push_cfun (child_cfun
);
1699 bind
= gimplify_body (child_fn
, false);
1700 gimple_seq_add_stmt (&seq
, bind
);
1701 new_seq
= maybe_catch_exception (seq
);
1704 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1706 gimple_seq_add_stmt (&seq
, bind
);
1708 gimple_set_body (child_fn
, seq
);
1711 /* Inform the callgraph about the new function. */
1712 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1713 node
->parallelized_function
= 1;
1714 cgraph_node::add_new_function (child_fn
, false);
1717 /* Destroy a omp_context data structures. Called through the splay tree
1718 value delete callback. */
1721 delete_omp_context (splay_tree_value value
)
1723 omp_context
*ctx
= (omp_context
*) value
;
1725 delete ctx
->cb
.decl_map
;
1728 splay_tree_delete (ctx
->field_map
);
1729 if (ctx
->sfield_map
)
1730 splay_tree_delete (ctx
->sfield_map
);
1732 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1733 it produces corrupt debug information. */
1734 if (ctx
->record_type
)
1737 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1738 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1740 if (ctx
->srecord_type
)
1743 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1744 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1747 if (is_task_ctx (ctx
))
1748 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1753 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1757 fixup_child_record_type (omp_context
*ctx
)
1759 tree f
, type
= ctx
->record_type
;
1761 /* ??? It isn't sufficient to just call remap_type here, because
1762 variably_modified_type_p doesn't work the way we expect for
1763 record types. Testing each field for whether it needs remapping
1764 and creating a new record by hand works, however. */
1765 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1766 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1770 tree name
, new_fields
= NULL
;
1772 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1773 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1774 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1775 TYPE_DECL
, name
, type
);
1776 TYPE_NAME (type
) = name
;
1778 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1780 tree new_f
= copy_node (f
);
1781 DECL_CONTEXT (new_f
) = type
;
1782 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1783 DECL_CHAIN (new_f
) = new_fields
;
1784 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1785 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1787 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1791 /* Arrange to be able to look up the receiver field
1792 given the sender field. */
1793 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1794 (splay_tree_value
) new_f
);
1796 TYPE_FIELDS (type
) = nreverse (new_fields
);
1800 /* In a target region we never modify any of the pointers in *.omp_data_i,
1801 so attempt to help the optimizers. */
1802 if (is_gimple_omp_offloaded (ctx
->stmt
))
1803 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1805 TREE_TYPE (ctx
->receiver_decl
)
1806 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1809 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1810 specified by CLAUSES. */
1813 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1816 bool scan_array_reductions
= false;
1818 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1822 switch (OMP_CLAUSE_CODE (c
))
1824 case OMP_CLAUSE_PRIVATE
:
1825 decl
= OMP_CLAUSE_DECL (c
);
1826 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1828 else if (!is_variable_sized (decl
))
1829 install_var_local (decl
, ctx
);
1832 case OMP_CLAUSE_SHARED
:
1833 decl
= OMP_CLAUSE_DECL (c
);
1834 /* Ignore shared directives in teams construct. */
1835 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1837 /* Global variables don't need to be copied,
1838 the receiver side will use them directly. */
1839 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1840 if (is_global_var (odecl
))
1842 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1845 gcc_assert (is_taskreg_ctx (ctx
));
1846 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1847 || !is_variable_sized (decl
));
1848 /* Global variables don't need to be copied,
1849 the receiver side will use them directly. */
1850 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1852 by_ref
= use_pointer_for_field (decl
, ctx
);
1853 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1855 if (! TREE_READONLY (decl
)
1856 || TREE_ADDRESSABLE (decl
)
1858 || is_reference (decl
))
1860 install_var_field (decl
, by_ref
, 3, ctx
);
1861 install_var_local (decl
, ctx
);
1864 /* We don't need to copy const scalar vars back. */
1865 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1868 case OMP_CLAUSE_REDUCTION
:
1869 decl
= OMP_CLAUSE_DECL (c
);
1870 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1871 && TREE_CODE (decl
) == MEM_REF
)
1873 tree t
= TREE_OPERAND (decl
, 0);
1874 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1875 t
= TREE_OPERAND (t
, 0);
1876 if (TREE_CODE (t
) == INDIRECT_REF
1877 || TREE_CODE (t
) == ADDR_EXPR
)
1878 t
= TREE_OPERAND (t
, 0);
1879 install_var_local (t
, ctx
);
1880 if (is_taskreg_ctx (ctx
)
1881 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1882 && !is_variable_sized (t
))
1884 by_ref
= use_pointer_for_field (t
, ctx
);
1885 install_var_field (t
, by_ref
, 3, ctx
);
1891 case OMP_CLAUSE_LASTPRIVATE
:
1892 /* Let the corresponding firstprivate clause create
1894 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1898 case OMP_CLAUSE_FIRSTPRIVATE
:
1899 case OMP_CLAUSE_LINEAR
:
1900 decl
= OMP_CLAUSE_DECL (c
);
1902 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1903 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1904 && is_gimple_omp_offloaded (ctx
->stmt
))
1906 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1907 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1908 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1909 install_var_field (decl
, true, 3, ctx
);
1911 install_var_field (decl
, false, 3, ctx
);
1913 if (is_variable_sized (decl
))
1915 if (is_task_ctx (ctx
))
1916 install_var_field (decl
, false, 1, ctx
);
1919 else if (is_taskreg_ctx (ctx
))
1922 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1923 by_ref
= use_pointer_for_field (decl
, NULL
);
1925 if (is_task_ctx (ctx
)
1926 && (global
|| by_ref
|| is_reference (decl
)))
1928 install_var_field (decl
, false, 1, ctx
);
1930 install_var_field (decl
, by_ref
, 2, ctx
);
1933 install_var_field (decl
, by_ref
, 3, ctx
);
1935 install_var_local (decl
, ctx
);
1938 case OMP_CLAUSE_USE_DEVICE_PTR
:
1939 decl
= OMP_CLAUSE_DECL (c
);
1940 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1941 install_var_field (decl
, true, 3, ctx
);
1943 install_var_field (decl
, false, 3, ctx
);
1944 if (DECL_SIZE (decl
)
1945 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1947 tree decl2
= DECL_VALUE_EXPR (decl
);
1948 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1949 decl2
= TREE_OPERAND (decl2
, 0);
1950 gcc_assert (DECL_P (decl2
));
1951 install_var_local (decl2
, ctx
);
1953 install_var_local (decl
, ctx
);
1956 case OMP_CLAUSE_IS_DEVICE_PTR
:
1957 decl
= OMP_CLAUSE_DECL (c
);
1960 case OMP_CLAUSE__LOOPTEMP_
:
1961 gcc_assert (is_taskreg_ctx (ctx
));
1962 decl
= OMP_CLAUSE_DECL (c
);
1963 install_var_field (decl
, false, 3, ctx
);
1964 install_var_local (decl
, ctx
);
1967 case OMP_CLAUSE_COPYPRIVATE
:
1968 case OMP_CLAUSE_COPYIN
:
1969 decl
= OMP_CLAUSE_DECL (c
);
1970 by_ref
= use_pointer_for_field (decl
, NULL
);
1971 install_var_field (decl
, by_ref
, 3, ctx
);
1974 case OMP_CLAUSE_DEFAULT
:
1975 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1978 case OMP_CLAUSE_FINAL
:
1980 case OMP_CLAUSE_NUM_THREADS
:
1981 case OMP_CLAUSE_NUM_TEAMS
:
1982 case OMP_CLAUSE_THREAD_LIMIT
:
1983 case OMP_CLAUSE_DEVICE
:
1984 case OMP_CLAUSE_SCHEDULE
:
1985 case OMP_CLAUSE_DIST_SCHEDULE
:
1986 case OMP_CLAUSE_DEPEND
:
1987 case OMP_CLAUSE_PRIORITY
:
1988 case OMP_CLAUSE_GRAINSIZE
:
1989 case OMP_CLAUSE_NUM_TASKS
:
1990 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1991 case OMP_CLAUSE_NUM_GANGS
:
1992 case OMP_CLAUSE_NUM_WORKERS
:
1993 case OMP_CLAUSE_VECTOR_LENGTH
:
1995 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1999 case OMP_CLAUSE_FROM
:
2000 case OMP_CLAUSE_MAP
:
2002 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2003 decl
= OMP_CLAUSE_DECL (c
);
2004 /* Global variables with "omp declare target" attribute
2005 don't need to be copied, the receiver side will use them
2007 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2009 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2010 && (OMP_CLAUSE_MAP_KIND (c
)
2011 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2012 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2013 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2014 && varpool_node::get_create (decl
)->offloadable
)
2016 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2017 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2019 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2020 not offloaded; there is nothing to map for those. */
2021 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2022 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2023 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2026 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2027 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2028 || (OMP_CLAUSE_MAP_KIND (c
)
2029 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2031 if (TREE_CODE (decl
) == COMPONENT_REF
2032 || (TREE_CODE (decl
) == INDIRECT_REF
2033 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2034 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2035 == REFERENCE_TYPE
)))
2037 if (DECL_SIZE (decl
)
2038 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2040 tree decl2
= DECL_VALUE_EXPR (decl
);
2041 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2042 decl2
= TREE_OPERAND (decl2
, 0);
2043 gcc_assert (DECL_P (decl2
));
2044 install_var_local (decl2
, ctx
);
2046 install_var_local (decl
, ctx
);
2051 if (DECL_SIZE (decl
)
2052 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2054 tree decl2
= DECL_VALUE_EXPR (decl
);
2055 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2056 decl2
= TREE_OPERAND (decl2
, 0);
2057 gcc_assert (DECL_P (decl2
));
2058 install_var_field (decl2
, true, 3, ctx
);
2059 install_var_local (decl2
, ctx
);
2060 install_var_local (decl
, ctx
);
2064 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2065 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2066 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2067 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2068 install_var_field (decl
, true, 7, ctx
);
2070 install_var_field (decl
, true, 3, ctx
);
2071 if (is_gimple_omp_offloaded (ctx
->stmt
))
2072 install_var_local (decl
, ctx
);
2077 tree base
= get_base_address (decl
);
2078 tree nc
= OMP_CLAUSE_CHAIN (c
);
2081 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2082 && OMP_CLAUSE_DECL (nc
) == base
2083 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2084 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2086 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2087 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2093 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2094 decl
= OMP_CLAUSE_DECL (c
);
2096 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2097 (splay_tree_key
) decl
));
2099 = build_decl (OMP_CLAUSE_LOCATION (c
),
2100 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2101 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2102 insert_field_into_struct (ctx
->record_type
, field
);
2103 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2104 (splay_tree_value
) field
);
2109 case OMP_CLAUSE_NOWAIT
:
2110 case OMP_CLAUSE_ORDERED
:
2111 case OMP_CLAUSE_COLLAPSE
:
2112 case OMP_CLAUSE_UNTIED
:
2113 case OMP_CLAUSE_MERGEABLE
:
2114 case OMP_CLAUSE_PROC_BIND
:
2115 case OMP_CLAUSE_SAFELEN
:
2116 case OMP_CLAUSE_SIMDLEN
:
2117 case OMP_CLAUSE_THREADS
:
2118 case OMP_CLAUSE_SIMD
:
2119 case OMP_CLAUSE_NOGROUP
:
2120 case OMP_CLAUSE_DEFAULTMAP
:
2121 case OMP_CLAUSE_ASYNC
:
2122 case OMP_CLAUSE_WAIT
:
2123 case OMP_CLAUSE_GANG
:
2124 case OMP_CLAUSE_WORKER
:
2125 case OMP_CLAUSE_VECTOR
:
2126 case OMP_CLAUSE_TILE
:
2127 case OMP_CLAUSE_INDEPENDENT
:
2128 case OMP_CLAUSE_AUTO
:
2129 case OMP_CLAUSE_SEQ
:
2132 case OMP_CLAUSE_ALIGNED
:
2133 decl
= OMP_CLAUSE_DECL (c
);
2134 if (is_global_var (decl
)
2135 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2136 install_var_local (decl
, ctx
);
2139 case OMP_CLAUSE_DEVICE_RESIDENT
:
2140 case OMP_CLAUSE_USE_DEVICE
:
2141 case OMP_CLAUSE__CACHE_
:
2142 sorry ("Clause not supported yet");
2150 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2152 switch (OMP_CLAUSE_CODE (c
))
2154 case OMP_CLAUSE_LASTPRIVATE
:
2155 /* Let the corresponding firstprivate clause create
2157 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2158 scan_array_reductions
= true;
2159 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2163 case OMP_CLAUSE_FIRSTPRIVATE
:
2164 case OMP_CLAUSE_PRIVATE
:
2165 case OMP_CLAUSE_LINEAR
:
2166 case OMP_CLAUSE_IS_DEVICE_PTR
:
2167 decl
= OMP_CLAUSE_DECL (c
);
2168 if (is_variable_sized (decl
))
2170 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2171 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2172 && is_gimple_omp_offloaded (ctx
->stmt
))
2174 tree decl2
= DECL_VALUE_EXPR (decl
);
2175 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2176 decl2
= TREE_OPERAND (decl2
, 0);
2177 gcc_assert (DECL_P (decl2
));
2178 install_var_local (decl2
, ctx
);
2179 fixup_remapped_decl (decl2
, ctx
, false);
2181 install_var_local (decl
, ctx
);
2183 fixup_remapped_decl (decl
, ctx
,
2184 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2185 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2186 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2187 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2188 scan_array_reductions
= true;
2191 case OMP_CLAUSE_REDUCTION
:
2192 decl
= OMP_CLAUSE_DECL (c
);
2193 if (TREE_CODE (decl
) != MEM_REF
)
2195 if (is_variable_sized (decl
))
2196 install_var_local (decl
, ctx
);
2197 fixup_remapped_decl (decl
, ctx
, false);
2199 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2200 scan_array_reductions
= true;
2203 case OMP_CLAUSE_SHARED
:
2204 /* Ignore shared directives in teams construct. */
2205 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2207 decl
= OMP_CLAUSE_DECL (c
);
2208 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2210 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2212 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2215 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2216 install_var_field (decl
, by_ref
, 11, ctx
);
2219 fixup_remapped_decl (decl
, ctx
, false);
2222 case OMP_CLAUSE_MAP
:
2223 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2225 decl
= OMP_CLAUSE_DECL (c
);
2227 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2228 && (OMP_CLAUSE_MAP_KIND (c
)
2229 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2230 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2231 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2232 && varpool_node::get_create (decl
)->offloadable
)
2236 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2237 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2238 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2239 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2241 tree new_decl
= lookup_decl (decl
, ctx
);
2242 TREE_TYPE (new_decl
)
2243 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2245 else if (DECL_SIZE (decl
)
2246 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2248 tree decl2
= DECL_VALUE_EXPR (decl
);
2249 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2250 decl2
= TREE_OPERAND (decl2
, 0);
2251 gcc_assert (DECL_P (decl2
));
2252 fixup_remapped_decl (decl2
, ctx
, false);
2253 fixup_remapped_decl (decl
, ctx
, true);
2256 fixup_remapped_decl (decl
, ctx
, false);
2260 case OMP_CLAUSE_COPYPRIVATE
:
2261 case OMP_CLAUSE_COPYIN
:
2262 case OMP_CLAUSE_DEFAULT
:
2264 case OMP_CLAUSE_NUM_THREADS
:
2265 case OMP_CLAUSE_NUM_TEAMS
:
2266 case OMP_CLAUSE_THREAD_LIMIT
:
2267 case OMP_CLAUSE_DEVICE
:
2268 case OMP_CLAUSE_SCHEDULE
:
2269 case OMP_CLAUSE_DIST_SCHEDULE
:
2270 case OMP_CLAUSE_NOWAIT
:
2271 case OMP_CLAUSE_ORDERED
:
2272 case OMP_CLAUSE_COLLAPSE
:
2273 case OMP_CLAUSE_UNTIED
:
2274 case OMP_CLAUSE_FINAL
:
2275 case OMP_CLAUSE_MERGEABLE
:
2276 case OMP_CLAUSE_PROC_BIND
:
2277 case OMP_CLAUSE_SAFELEN
:
2278 case OMP_CLAUSE_SIMDLEN
:
2279 case OMP_CLAUSE_ALIGNED
:
2280 case OMP_CLAUSE_DEPEND
:
2281 case OMP_CLAUSE__LOOPTEMP_
:
2283 case OMP_CLAUSE_FROM
:
2284 case OMP_CLAUSE_PRIORITY
:
2285 case OMP_CLAUSE_GRAINSIZE
:
2286 case OMP_CLAUSE_NUM_TASKS
:
2287 case OMP_CLAUSE_THREADS
:
2288 case OMP_CLAUSE_SIMD
:
2289 case OMP_CLAUSE_NOGROUP
:
2290 case OMP_CLAUSE_DEFAULTMAP
:
2291 case OMP_CLAUSE_USE_DEVICE_PTR
:
2292 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2293 case OMP_CLAUSE_ASYNC
:
2294 case OMP_CLAUSE_WAIT
:
2295 case OMP_CLAUSE_NUM_GANGS
:
2296 case OMP_CLAUSE_NUM_WORKERS
:
2297 case OMP_CLAUSE_VECTOR_LENGTH
:
2298 case OMP_CLAUSE_GANG
:
2299 case OMP_CLAUSE_WORKER
:
2300 case OMP_CLAUSE_VECTOR
:
2301 case OMP_CLAUSE_TILE
:
2302 case OMP_CLAUSE_INDEPENDENT
:
2303 case OMP_CLAUSE_AUTO
:
2304 case OMP_CLAUSE_SEQ
:
2307 case OMP_CLAUSE_DEVICE_RESIDENT
:
2308 case OMP_CLAUSE_USE_DEVICE
:
2309 case OMP_CLAUSE__CACHE_
:
2310 sorry ("Clause not supported yet");
2318 gcc_checking_assert (!scan_array_reductions
2319 || !is_gimple_omp_oacc (ctx
->stmt
));
2320 if (scan_array_reductions
)
2321 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2322 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2323 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2325 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2326 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2328 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2329 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2330 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2331 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2332 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2333 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2336 /* Create a new name for omp child function. Returns an identifier. If
2337 IS_CILK_FOR is true then the suffix for the child function is
2341 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2344 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2345 return clone_function_name (current_function_decl
,
2346 task_copy
? "_omp_cpyfn" : "_omp_fn");
2349 /* Returns the type of the induction variable for the child function for
2350 _Cilk_for and the types for _high and _low variables based on TYPE. */
2353 cilk_for_check_loop_diff_type (tree type
)
2355 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2357 if (TYPE_UNSIGNED (type
))
2358 return uint32_type_node
;
2360 return integer_type_node
;
2364 if (TYPE_UNSIGNED (type
))
2365 return uint64_type_node
;
2367 return long_long_integer_type_node
;
2371 /* Build a decl for the omp child function. It'll not contain a body
2372 yet, just the bare decl. */
2375 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2377 tree decl
, type
, name
, t
;
2380 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2381 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2382 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2383 tree cilk_var_type
= NULL_TREE
;
2385 name
= create_omp_child_function_name (task_copy
,
2386 cilk_for_count
!= NULL_TREE
);
2388 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2389 ptr_type_node
, NULL_TREE
);
2390 else if (cilk_for_count
)
2392 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2393 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2394 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2395 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2398 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2400 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2402 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2405 ctx
->cb
.dst_fn
= decl
;
2407 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2409 TREE_STATIC (decl
) = 1;
2410 TREE_USED (decl
) = 1;
2411 DECL_ARTIFICIAL (decl
) = 1;
2412 DECL_IGNORED_P (decl
) = 0;
2413 TREE_PUBLIC (decl
) = 0;
2414 DECL_UNINLINABLE (decl
) = 1;
2415 DECL_EXTERNAL (decl
) = 0;
2416 DECL_CONTEXT (decl
) = NULL_TREE
;
2417 DECL_INITIAL (decl
) = make_node (BLOCK
);
2418 if (cgraph_node::get (current_function_decl
)->offloadable
)
2419 cgraph_node::get_create (decl
)->offloadable
= 1;
2423 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2424 if (is_gimple_omp_offloaded (octx
->stmt
))
2426 cgraph_node::get_create (decl
)->offloadable
= 1;
2427 #ifdef ENABLE_OFFLOADING
2428 g
->have_offload
= true;
2434 if (cgraph_node::get_create (decl
)->offloadable
2435 && !lookup_attribute ("omp declare target",
2436 DECL_ATTRIBUTES (current_function_decl
)))
2437 DECL_ATTRIBUTES (decl
)
2438 = tree_cons (get_identifier ("omp target entrypoint"),
2439 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2441 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2442 RESULT_DECL
, NULL_TREE
, void_type_node
);
2443 DECL_ARTIFICIAL (t
) = 1;
2444 DECL_IGNORED_P (t
) = 1;
2445 DECL_CONTEXT (t
) = decl
;
2446 DECL_RESULT (decl
) = t
;
2448 /* _Cilk_for's child function requires two extra parameters called
2449 __low and __high that are set the by Cilk runtime when it calls this
2453 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2454 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2455 DECL_ARTIFICIAL (t
) = 1;
2456 DECL_NAMELESS (t
) = 1;
2457 DECL_ARG_TYPE (t
) = ptr_type_node
;
2458 DECL_CONTEXT (t
) = current_function_decl
;
2460 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2461 DECL_ARGUMENTS (decl
) = t
;
2463 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2464 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2465 DECL_ARTIFICIAL (t
) = 1;
2466 DECL_NAMELESS (t
) = 1;
2467 DECL_ARG_TYPE (t
) = ptr_type_node
;
2468 DECL_CONTEXT (t
) = current_function_decl
;
2470 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2471 DECL_ARGUMENTS (decl
) = t
;
2474 tree data_name
= get_identifier (".omp_data_i");
2475 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2477 DECL_ARTIFICIAL (t
) = 1;
2478 DECL_NAMELESS (t
) = 1;
2479 DECL_ARG_TYPE (t
) = ptr_type_node
;
2480 DECL_CONTEXT (t
) = current_function_decl
;
2482 TREE_READONLY (t
) = 1;
2484 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2485 DECL_ARGUMENTS (decl
) = t
;
2487 ctx
->receiver_decl
= t
;
2490 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2491 PARM_DECL
, get_identifier (".omp_data_o"),
2493 DECL_ARTIFICIAL (t
) = 1;
2494 DECL_NAMELESS (t
) = 1;
2495 DECL_ARG_TYPE (t
) = ptr_type_node
;
2496 DECL_CONTEXT (t
) = current_function_decl
;
2498 TREE_ADDRESSABLE (t
) = 1;
2499 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2500 DECL_ARGUMENTS (decl
) = t
;
2503 /* Allocate memory for the function structure. The call to
2504 allocate_struct_function clobbers CFUN, so we need to restore
2506 push_struct_function (decl
);
2507 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2511 /* Callback for walk_gimple_seq. Check if combined parallel
2512 contains gimple_omp_for_combined_into_p OMP_FOR. */
2515 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2516 bool *handled_ops_p
,
2517 struct walk_stmt_info
*wi
)
2519 gimple
*stmt
= gsi_stmt (*gsi_p
);
2521 *handled_ops_p
= true;
2522 switch (gimple_code (stmt
))
2526 case GIMPLE_OMP_FOR
:
2527 if (gimple_omp_for_combined_into_p (stmt
)
2528 && gimple_omp_for_kind (stmt
)
2529 == *(const enum gf_mask
*) (wi
->info
))
2532 return integer_zero_node
;
2541 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2544 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2545 omp_context
*outer_ctx
)
2547 struct walk_stmt_info wi
;
2549 memset (&wi
, 0, sizeof (wi
));
2551 wi
.info
= (void *) &msk
;
2552 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2553 if (wi
.info
!= (void *) &msk
)
2555 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2556 struct omp_for_data fd
;
2557 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2558 /* We need two temporaries with fd.loop.v type (istart/iend)
2559 and then (fd.collapse - 1) temporaries with the same
2560 type for count2 ... countN-1 vars if not constant. */
2561 size_t count
= 2, i
;
2562 tree type
= fd
.iter_type
;
2564 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2566 count
+= fd
.collapse
- 1;
2567 /* If there are lastprivate clauses on the inner
2568 GIMPLE_OMP_FOR, add one more temporaries for the total number
2569 of iterations (product of count1 ... countN-1). */
2570 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2571 OMP_CLAUSE_LASTPRIVATE
))
2573 else if (msk
== GF_OMP_FOR_KIND_FOR
2574 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2575 OMP_CLAUSE_LASTPRIVATE
))
2578 for (i
= 0; i
< count
; i
++)
2580 tree temp
= create_tmp_var (type
);
2581 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2582 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2583 OMP_CLAUSE_DECL (c
) = temp
;
2584 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2585 gimple_omp_taskreg_set_clauses (stmt
, c
);
2590 /* Scan an OpenMP parallel directive. */
2593 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2597 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2599 /* Ignore parallel directives with empty bodies, unless there
2600 are copyin clauses. */
2602 && empty_body_p (gimple_omp_body (stmt
))
2603 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2604 OMP_CLAUSE_COPYIN
) == NULL
)
2606 gsi_replace (gsi
, gimple_build_nop (), false);
2610 if (gimple_omp_parallel_combined_p (stmt
))
2611 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2613 ctx
= new_omp_context (stmt
, outer_ctx
);
2614 taskreg_contexts
.safe_push (ctx
);
2615 if (taskreg_nesting_level
> 1)
2616 ctx
->is_nested
= true;
2617 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2618 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2619 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2620 name
= create_tmp_var_name (".omp_data_s");
2621 name
= build_decl (gimple_location (stmt
),
2622 TYPE_DECL
, name
, ctx
->record_type
);
2623 DECL_ARTIFICIAL (name
) = 1;
2624 DECL_NAMELESS (name
) = 1;
2625 TYPE_NAME (ctx
->record_type
) = name
;
2626 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2627 create_omp_child_function (ctx
, false);
2628 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2630 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2631 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2633 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2634 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2637 /* Scan an OpenMP task directive. */
2640 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2644 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2646 /* Ignore task directives with empty bodies. */
2648 && empty_body_p (gimple_omp_body (stmt
)))
2650 gsi_replace (gsi
, gimple_build_nop (), false);
2654 if (gimple_omp_task_taskloop_p (stmt
))
2655 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2657 ctx
= new_omp_context (stmt
, outer_ctx
);
2658 taskreg_contexts
.safe_push (ctx
);
2659 if (taskreg_nesting_level
> 1)
2660 ctx
->is_nested
= true;
2661 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2662 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2663 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2664 name
= create_tmp_var_name (".omp_data_s");
2665 name
= build_decl (gimple_location (stmt
),
2666 TYPE_DECL
, name
, ctx
->record_type
);
2667 DECL_ARTIFICIAL (name
) = 1;
2668 DECL_NAMELESS (name
) = 1;
2669 TYPE_NAME (ctx
->record_type
) = name
;
2670 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2671 create_omp_child_function (ctx
, false);
2672 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2674 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2676 if (ctx
->srecord_type
)
2678 name
= create_tmp_var_name (".omp_data_a");
2679 name
= build_decl (gimple_location (stmt
),
2680 TYPE_DECL
, name
, ctx
->srecord_type
);
2681 DECL_ARTIFICIAL (name
) = 1;
2682 DECL_NAMELESS (name
) = 1;
2683 TYPE_NAME (ctx
->srecord_type
) = name
;
2684 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2685 create_omp_child_function (ctx
, true);
2688 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2690 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2692 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2693 t
= build_int_cst (long_integer_type_node
, 0);
2694 gimple_omp_task_set_arg_size (stmt
, t
);
2695 t
= build_int_cst (long_integer_type_node
, 1);
2696 gimple_omp_task_set_arg_align (stmt
, t
);
2701 /* If any decls have been made addressable during scan_omp,
2702 adjust their fields if needed, and layout record types
2703 of parallel/task constructs. */
2706 finish_taskreg_scan (omp_context
*ctx
)
2708 if (ctx
->record_type
== NULL_TREE
)
2711 /* If any task_shared_vars were needed, verify all
2712 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2713 statements if use_pointer_for_field hasn't changed
2714 because of that. If it did, update field types now. */
2715 if (task_shared_vars
)
2719 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2720 c
; c
= OMP_CLAUSE_CHAIN (c
))
2721 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2722 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2724 tree decl
= OMP_CLAUSE_DECL (c
);
2726 /* Global variables don't need to be copied,
2727 the receiver side will use them directly. */
2728 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2730 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2731 || !use_pointer_for_field (decl
, ctx
))
2733 tree field
= lookup_field (decl
, ctx
);
2734 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2735 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2737 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2738 TREE_THIS_VOLATILE (field
) = 0;
2739 DECL_USER_ALIGN (field
) = 0;
2740 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2741 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2742 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2743 if (ctx
->srecord_type
)
2745 tree sfield
= lookup_sfield (decl
, ctx
);
2746 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2747 TREE_THIS_VOLATILE (sfield
) = 0;
2748 DECL_USER_ALIGN (sfield
) = 0;
2749 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2750 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2751 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2756 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2758 layout_type (ctx
->record_type
);
2759 fixup_child_record_type (ctx
);
2763 location_t loc
= gimple_location (ctx
->stmt
);
2764 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2765 /* Move VLA fields to the end. */
2766 p
= &TYPE_FIELDS (ctx
->record_type
);
2768 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2769 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2772 *p
= TREE_CHAIN (*p
);
2773 TREE_CHAIN (*q
) = NULL_TREE
;
2774 q
= &TREE_CHAIN (*q
);
2777 p
= &DECL_CHAIN (*p
);
2779 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2781 /* Move fields corresponding to first and second _looptemp_
2782 clause first. There are filled by GOMP_taskloop
2783 and thus need to be in specific positions. */
2784 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2785 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2786 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2787 OMP_CLAUSE__LOOPTEMP_
);
2788 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2789 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2790 p
= &TYPE_FIELDS (ctx
->record_type
);
2792 if (*p
== f1
|| *p
== f2
)
2793 *p
= DECL_CHAIN (*p
);
2795 p
= &DECL_CHAIN (*p
);
2796 DECL_CHAIN (f1
) = f2
;
2797 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2798 TYPE_FIELDS (ctx
->record_type
) = f1
;
2799 if (ctx
->srecord_type
)
2801 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2802 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2803 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2805 if (*p
== f1
|| *p
== f2
)
2806 *p
= DECL_CHAIN (*p
);
2808 p
= &DECL_CHAIN (*p
);
2809 DECL_CHAIN (f1
) = f2
;
2810 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2811 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2814 layout_type (ctx
->record_type
);
2815 fixup_child_record_type (ctx
);
2816 if (ctx
->srecord_type
)
2817 layout_type (ctx
->srecord_type
);
2818 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2819 TYPE_SIZE_UNIT (ctx
->record_type
));
2820 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2821 t
= build_int_cst (long_integer_type_node
,
2822 TYPE_ALIGN_UNIT (ctx
->record_type
));
2823 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2827 /* Find the enclosing offload context. */
2829 static omp_context
*
2830 enclosing_target_ctx (omp_context
*ctx
)
2832 for (; ctx
; ctx
= ctx
->outer
)
2833 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2839 /* Return true if ctx is part of an oacc kernels region. */
2842 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2844 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2846 gimple
*stmt
= ctx
->stmt
;
2847 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2848 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2855 /* Check the parallelism clauses inside a kernels regions.
2856 Until kernels handling moves to use the same loop indirection
2857 scheme as parallel, we need to do this checking early. */
2860 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2862 bool checking
= true;
2863 unsigned outer_mask
= 0;
2864 unsigned this_mask
= 0;
2865 bool has_seq
= false, has_auto
= false;
2868 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2872 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2874 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2877 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2879 switch (OMP_CLAUSE_CODE (c
))
2881 case OMP_CLAUSE_GANG
:
2882 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2884 case OMP_CLAUSE_WORKER
:
2885 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2887 case OMP_CLAUSE_VECTOR
:
2888 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2890 case OMP_CLAUSE_SEQ
:
2893 case OMP_CLAUSE_AUTO
:
2903 if (has_seq
&& (this_mask
|| has_auto
))
2904 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2905 " OpenACC loop specifiers");
2906 else if (has_auto
&& this_mask
)
2907 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2908 " OpenACC loop specifiers");
2910 if (this_mask
& outer_mask
)
2911 error_at (gimple_location (stmt
), "inner loop uses same"
2912 " OpenACC parallelism as containing loop");
2915 return outer_mask
| this_mask
;
2918 /* Scan a GIMPLE_OMP_FOR. */
2921 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2925 tree clauses
= gimple_omp_for_clauses (stmt
);
2927 ctx
= new_omp_context (stmt
, outer_ctx
);
2929 if (is_gimple_omp_oacc (stmt
))
2931 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2933 if (!tgt
|| is_oacc_parallel (tgt
))
2934 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2936 char const *check
= NULL
;
2938 switch (OMP_CLAUSE_CODE (c
))
2940 case OMP_CLAUSE_GANG
:
2944 case OMP_CLAUSE_WORKER
:
2948 case OMP_CLAUSE_VECTOR
:
2956 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
2957 error_at (gimple_location (stmt
),
2958 "argument not permitted on %qs clause in"
2959 " OpenACC %<parallel%>", check
);
2962 if (tgt
&& is_oacc_kernels (tgt
))
2964 /* Strip out reductions, as they are not handled yet. */
2965 tree
*prev_ptr
= &clauses
;
2967 while (tree probe
= *prev_ptr
)
2969 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
2971 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
2972 *prev_ptr
= *next_ptr
;
2974 prev_ptr
= next_ptr
;
2977 gimple_omp_for_set_clauses (stmt
, clauses
);
2978 check_oacc_kernel_gwv (stmt
, ctx
);
2982 scan_sharing_clauses (clauses
, ctx
);
2984 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2985 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2987 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2988 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2989 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2990 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2992 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2995 /* Scan an OpenMP sections directive. */
2998 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3002 ctx
= new_omp_context (stmt
, outer_ctx
);
3003 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3004 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3007 /* Scan an OpenMP single directive. */
3010 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3015 ctx
= new_omp_context (stmt
, outer_ctx
);
3016 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3017 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3018 name
= create_tmp_var_name (".omp_copy_s");
3019 name
= build_decl (gimple_location (stmt
),
3020 TYPE_DECL
, name
, ctx
->record_type
);
3021 TYPE_NAME (ctx
->record_type
) = name
;
3023 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3024 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3026 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3027 ctx
->record_type
= NULL
;
3029 layout_type (ctx
->record_type
);
3032 /* Scan a GIMPLE_OMP_TARGET. */
3035 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3039 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3040 tree clauses
= gimple_omp_target_clauses (stmt
);
3042 ctx
= new_omp_context (stmt
, outer_ctx
);
3043 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3044 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3045 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3046 name
= create_tmp_var_name (".omp_data_t");
3047 name
= build_decl (gimple_location (stmt
),
3048 TYPE_DECL
, name
, ctx
->record_type
);
3049 DECL_ARTIFICIAL (name
) = 1;
3050 DECL_NAMELESS (name
) = 1;
3051 TYPE_NAME (ctx
->record_type
) = name
;
3052 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3055 create_omp_child_function (ctx
, false);
3056 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3059 scan_sharing_clauses (clauses
, ctx
);
3060 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3062 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3063 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3066 TYPE_FIELDS (ctx
->record_type
)
3067 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3070 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3071 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3073 field
= DECL_CHAIN (field
))
3074 gcc_assert (DECL_ALIGN (field
) == align
);
3076 layout_type (ctx
->record_type
);
3078 fixup_child_record_type (ctx
);
3082 /* Scan an OpenMP teams directive. */
3085 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3087 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3088 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3089 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3092 /* Check nesting restrictions. */
3094 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3098 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3099 inside an OpenACC CTX. */
3100 if (!(is_gimple_omp (stmt
)
3101 && is_gimple_omp_oacc (stmt
)))
3103 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3104 if (is_gimple_omp (octx
->stmt
)
3105 && is_gimple_omp_oacc (octx
->stmt
)
3106 /* Except for atomic codes that we share with OpenMP. */
3107 && ! (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3108 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3110 error_at (gimple_location (stmt
),
3111 "non-OpenACC construct inside of OpenACC region");
3118 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3119 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3122 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3124 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3125 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3127 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3128 && (ctx
->outer
== NULL
3129 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3130 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3131 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3132 != GF_OMP_FOR_KIND_FOR
)
3133 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3135 error_at (gimple_location (stmt
),
3136 "%<ordered simd threads%> must be closely "
3137 "nested inside of %<for simd%> region");
3143 error_at (gimple_location (stmt
),
3144 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3145 " may not be nested inside %<simd%> region");
3148 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3150 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3151 || (gimple_omp_for_kind (stmt
)
3152 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3153 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3155 error_at (gimple_location (stmt
),
3156 "only %<distribute%> or %<parallel%> regions are "
3157 "allowed to be strictly nested inside %<teams%> "
3163 switch (gimple_code (stmt
))
3165 case GIMPLE_OMP_FOR
:
3166 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3168 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3170 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3172 error_at (gimple_location (stmt
),
3173 "%<distribute%> region must be strictly nested "
3174 "inside %<teams%> construct");
3179 /* We split taskloop into task and nested taskloop in it. */
3180 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3182 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3187 switch (gimple_code (ctx
->stmt
))
3189 case GIMPLE_OMP_FOR
:
3190 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3191 == GF_OMP_FOR_KIND_OACC_LOOP
);
3194 case GIMPLE_OMP_TARGET
:
3195 switch (gimple_omp_target_kind (ctx
->stmt
))
3197 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3198 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3209 else if (get_oacc_fn_attrib (current_function_decl
))
3213 error_at (gimple_location (stmt
),
3214 "OpenACC loop directive must be associated with"
3215 " an OpenACC compute region");
3221 if (is_gimple_call (stmt
)
3222 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3223 == BUILT_IN_GOMP_CANCEL
3224 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3225 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3227 const char *bad
= NULL
;
3228 const char *kind
= NULL
;
3229 const char *construct
3230 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3231 == BUILT_IN_GOMP_CANCEL
)
3232 ? "#pragma omp cancel"
3233 : "#pragma omp cancellation point";
3236 error_at (gimple_location (stmt
), "orphaned %qs construct",
3240 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3241 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3245 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3246 bad
= "#pragma omp parallel";
3247 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3248 == BUILT_IN_GOMP_CANCEL
3249 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3250 ctx
->cancellable
= true;
3254 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3255 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3256 bad
= "#pragma omp for";
3257 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3258 == BUILT_IN_GOMP_CANCEL
3259 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3261 ctx
->cancellable
= true;
3262 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3264 warning_at (gimple_location (stmt
), 0,
3265 "%<#pragma omp cancel for%> inside "
3266 "%<nowait%> for construct");
3267 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3268 OMP_CLAUSE_ORDERED
))
3269 warning_at (gimple_location (stmt
), 0,
3270 "%<#pragma omp cancel for%> inside "
3271 "%<ordered%> for construct");
3276 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3277 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3278 bad
= "#pragma omp sections";
3279 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3280 == BUILT_IN_GOMP_CANCEL
3281 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3283 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3285 ctx
->cancellable
= true;
3286 if (find_omp_clause (gimple_omp_sections_clauses
3289 warning_at (gimple_location (stmt
), 0,
3290 "%<#pragma omp cancel sections%> inside "
3291 "%<nowait%> sections construct");
3295 gcc_assert (ctx
->outer
3296 && gimple_code (ctx
->outer
->stmt
)
3297 == GIMPLE_OMP_SECTIONS
);
3298 ctx
->outer
->cancellable
= true;
3299 if (find_omp_clause (gimple_omp_sections_clauses
3302 warning_at (gimple_location (stmt
), 0,
3303 "%<#pragma omp cancel sections%> inside "
3304 "%<nowait%> sections construct");
3310 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3311 bad
= "#pragma omp task";
3314 for (omp_context
*octx
= ctx
->outer
;
3315 octx
; octx
= octx
->outer
)
3317 switch (gimple_code (octx
->stmt
))
3319 case GIMPLE_OMP_TASKGROUP
:
3321 case GIMPLE_OMP_TARGET
:
3322 if (gimple_omp_target_kind (octx
->stmt
)
3323 != GF_OMP_TARGET_KIND_REGION
)
3326 case GIMPLE_OMP_PARALLEL
:
3327 case GIMPLE_OMP_TEAMS
:
3328 error_at (gimple_location (stmt
),
3329 "%<%s taskgroup%> construct not closely "
3330 "nested inside of %<taskgroup%> region",
3338 ctx
->cancellable
= true;
3343 error_at (gimple_location (stmt
), "invalid arguments");
3348 error_at (gimple_location (stmt
),
3349 "%<%s %s%> construct not closely nested inside of %qs",
3350 construct
, kind
, bad
);
3355 case GIMPLE_OMP_SECTIONS
:
3356 case GIMPLE_OMP_SINGLE
:
3357 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3358 switch (gimple_code (ctx
->stmt
))
3360 case GIMPLE_OMP_FOR
:
3361 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3362 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3365 case GIMPLE_OMP_SECTIONS
:
3366 case GIMPLE_OMP_SINGLE
:
3367 case GIMPLE_OMP_ORDERED
:
3368 case GIMPLE_OMP_MASTER
:
3369 case GIMPLE_OMP_TASK
:
3370 case GIMPLE_OMP_CRITICAL
:
3371 if (is_gimple_call (stmt
))
3373 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3374 != BUILT_IN_GOMP_BARRIER
)
3376 error_at (gimple_location (stmt
),
3377 "barrier region may not be closely nested inside "
3378 "of work-sharing, %<critical%>, %<ordered%>, "
3379 "%<master%>, explicit %<task%> or %<taskloop%> "
3383 error_at (gimple_location (stmt
),
3384 "work-sharing region may not be closely nested inside "
3385 "of work-sharing, %<critical%>, %<ordered%>, "
3386 "%<master%>, explicit %<task%> or %<taskloop%> region");
3388 case GIMPLE_OMP_PARALLEL
:
3389 case GIMPLE_OMP_TEAMS
:
3391 case GIMPLE_OMP_TARGET
:
3392 if (gimple_omp_target_kind (ctx
->stmt
)
3393 == GF_OMP_TARGET_KIND_REGION
)
3400 case GIMPLE_OMP_MASTER
:
3401 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3402 switch (gimple_code (ctx
->stmt
))
3404 case GIMPLE_OMP_FOR
:
3405 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3406 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3409 case GIMPLE_OMP_SECTIONS
:
3410 case GIMPLE_OMP_SINGLE
:
3411 case GIMPLE_OMP_TASK
:
3412 error_at (gimple_location (stmt
),
3413 "%<master%> region may not be closely nested inside "
3414 "of work-sharing, explicit %<task%> or %<taskloop%> "
3417 case GIMPLE_OMP_PARALLEL
:
3418 case GIMPLE_OMP_TEAMS
:
3420 case GIMPLE_OMP_TARGET
:
3421 if (gimple_omp_target_kind (ctx
->stmt
)
3422 == GF_OMP_TARGET_KIND_REGION
)
3429 case GIMPLE_OMP_TASK
:
3430 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3431 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3432 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3433 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3435 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3436 error_at (OMP_CLAUSE_LOCATION (c
),
3437 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3438 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3442 case GIMPLE_OMP_ORDERED
:
3443 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3444 c
; c
= OMP_CLAUSE_CHAIN (c
))
3446 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3448 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3449 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3452 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3453 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3454 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3457 /* Look for containing ordered(N) loop. */
3459 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3461 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3462 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3464 error_at (OMP_CLAUSE_LOCATION (c
),
3465 "%<ordered%> construct with %<depend%> clause "
3466 "must be closely nested inside an %<ordered%> "
3470 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3472 error_at (OMP_CLAUSE_LOCATION (c
),
3473 "%<ordered%> construct with %<depend%> clause "
3474 "must be closely nested inside a loop with "
3475 "%<ordered%> clause with a parameter");
3481 error_at (OMP_CLAUSE_LOCATION (c
),
3482 "invalid depend kind in omp %<ordered%> %<depend%>");
3486 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3487 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3489 /* ordered simd must be closely nested inside of simd region,
3490 and simd region must not encounter constructs other than
3491 ordered simd, therefore ordered simd may be either orphaned,
3492 or ctx->stmt must be simd. The latter case is handled already
3496 error_at (gimple_location (stmt
),
3497 "%<ordered%> %<simd%> must be closely nested inside "
3502 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3503 switch (gimple_code (ctx
->stmt
))
3505 case GIMPLE_OMP_CRITICAL
:
3506 case GIMPLE_OMP_TASK
:
3507 case GIMPLE_OMP_ORDERED
:
3508 ordered_in_taskloop
:
3509 error_at (gimple_location (stmt
),
3510 "%<ordered%> region may not be closely nested inside "
3511 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3512 "%<taskloop%> region");
3514 case GIMPLE_OMP_FOR
:
3515 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3516 goto ordered_in_taskloop
;
3517 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3518 OMP_CLAUSE_ORDERED
) == NULL
)
3520 error_at (gimple_location (stmt
),
3521 "%<ordered%> region must be closely nested inside "
3522 "a loop region with an %<ordered%> clause");
3526 case GIMPLE_OMP_TARGET
:
3527 if (gimple_omp_target_kind (ctx
->stmt
)
3528 != GF_OMP_TARGET_KIND_REGION
)
3531 case GIMPLE_OMP_PARALLEL
:
3532 case GIMPLE_OMP_TEAMS
:
3533 error_at (gimple_location (stmt
),
3534 "%<ordered%> region must be closely nested inside "
3535 "a loop region with an %<ordered%> clause");
3541 case GIMPLE_OMP_CRITICAL
:
3544 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3545 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3546 if (gomp_critical
*other_crit
3547 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3548 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3550 error_at (gimple_location (stmt
),
3551 "%<critical%> region may not be nested inside "
3552 "a %<critical%> region with the same name");
3557 case GIMPLE_OMP_TEAMS
:
3559 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3560 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3562 error_at (gimple_location (stmt
),
3563 "%<teams%> construct not closely nested inside of "
3564 "%<target%> construct");
3568 case GIMPLE_OMP_TARGET
:
3569 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3570 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3571 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3572 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3574 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3575 error_at (OMP_CLAUSE_LOCATION (c
),
3576 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3577 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3580 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3582 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3584 if (is_gimple_omp (stmt
)
3585 && is_gimple_omp_oacc (stmt
)
3586 && is_gimple_omp (ctx
->stmt
))
3588 error_at (gimple_location (stmt
),
3589 "OpenACC construct inside of non-OpenACC region");
3595 const char *stmt_name
, *ctx_stmt_name
;
3596 switch (gimple_omp_target_kind (stmt
))
3598 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3599 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3600 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3601 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3602 stmt_name
= "target enter data"; break;
3603 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3604 stmt_name
= "target exit data"; break;
3605 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3606 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3607 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3608 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3609 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3610 stmt_name
= "enter/exit data"; break;
3611 default: gcc_unreachable ();
3613 switch (gimple_omp_target_kind (ctx
->stmt
))
3615 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3616 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3617 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3618 ctx_stmt_name
= "parallel"; break;
3619 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3620 ctx_stmt_name
= "kernels"; break;
3621 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3622 default: gcc_unreachable ();
3625 /* OpenACC/OpenMP mismatch? */
3626 if (is_gimple_omp_oacc (stmt
)
3627 != is_gimple_omp_oacc (ctx
->stmt
))
3629 error_at (gimple_location (stmt
),
3630 "%s %qs construct inside of %s %qs region",
3631 (is_gimple_omp_oacc (stmt
)
3632 ? "OpenACC" : "OpenMP"), stmt_name
,
3633 (is_gimple_omp_oacc (ctx
->stmt
)
3634 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3637 if (is_gimple_omp_offloaded (ctx
->stmt
))
3639 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3640 if (is_gimple_omp_oacc (ctx
->stmt
))
3642 error_at (gimple_location (stmt
),
3643 "%qs construct inside of %qs region",
3644 stmt_name
, ctx_stmt_name
);
3649 warning_at (gimple_location (stmt
), 0,
3650 "%qs construct inside of %qs region",
3651 stmt_name
, ctx_stmt_name
);
3663 /* Helper function scan_omp.
3665 Callback for walk_tree or operators in walk_gimple_stmt used to
3666 scan for OMP directives in TP. */
3669 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3671 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3672 omp_context
*ctx
= (omp_context
*) wi
->info
;
3675 switch (TREE_CODE (t
))
3682 *tp
= remap_decl (t
, &ctx
->cb
);
3686 if (ctx
&& TYPE_P (t
))
3687 *tp
= remap_type (t
, &ctx
->cb
);
3688 else if (!DECL_P (t
))
3693 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3694 if (tem
!= TREE_TYPE (t
))
3696 if (TREE_CODE (t
) == INTEGER_CST
)
3697 *tp
= wide_int_to_tree (tem
, t
);
3699 TREE_TYPE (t
) = tem
;
3709 /* Return true if FNDECL is a setjmp or a longjmp. */
3712 setjmp_or_longjmp_p (const_tree fndecl
)
3714 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3715 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3716 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3719 tree declname
= DECL_NAME (fndecl
);
3722 const char *name
= IDENTIFIER_POINTER (declname
);
3723 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3727 /* Helper function for scan_omp.
3729 Callback for walk_gimple_stmt used to scan for OMP directives in
3730 the current statement in GSI. */
3733 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3734 struct walk_stmt_info
*wi
)
3736 gimple
*stmt
= gsi_stmt (*gsi
);
3737 omp_context
*ctx
= (omp_context
*) wi
->info
;
3739 if (gimple_has_location (stmt
))
3740 input_location
= gimple_location (stmt
);
3742 /* Check the nesting restrictions. */
3743 bool remove
= false;
3744 if (is_gimple_omp (stmt
))
3745 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3746 else if (is_gimple_call (stmt
))
3748 tree fndecl
= gimple_call_fndecl (stmt
);
3751 if (setjmp_or_longjmp_p (fndecl
)
3753 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3754 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3757 error_at (gimple_location (stmt
),
3758 "setjmp/longjmp inside simd construct");
3760 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3761 switch (DECL_FUNCTION_CODE (fndecl
))
3763 case BUILT_IN_GOMP_BARRIER
:
3764 case BUILT_IN_GOMP_CANCEL
:
3765 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3766 case BUILT_IN_GOMP_TASKYIELD
:
3767 case BUILT_IN_GOMP_TASKWAIT
:
3768 case BUILT_IN_GOMP_TASKGROUP_START
:
3769 case BUILT_IN_GOMP_TASKGROUP_END
:
3770 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3779 stmt
= gimple_build_nop ();
3780 gsi_replace (gsi
, stmt
, false);
3783 *handled_ops_p
= true;
3785 switch (gimple_code (stmt
))
3787 case GIMPLE_OMP_PARALLEL
:
3788 taskreg_nesting_level
++;
3789 scan_omp_parallel (gsi
, ctx
);
3790 taskreg_nesting_level
--;
3793 case GIMPLE_OMP_TASK
:
3794 taskreg_nesting_level
++;
3795 scan_omp_task (gsi
, ctx
);
3796 taskreg_nesting_level
--;
3799 case GIMPLE_OMP_FOR
:
3800 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3803 case GIMPLE_OMP_SECTIONS
:
3804 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3807 case GIMPLE_OMP_SINGLE
:
3808 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3811 case GIMPLE_OMP_SECTION
:
3812 case GIMPLE_OMP_MASTER
:
3813 case GIMPLE_OMP_TASKGROUP
:
3814 case GIMPLE_OMP_ORDERED
:
3815 case GIMPLE_OMP_CRITICAL
:
3816 ctx
= new_omp_context (stmt
, ctx
);
3817 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3820 case GIMPLE_OMP_TARGET
:
3821 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3824 case GIMPLE_OMP_TEAMS
:
3825 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3832 *handled_ops_p
= false;
3834 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3836 var
= DECL_CHAIN (var
))
3837 insert_decl_map (&ctx
->cb
, var
, var
);
3841 *handled_ops_p
= false;
3849 /* Scan all the statements starting at the current statement. CTX
3850 contains context information about the OMP directives and
3851 clauses found during the scan. */
3854 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3856 location_t saved_location
;
3857 struct walk_stmt_info wi
;
3859 memset (&wi
, 0, sizeof (wi
));
3861 wi
.want_locations
= true;
3863 saved_location
= input_location
;
3864 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3865 input_location
= saved_location
;
3868 /* Re-gimplification and code generation routines. */
3870 /* Build a call to GOMP_barrier. */
3873 build_omp_barrier (tree lhs
)
3875 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3876 : BUILT_IN_GOMP_BARRIER
);
3877 gcall
*g
= gimple_build_call (fndecl
, 0);
3879 gimple_call_set_lhs (g
, lhs
);
3883 /* If a context was created for STMT when it was scanned, return it. */
3885 static omp_context
*
3886 maybe_lookup_ctx (gimple
*stmt
)
3889 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3890 return n
? (omp_context
*) n
->value
: NULL
;
3894 /* Find the mapping for DECL in CTX or the immediately enclosing
3895 context that has a mapping for DECL.
3897 If CTX is a nested parallel directive, we may have to use the decl
3898 mappings created in CTX's parent context. Suppose that we have the
3899 following parallel nesting (variable UIDs showed for clarity):
3902 #omp parallel shared(iD.1562) -> outer parallel
3903 iD.1562 = iD.1562 + 1;
3905 #omp parallel shared (iD.1562) -> inner parallel
3906 iD.1562 = iD.1562 - 1;
3908 Each parallel structure will create a distinct .omp_data_s structure
3909 for copying iD.1562 in/out of the directive:
3911 outer parallel .omp_data_s.1.i -> iD.1562
3912 inner parallel .omp_data_s.2.i -> iD.1562
3914 A shared variable mapping will produce a copy-out operation before
3915 the parallel directive and a copy-in operation after it. So, in
3916 this case we would have:
3919 .omp_data_o.1.i = iD.1562;
3920 #omp parallel shared(iD.1562) -> outer parallel
3921 .omp_data_i.1 = &.omp_data_o.1
3922 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3924 .omp_data_o.2.i = iD.1562; -> **
3925 #omp parallel shared(iD.1562) -> inner parallel
3926 .omp_data_i.2 = &.omp_data_o.2
3927 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3930 ** This is a problem. The symbol iD.1562 cannot be referenced
3931 inside the body of the outer parallel region. But since we are
3932 emitting this copy operation while expanding the inner parallel
3933 directive, we need to access the CTX structure of the outer
3934 parallel directive to get the correct mapping:
3936 .omp_data_o.2.i = .omp_data_i.1->i
3938 Since there may be other workshare or parallel directives enclosing
3939 the parallel directive, it may be necessary to walk up the context
3940 parent chain. This is not a problem in general because nested
3941 parallelism happens only rarely. */
3944 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3949 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3950 t
= maybe_lookup_decl (decl
, up
);
3952 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3954 return t
? t
: decl
;
3958 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3959 in outer contexts. */
3962 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3967 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3968 t
= maybe_lookup_decl (decl
, up
);
3970 return t
? t
: decl
;
3974 /* Construct the initialization value for reduction operation OP. */
3977 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
3986 case TRUTH_ORIF_EXPR
:
3987 case TRUTH_XOR_EXPR
:
3989 return build_zero_cst (type
);
3992 case TRUTH_AND_EXPR
:
3993 case TRUTH_ANDIF_EXPR
:
3995 return fold_convert_loc (loc
, type
, integer_one_node
);
3998 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4001 if (SCALAR_FLOAT_TYPE_P (type
))
4003 REAL_VALUE_TYPE max
, min
;
4004 if (HONOR_INFINITIES (type
))
4007 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4010 real_maxval (&min
, 1, TYPE_MODE (type
));
4011 return build_real (type
, min
);
4013 else if (POINTER_TYPE_P (type
))
4016 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4017 return wide_int_to_tree (type
, min
);
4021 gcc_assert (INTEGRAL_TYPE_P (type
));
4022 return TYPE_MIN_VALUE (type
);
4026 if (SCALAR_FLOAT_TYPE_P (type
))
4028 REAL_VALUE_TYPE max
;
4029 if (HONOR_INFINITIES (type
))
4032 real_maxval (&max
, 0, TYPE_MODE (type
));
4033 return build_real (type
, max
);
4035 else if (POINTER_TYPE_P (type
))
4038 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4039 return wide_int_to_tree (type
, max
);
4043 gcc_assert (INTEGRAL_TYPE_P (type
));
4044 return TYPE_MAX_VALUE (type
);
4052 /* Construct the initialization value for reduction CLAUSE. */
4055 omp_reduction_init (tree clause
, tree type
)
4057 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4058 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4061 /* Return alignment to be assumed for var in CLAUSE, which should be
4062 OMP_CLAUSE_ALIGNED. */
4065 omp_clause_aligned_alignment (tree clause
)
4067 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4068 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4070 /* Otherwise return implementation defined alignment. */
4071 unsigned int al
= 1;
4072 machine_mode mode
, vmode
;
4073 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4075 vs
= 1 << floor_log2 (vs
);
4076 static enum mode_class classes
[]
4077 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4078 for (int i
= 0; i
< 4; i
+= 2)
4079 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4081 mode
= GET_MODE_WIDER_MODE (mode
))
4083 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4084 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4087 && GET_MODE_SIZE (vmode
) < vs
4088 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4089 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4091 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4092 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4094 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4095 / GET_MODE_SIZE (mode
));
4096 if (TYPE_MODE (type
) != vmode
)
4098 if (TYPE_ALIGN_UNIT (type
) > al
)
4099 al
= TYPE_ALIGN_UNIT (type
);
4101 return build_int_cst (integer_type_node
, al
);
4104 /* Return maximum possible vectorization factor for the target. */
4111 || !flag_tree_loop_optimize
4112 || (!flag_tree_loop_vectorize
4113 && (global_options_set
.x_flag_tree_loop_vectorize
4114 || global_options_set
.x_flag_tree_vectorize
)))
4117 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4120 vs
= 1 << floor_log2 (vs
);
4123 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4124 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4125 return GET_MODE_NUNITS (vqimode
);
4129 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4133 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4134 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4138 max_vf
= omp_max_vf ();
4141 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4142 OMP_CLAUSE_SAFELEN
);
4143 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4145 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4147 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4151 idx
= create_tmp_var (unsigned_type_node
);
4152 lane
= create_tmp_var (unsigned_type_node
);
4158 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4159 tree avar
= create_tmp_var_raw (atype
);
4160 if (TREE_ADDRESSABLE (new_var
))
4161 TREE_ADDRESSABLE (avar
) = 1;
4162 DECL_ATTRIBUTES (avar
)
4163 = tree_cons (get_identifier ("omp simd array"), NULL
,
4164 DECL_ATTRIBUTES (avar
));
4165 gimple_add_tmp_var (avar
);
4166 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4167 NULL_TREE
, NULL_TREE
);
4168 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4169 NULL_TREE
, NULL_TREE
);
4170 if (DECL_P (new_var
))
4172 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4173 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4178 /* Helper function of lower_rec_input_clauses. For a reference
4179 in simd reduction, add an underlying variable it will reference. */
4182 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4184 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4185 if (TREE_CONSTANT (z
))
4187 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4188 get_name (new_vard
));
4189 gimple_add_tmp_var (z
);
4190 TREE_ADDRESSABLE (z
) = 1;
4191 z
= build_fold_addr_expr_loc (loc
, z
);
4192 gimplify_assign (new_vard
, z
, ilist
);
4196 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4197 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4198 private variables. Initialization statements go in ILIST, while calls
4199 to destructors go in DLIST. */
4202 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4203 omp_context
*ctx
, struct omp_for_data
*fd
)
4205 tree c
, dtor
, copyin_seq
, x
, ptr
;
4206 bool copyin_by_ref
= false;
4207 bool lastprivate_firstprivate
= false;
4208 bool reduction_omp_orig_ref
= false;
4210 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4211 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4213 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4214 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4215 gimple_seq llist
[2] = { NULL
, NULL
};
4219 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4220 with data sharing clauses referencing variable sized vars. That
4221 is unnecessarily hard to support and very unlikely to result in
4222 vectorized code anyway. */
4224 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4225 switch (OMP_CLAUSE_CODE (c
))
4227 case OMP_CLAUSE_LINEAR
:
4228 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4231 case OMP_CLAUSE_PRIVATE
:
4232 case OMP_CLAUSE_FIRSTPRIVATE
:
4233 case OMP_CLAUSE_LASTPRIVATE
:
4234 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4237 case OMP_CLAUSE_REDUCTION
:
4238 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4239 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4246 /* Do all the fixed sized types in the first pass, and the variable sized
4247 types in the second pass. This makes sure that the scalar arguments to
4248 the variable sized types are processed before we use them in the
4249 variable sized operations. */
4250 for (pass
= 0; pass
< 2; ++pass
)
4252 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4254 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4257 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4261 case OMP_CLAUSE_PRIVATE
:
4262 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4265 case OMP_CLAUSE_SHARED
:
4266 /* Ignore shared directives in teams construct. */
4267 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4269 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4271 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4272 || is_global_var (OMP_CLAUSE_DECL (c
)));
4275 case OMP_CLAUSE_FIRSTPRIVATE
:
4276 case OMP_CLAUSE_COPYIN
:
4278 case OMP_CLAUSE_LINEAR
:
4279 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4280 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4281 lastprivate_firstprivate
= true;
4283 case OMP_CLAUSE_REDUCTION
:
4284 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4285 reduction_omp_orig_ref
= true;
4287 case OMP_CLAUSE__LOOPTEMP_
:
4288 /* Handle _looptemp_ clauses only on parallel/task. */
4292 case OMP_CLAUSE_LASTPRIVATE
:
4293 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4295 lastprivate_firstprivate
= true;
4296 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4299 /* Even without corresponding firstprivate, if
4300 decl is Fortran allocatable, it needs outer var
4303 && lang_hooks
.decls
.omp_private_outer_ref
4304 (OMP_CLAUSE_DECL (c
)))
4305 lastprivate_firstprivate
= true;
4307 case OMP_CLAUSE_ALIGNED
:
4310 var
= OMP_CLAUSE_DECL (c
);
4311 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4312 && !is_global_var (var
))
4314 new_var
= maybe_lookup_decl (var
, ctx
);
4315 if (new_var
== NULL_TREE
)
4316 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4317 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4318 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4319 omp_clause_aligned_alignment (c
));
4320 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4321 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4322 gimplify_and_add (x
, ilist
);
4324 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4325 && is_global_var (var
))
4327 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4328 new_var
= lookup_decl (var
, ctx
);
4329 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4330 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4331 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4332 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4333 omp_clause_aligned_alignment (c
));
4334 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4335 x
= create_tmp_var (ptype
);
4336 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4337 gimplify_and_add (t
, ilist
);
4338 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4339 SET_DECL_VALUE_EXPR (new_var
, t
);
4340 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4347 new_var
= var
= OMP_CLAUSE_DECL (c
);
4348 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4350 var
= TREE_OPERAND (var
, 0);
4351 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4352 var
= TREE_OPERAND (var
, 0);
4353 if (TREE_CODE (var
) == INDIRECT_REF
4354 || TREE_CODE (var
) == ADDR_EXPR
)
4355 var
= TREE_OPERAND (var
, 0);
4356 if (is_variable_sized (var
))
4358 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4359 var
= DECL_VALUE_EXPR (var
);
4360 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4361 var
= TREE_OPERAND (var
, 0);
4362 gcc_assert (DECL_P (var
));
4366 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4367 new_var
= lookup_decl (var
, ctx
);
4369 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4374 /* C/C++ array section reductions. */
4375 else if (c_kind
== OMP_CLAUSE_REDUCTION
4376 && var
!= OMP_CLAUSE_DECL (c
))
4381 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4382 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4383 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4385 tree b
= TREE_OPERAND (orig_var
, 1);
4386 b
= maybe_lookup_decl (b
, ctx
);
4389 b
= TREE_OPERAND (orig_var
, 1);
4390 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4392 if (integer_zerop (bias
))
4396 bias
= fold_convert_loc (clause_loc
,
4397 TREE_TYPE (b
), bias
);
4398 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4399 TREE_TYPE (b
), b
, bias
);
4401 orig_var
= TREE_OPERAND (orig_var
, 0);
4403 if (TREE_CODE (orig_var
) == INDIRECT_REF
4404 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4405 orig_var
= TREE_OPERAND (orig_var
, 0);
4406 tree d
= OMP_CLAUSE_DECL (c
);
4407 tree type
= TREE_TYPE (d
);
4408 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4409 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4410 const char *name
= get_name (orig_var
);
4411 if (TREE_CONSTANT (v
))
4413 x
= create_tmp_var_raw (type
, name
);
4414 gimple_add_tmp_var (x
);
4415 TREE_ADDRESSABLE (x
) = 1;
4416 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4421 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4422 tree t
= maybe_lookup_decl (v
, ctx
);
4426 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4427 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4428 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4430 build_int_cst (TREE_TYPE (v
), 1));
4431 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4433 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4434 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4435 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4438 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4439 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4440 tree y
= create_tmp_var (ptype
, name
);
4441 gimplify_assign (y
, x
, ilist
);
4445 if (!integer_zerop (bias
))
4447 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
4448 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
,
4450 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
4451 TREE_TYPE (x
), x
, bias
);
4452 yb
= create_tmp_var (ptype
, name
);
4453 gimplify_assign (yb
, x
, ilist
);
4457 d
= TREE_OPERAND (d
, 0);
4458 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4459 d
= TREE_OPERAND (d
, 0);
4460 if (TREE_CODE (d
) == ADDR_EXPR
)
4462 if (orig_var
!= var
)
4464 gcc_assert (is_variable_sized (orig_var
));
4465 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4467 gimplify_assign (new_var
, x
, ilist
);
4468 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4469 tree t
= build_fold_indirect_ref (new_var
);
4470 DECL_IGNORED_P (new_var
) = 0;
4471 TREE_THIS_NOTRAP (t
);
4472 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4473 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4477 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4478 build_int_cst (ptype
, 0));
4479 SET_DECL_VALUE_EXPR (new_var
, x
);
4480 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4485 gcc_assert (orig_var
== var
);
4486 if (TREE_CODE (d
) == INDIRECT_REF
)
4488 x
= create_tmp_var (ptype
, name
);
4489 TREE_ADDRESSABLE (x
) = 1;
4490 gimplify_assign (x
, yb
, ilist
);
4491 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4493 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4494 gimplify_assign (new_var
, x
, ilist
);
4496 tree y1
= create_tmp_var (ptype
, NULL
);
4497 gimplify_assign (y1
, y
, ilist
);
4498 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4499 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4500 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4501 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4503 y2
= create_tmp_var (ptype
, NULL
);
4504 gimplify_assign (y2
, y
, ilist
);
4505 tree ref
= build_outer_var_ref (var
, ctx
);
4506 /* For ref build_outer_var_ref already performs this. */
4507 if (TREE_CODE (d
) == INDIRECT_REF
)
4508 gcc_assert (is_reference (var
));
4509 else if (TREE_CODE (d
) == ADDR_EXPR
)
4510 ref
= build_fold_addr_expr (ref
);
4511 else if (is_reference (var
))
4512 ref
= build_fold_addr_expr (ref
);
4513 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4514 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4515 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4517 y3
= create_tmp_var (ptype
, NULL
);
4518 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4522 y4
= create_tmp_var (ptype
, NULL
);
4523 gimplify_assign (y4
, ref
, dlist
);
4526 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4527 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4528 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4529 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4530 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4533 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4534 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4535 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4536 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4537 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4539 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4541 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4542 tree decl_placeholder
4543 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4544 SET_DECL_VALUE_EXPR (decl_placeholder
,
4545 build_simple_mem_ref (y1
));
4546 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4547 SET_DECL_VALUE_EXPR (placeholder
,
4548 y3
? build_simple_mem_ref (y3
)
4550 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4551 x
= lang_hooks
.decls
.omp_clause_default_ctor
4552 (c
, build_simple_mem_ref (y1
),
4553 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4555 gimplify_and_add (x
, ilist
);
4556 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4558 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4559 lower_omp (&tseq
, ctx
);
4560 gimple_seq_add_seq (ilist
, tseq
);
4562 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4565 SET_DECL_VALUE_EXPR (decl_placeholder
,
4566 build_simple_mem_ref (y2
));
4567 SET_DECL_VALUE_EXPR (placeholder
,
4568 build_simple_mem_ref (y4
));
4569 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4570 lower_omp (&tseq
, ctx
);
4571 gimple_seq_add_seq (dlist
, tseq
);
4572 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4574 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4575 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4576 x
= lang_hooks
.decls
.omp_clause_dtor
4577 (c
, build_simple_mem_ref (y2
));
4580 gimple_seq tseq
= NULL
;
4582 gimplify_stmt (&dtor
, &tseq
);
4583 gimple_seq_add_seq (dlist
, tseq
);
4588 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4589 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4591 /* reduction(-:var) sums up the partial results, so it
4592 acts identically to reduction(+:var). */
4593 if (code
== MINUS_EXPR
)
4596 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4599 x
= build2 (code
, TREE_TYPE (type
),
4600 build_simple_mem_ref (y4
),
4601 build_simple_mem_ref (y2
));
4602 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4606 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4607 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4608 gimple_seq_add_stmt (ilist
, g
);
4611 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4612 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4613 gimple_seq_add_stmt (ilist
, g
);
4615 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4616 build_int_cst (TREE_TYPE (i
), 1));
4617 gimple_seq_add_stmt (ilist
, g
);
4618 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4619 gimple_seq_add_stmt (ilist
, g
);
4620 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4623 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4624 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4625 gimple_seq_add_stmt (dlist
, g
);
4628 g
= gimple_build_assign
4629 (y4
, POINTER_PLUS_EXPR
, y4
,
4630 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4631 gimple_seq_add_stmt (dlist
, g
);
4633 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4634 build_int_cst (TREE_TYPE (i2
), 1));
4635 gimple_seq_add_stmt (dlist
, g
);
4636 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4637 gimple_seq_add_stmt (dlist
, g
);
4638 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4642 else if (is_variable_sized (var
))
4644 /* For variable sized types, we need to allocate the
4645 actual storage here. Call alloca and store the
4646 result in the pointer decl that we created elsewhere. */
4650 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4655 ptr
= DECL_VALUE_EXPR (new_var
);
4656 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4657 ptr
= TREE_OPERAND (ptr
, 0);
4658 gcc_assert (DECL_P (ptr
));
4659 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4661 /* void *tmp = __builtin_alloca */
4662 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4663 stmt
= gimple_build_call (atmp
, 2, x
,
4664 size_int (DECL_ALIGN (var
)));
4665 tmp
= create_tmp_var_raw (ptr_type_node
);
4666 gimple_add_tmp_var (tmp
);
4667 gimple_call_set_lhs (stmt
, tmp
);
4669 gimple_seq_add_stmt (ilist
, stmt
);
4671 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4672 gimplify_assign (ptr
, x
, ilist
);
4675 else if (is_reference (var
) && !is_oacc_parallel (ctx
))
4677 /* For references that are being privatized for Fortran,
4678 allocate new backing storage for the new pointer
4679 variable. This allows us to avoid changing all the
4680 code that expects a pointer to something that expects
4681 a direct variable. */
4685 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4686 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4688 x
= build_receiver_ref (var
, false, ctx
);
4689 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4691 else if (TREE_CONSTANT (x
))
4693 /* For reduction in SIMD loop, defer adding the
4694 initialization of the reference, because if we decide
4695 to use SIMD array for it, the initilization could cause
4697 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4701 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4703 gimple_add_tmp_var (x
);
4704 TREE_ADDRESSABLE (x
) = 1;
4705 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4711 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4712 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4713 tree al
= size_int (TYPE_ALIGN (rtype
));
4714 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4719 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4720 gimplify_assign (new_var
, x
, ilist
);
4723 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4725 else if (c_kind
== OMP_CLAUSE_REDUCTION
4726 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4734 switch (OMP_CLAUSE_CODE (c
))
4736 case OMP_CLAUSE_SHARED
:
4737 /* Ignore shared directives in teams construct. */
4738 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4740 /* Shared global vars are just accessed directly. */
4741 if (is_global_var (new_var
))
4743 /* For taskloop firstprivate/lastprivate, represented
4744 as firstprivate and shared clause on the task, new_var
4745 is the firstprivate var. */
4746 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4748 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4749 needs to be delayed until after fixup_child_record_type so
4750 that we get the correct type during the dereference. */
4751 by_ref
= use_pointer_for_field (var
, ctx
);
4752 x
= build_receiver_ref (var
, by_ref
, ctx
);
4753 SET_DECL_VALUE_EXPR (new_var
, x
);
4754 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4756 /* ??? If VAR is not passed by reference, and the variable
4757 hasn't been initialized yet, then we'll get a warning for
4758 the store into the omp_data_s structure. Ideally, we'd be
4759 able to notice this and not store anything at all, but
4760 we're generating code too early. Suppress the warning. */
4762 TREE_NO_WARNING (var
) = 1;
4765 case OMP_CLAUSE_LASTPRIVATE
:
4766 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4770 case OMP_CLAUSE_PRIVATE
:
4771 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4772 x
= build_outer_var_ref (var
, ctx
);
4773 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4775 if (is_task_ctx (ctx
))
4776 x
= build_receiver_ref (var
, false, ctx
);
4778 x
= build_outer_var_ref (var
, ctx
);
4784 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4785 (c
, unshare_expr (new_var
), x
);
4788 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4789 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4790 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4791 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4792 idx
, lane
, ivar
, lvar
))
4795 x
= lang_hooks
.decls
.omp_clause_default_ctor
4796 (c
, unshare_expr (ivar
), x
);
4798 gimplify_and_add (x
, &llist
[0]);
4801 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4804 gimple_seq tseq
= NULL
;
4807 gimplify_stmt (&dtor
, &tseq
);
4808 gimple_seq_add_seq (&llist
[1], tseq
);
4815 gimplify_and_add (nx
, ilist
);
4819 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4822 gimple_seq tseq
= NULL
;
4825 gimplify_stmt (&dtor
, &tseq
);
4826 gimple_seq_add_seq (dlist
, tseq
);
4830 case OMP_CLAUSE_LINEAR
:
4831 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4832 goto do_firstprivate
;
4833 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4836 x
= build_outer_var_ref (var
, ctx
);
4839 case OMP_CLAUSE_FIRSTPRIVATE
:
4840 if (is_task_ctx (ctx
))
4842 if (is_reference (var
) || is_variable_sized (var
))
4844 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4846 || use_pointer_for_field (var
, NULL
))
4848 x
= build_receiver_ref (var
, false, ctx
);
4849 SET_DECL_VALUE_EXPR (new_var
, x
);
4850 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4855 x
= build_outer_var_ref (var
, ctx
);
4858 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4859 && gimple_omp_for_combined_into_p (ctx
->stmt
))
4861 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4862 tree stept
= TREE_TYPE (t
);
4863 tree ct
= find_omp_clause (clauses
,
4864 OMP_CLAUSE__LOOPTEMP_
);
4866 tree l
= OMP_CLAUSE_DECL (ct
);
4867 tree n1
= fd
->loop
.n1
;
4868 tree step
= fd
->loop
.step
;
4869 tree itype
= TREE_TYPE (l
);
4870 if (POINTER_TYPE_P (itype
))
4871 itype
= signed_type_for (itype
);
4872 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
4873 if (TYPE_UNSIGNED (itype
)
4874 && fd
->loop
.cond_code
== GT_EXPR
)
4875 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4876 fold_build1 (NEGATE_EXPR
, itype
, l
),
4877 fold_build1 (NEGATE_EXPR
,
4880 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
4881 t
= fold_build2 (MULT_EXPR
, stept
,
4882 fold_convert (stept
, l
), t
);
4884 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4886 x
= lang_hooks
.decls
.omp_clause_linear_ctor
4888 gimplify_and_add (x
, ilist
);
4892 if (POINTER_TYPE_P (TREE_TYPE (x
)))
4893 x
= fold_build2 (POINTER_PLUS_EXPR
,
4894 TREE_TYPE (x
), x
, t
);
4896 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4899 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4900 || TREE_ADDRESSABLE (new_var
))
4901 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4902 idx
, lane
, ivar
, lvar
))
4904 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4906 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4907 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4908 gimplify_and_add (x
, ilist
);
4909 gimple_stmt_iterator gsi
4910 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4912 = gimple_build_assign (unshare_expr (lvar
), iv
);
4913 gsi_insert_before_without_update (&gsi
, g
,
4915 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4916 enum tree_code code
= PLUS_EXPR
;
4917 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4918 code
= POINTER_PLUS_EXPR
;
4919 g
= gimple_build_assign (iv
, code
, iv
, t
);
4920 gsi_insert_before_without_update (&gsi
, g
,
4924 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4925 (c
, unshare_expr (ivar
), x
);
4926 gimplify_and_add (x
, &llist
[0]);
4927 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4930 gimple_seq tseq
= NULL
;
4933 gimplify_stmt (&dtor
, &tseq
);
4934 gimple_seq_add_seq (&llist
[1], tseq
);
4939 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4940 (c
, unshare_expr (new_var
), x
);
4941 gimplify_and_add (x
, ilist
);
4944 case OMP_CLAUSE__LOOPTEMP_
:
4945 gcc_assert (is_taskreg_ctx (ctx
));
4946 x
= build_outer_var_ref (var
, ctx
);
4947 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4948 gimplify_and_add (x
, ilist
);
4951 case OMP_CLAUSE_COPYIN
:
4952 by_ref
= use_pointer_for_field (var
, NULL
);
4953 x
= build_receiver_ref (var
, by_ref
, ctx
);
4954 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4955 append_to_statement_list (x
, ©in_seq
);
4956 copyin_by_ref
|= by_ref
;
4959 case OMP_CLAUSE_REDUCTION
:
4960 /* OpenACC reductions are initialized using the
4961 GOACC_REDUCTION internal function. */
4962 if (is_gimple_omp_oacc (ctx
->stmt
))
4964 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4966 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4968 x
= build_outer_var_ref (var
, ctx
);
4970 if (is_reference (var
)
4971 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4973 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4974 SET_DECL_VALUE_EXPR (placeholder
, x
);
4975 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4976 tree new_vard
= new_var
;
4977 if (is_reference (var
))
4979 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4980 new_vard
= TREE_OPERAND (new_var
, 0);
4981 gcc_assert (DECL_P (new_vard
));
4984 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4985 idx
, lane
, ivar
, lvar
))
4987 if (new_vard
== new_var
)
4989 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4990 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4994 SET_DECL_VALUE_EXPR (new_vard
,
4995 build_fold_addr_expr (ivar
));
4996 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4998 x
= lang_hooks
.decls
.omp_clause_default_ctor
4999 (c
, unshare_expr (ivar
),
5000 build_outer_var_ref (var
, ctx
));
5002 gimplify_and_add (x
, &llist
[0]);
5003 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5005 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5006 lower_omp (&tseq
, ctx
);
5007 gimple_seq_add_seq (&llist
[0], tseq
);
5009 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5010 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5011 lower_omp (&tseq
, ctx
);
5012 gimple_seq_add_seq (&llist
[1], tseq
);
5013 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5014 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5015 if (new_vard
== new_var
)
5016 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5018 SET_DECL_VALUE_EXPR (new_vard
,
5019 build_fold_addr_expr (lvar
));
5020 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5025 gimplify_stmt (&dtor
, &tseq
);
5026 gimple_seq_add_seq (&llist
[1], tseq
);
5030 /* If this is a reference to constant size reduction var
5031 with placeholder, we haven't emitted the initializer
5032 for it because it is undesirable if SIMD arrays are used.
5033 But if they aren't used, we need to emit the deferred
5034 initialization now. */
5035 else if (is_reference (var
) && is_simd
)
5036 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5037 x
= lang_hooks
.decls
.omp_clause_default_ctor
5038 (c
, unshare_expr (new_var
),
5039 build_outer_var_ref (var
, ctx
));
5041 gimplify_and_add (x
, ilist
);
5042 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5044 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5045 lower_omp (&tseq
, ctx
);
5046 gimple_seq_add_seq (ilist
, tseq
);
5048 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5051 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5052 lower_omp (&tseq
, ctx
);
5053 gimple_seq_add_seq (dlist
, tseq
);
5054 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5056 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5061 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5062 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5063 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5065 /* reduction(-:var) sums up the partial results, so it
5066 acts identically to reduction(+:var). */
5067 if (code
== MINUS_EXPR
)
5070 tree new_vard
= new_var
;
5071 if (is_simd
&& is_reference (var
))
5073 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5074 new_vard
= TREE_OPERAND (new_var
, 0);
5075 gcc_assert (DECL_P (new_vard
));
5078 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5079 idx
, lane
, ivar
, lvar
))
5081 tree ref
= build_outer_var_ref (var
, ctx
);
5083 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5085 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5086 ref
= build_outer_var_ref (var
, ctx
);
5087 gimplify_assign (ref
, x
, &llist
[1]);
5089 if (new_vard
!= new_var
)
5091 SET_DECL_VALUE_EXPR (new_vard
,
5092 build_fold_addr_expr (lvar
));
5093 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5098 if (is_reference (var
) && is_simd
)
5099 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5100 gimplify_assign (new_var
, x
, ilist
);
5103 tree ref
= build_outer_var_ref (var
, ctx
);
5105 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5106 ref
= build_outer_var_ref (var
, ctx
);
5107 gimplify_assign (ref
, x
, dlist
);
5121 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5122 /* Don't want uninit warnings on simduid, it is always uninitialized,
5123 but we use it not for the value, but for the DECL_UID only. */
5124 TREE_NO_WARNING (uid
) = 1;
5126 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5127 gimple_call_set_lhs (g
, lane
);
5128 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5129 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5130 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5131 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5132 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5133 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5134 g
= gimple_build_assign (lane
, INTEGER_CST
,
5135 build_int_cst (unsigned_type_node
, 0));
5136 gimple_seq_add_stmt (ilist
, g
);
5137 for (int i
= 0; i
< 2; i
++)
5140 tree vf
= create_tmp_var (unsigned_type_node
);
5141 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5142 gimple_call_set_lhs (g
, vf
);
5143 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5144 gimple_seq_add_stmt (seq
, g
);
5145 tree t
= build_int_cst (unsigned_type_node
, 0);
5146 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5147 gimple_seq_add_stmt (seq
, g
);
5148 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5149 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5150 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5151 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5152 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5153 gimple_seq_add_seq (seq
, llist
[i
]);
5154 t
= build_int_cst (unsigned_type_node
, 1);
5155 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5156 gimple_seq_add_stmt (seq
, g
);
5157 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5158 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5159 gimple_seq_add_stmt (seq
, g
);
5160 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5164 /* The copyin sequence is not to be executed by the main thread, since
5165 that would result in self-copies. Perhaps not visible to scalars,
5166 but it certainly is to C++ operator=. */
5169 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5171 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5172 build_int_cst (TREE_TYPE (x
), 0));
5173 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5174 gimplify_and_add (x
, ilist
);
5177 /* If any copyin variable is passed by reference, we must ensure the
5178 master thread doesn't modify it before it is copied over in all
5179 threads. Similarly for variables in both firstprivate and
5180 lastprivate clauses we need to ensure the lastprivate copying
5181 happens after firstprivate copying in all threads. And similarly
5182 for UDRs if initializer expression refers to omp_orig. */
5183 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5185 /* Don't add any barrier for #pragma omp simd or
5186 #pragma omp distribute. */
5187 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5188 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5189 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5192 /* If max_vf is non-zero, then we can use only a vectorization factor
5193 up to the max_vf we chose. So stick it into the safelen clause. */
5196 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5197 OMP_CLAUSE_SAFELEN
);
5199 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5200 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5203 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5204 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5206 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5207 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5213 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5214 both parallel and workshare constructs. PREDICATE may be NULL if it's
5218 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5221 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5222 bool par_clauses
= false;
5223 tree simduid
= NULL
, lastlane
= NULL
;
5225 /* Early exit if there are no lastprivate or linear clauses. */
5226 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5227 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5228 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5229 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5231 if (clauses
== NULL
)
5233 /* If this was a workshare clause, see if it had been combined
5234 with its parallel. In that case, look for the clauses on the
5235 parallel statement itself. */
5236 if (is_parallel_ctx (ctx
))
5240 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5243 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5244 OMP_CLAUSE_LASTPRIVATE
);
5245 if (clauses
== NULL
)
5253 tree label_true
, arm1
, arm2
;
5255 label
= create_artificial_label (UNKNOWN_LOCATION
);
5256 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5257 arm1
= TREE_OPERAND (predicate
, 0);
5258 arm2
= TREE_OPERAND (predicate
, 1);
5259 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5260 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5261 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5263 gimple_seq_add_stmt (stmt_list
, stmt
);
5264 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5267 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5268 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5270 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5272 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5275 for (c
= clauses
; c
;)
5278 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5280 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5281 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5282 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5284 var
= OMP_CLAUSE_DECL (c
);
5285 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5286 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5287 && is_taskloop_ctx (ctx
))
5289 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5290 new_var
= lookup_decl (var
, ctx
->outer
);
5293 new_var
= lookup_decl (var
, ctx
);
5295 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5297 tree val
= DECL_VALUE_EXPR (new_var
);
5298 if (TREE_CODE (val
) == ARRAY_REF
5299 && VAR_P (TREE_OPERAND (val
, 0))
5300 && lookup_attribute ("omp simd array",
5301 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5304 if (lastlane
== NULL
)
5306 lastlane
= create_tmp_var (unsigned_type_node
);
5308 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5310 TREE_OPERAND (val
, 1));
5311 gimple_call_set_lhs (g
, lastlane
);
5312 gimple_seq_add_stmt (stmt_list
, g
);
5314 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5315 TREE_OPERAND (val
, 0), lastlane
,
5316 NULL_TREE
, NULL_TREE
);
5320 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5321 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5323 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5324 gimple_seq_add_seq (stmt_list
,
5325 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5326 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5328 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5329 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5331 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5332 gimple_seq_add_seq (stmt_list
,
5333 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5334 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5338 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5339 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5341 gcc_checking_assert (is_taskloop_ctx (ctx
));
5342 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5344 if (is_global_var (ovar
))
5348 x
= build_outer_var_ref (var
, ctx
, true);
5349 if (is_reference (var
))
5350 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5351 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5352 gimplify_and_add (x
, stmt_list
);
5354 c
= OMP_CLAUSE_CHAIN (c
);
5355 if (c
== NULL
&& !par_clauses
)
5357 /* If this was a workshare clause, see if it had been combined
5358 with its parallel. In that case, continue looking for the
5359 clauses also on the parallel statement itself. */
5360 if (is_parallel_ctx (ctx
))
5364 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5367 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5368 OMP_CLAUSE_LASTPRIVATE
);
5374 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5377 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5378 (which might be a placeholder). INNER is true if this is an inner
5379 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5380 join markers. Generate the before-loop forking sequence in
5381 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5382 general form of these sequences is
5384 GOACC_REDUCTION_SETUP
5386 GOACC_REDUCTION_INIT
5388 GOACC_REDUCTION_FINI
5390 GOACC_REDUCTION_TEARDOWN. */
5393 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5394 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5395 gimple_seq
*join_seq
, omp_context
*ctx
)
5397 gimple_seq before_fork
= NULL
;
5398 gimple_seq after_fork
= NULL
;
5399 gimple_seq before_join
= NULL
;
5400 gimple_seq after_join
= NULL
;
5401 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5402 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5403 unsigned offset
= 0;
5405 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5406 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5408 tree orig
= OMP_CLAUSE_DECL (c
);
5409 tree var
= maybe_lookup_decl (orig
, ctx
);
5410 tree ref_to_res
= NULL_TREE
;
5411 tree incoming
, outgoing
;
5413 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5414 if (rcode
== MINUS_EXPR
)
5416 else if (rcode
== TRUTH_ANDIF_EXPR
)
5417 rcode
= BIT_AND_EXPR
;
5418 else if (rcode
== TRUTH_ORIF_EXPR
)
5419 rcode
= BIT_IOR_EXPR
;
5420 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5424 gcc_assert (!is_reference (var
));
5426 incoming
= outgoing
= var
;
5430 /* See if an outer construct also reduces this variable. */
5431 omp_context
*outer
= ctx
;
5433 while (omp_context
*probe
= outer
->outer
)
5435 enum gimple_code type
= gimple_code (probe
->stmt
);
5440 case GIMPLE_OMP_FOR
:
5441 cls
= gimple_omp_for_clauses (probe
->stmt
);
5444 case GIMPLE_OMP_TARGET
:
5445 if (gimple_omp_target_kind (probe
->stmt
)
5446 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5449 cls
= gimple_omp_target_clauses (probe
->stmt
);
5457 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5458 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5459 && orig
== OMP_CLAUSE_DECL (cls
))
5460 goto has_outer_reduction
;
5464 /* This is the outermost construct with this reduction,
5465 see if there's a mapping for it. */
5466 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5467 && maybe_lookup_field (orig
, outer
))
5469 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5470 if (is_reference (orig
))
5471 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5474 incoming
= omp_reduction_init_op (loc
, rcode
, TREE_TYPE (var
));
5477 incoming
= outgoing
= orig
;
5479 has_outer_reduction
:;
5483 ref_to_res
= integer_zero_node
;
5485 /* Determine position in reduction buffer, which may be used
5487 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5488 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5489 offset
= (offset
+ align
- 1) & ~(align
- 1);
5490 tree off
= build_int_cst (sizetype
, offset
);
5491 offset
+= GET_MODE_SIZE (mode
);
5495 init_code
= build_int_cst (integer_type_node
,
5496 IFN_GOACC_REDUCTION_INIT
);
5497 fini_code
= build_int_cst (integer_type_node
,
5498 IFN_GOACC_REDUCTION_FINI
);
5499 setup_code
= build_int_cst (integer_type_node
,
5500 IFN_GOACC_REDUCTION_SETUP
);
5501 teardown_code
= build_int_cst (integer_type_node
,
5502 IFN_GOACC_REDUCTION_TEARDOWN
);
5506 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5507 TREE_TYPE (var
), 6, setup_code
,
5508 unshare_expr (ref_to_res
),
5509 incoming
, level
, op
, off
);
5511 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5512 TREE_TYPE (var
), 6, init_code
,
5513 unshare_expr (ref_to_res
),
5514 var
, level
, op
, off
);
5516 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5517 TREE_TYPE (var
), 6, fini_code
,
5518 unshare_expr (ref_to_res
),
5519 var
, level
, op
, off
);
5521 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5522 TREE_TYPE (var
), 6, teardown_code
,
5523 ref_to_res
, var
, level
, op
, off
);
5525 gimplify_assign (var
, setup_call
, &before_fork
);
5526 gimplify_assign (var
, init_call
, &after_fork
);
5527 gimplify_assign (var
, fini_call
, &before_join
);
5528 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5531 /* Now stitch things together. */
5532 gimple_seq_add_seq (fork_seq
, before_fork
);
5534 gimple_seq_add_stmt (fork_seq
, fork
);
5535 gimple_seq_add_seq (fork_seq
, after_fork
);
5537 gimple_seq_add_seq (join_seq
, before_join
);
5539 gimple_seq_add_stmt (join_seq
, join
);
5540 gimple_seq_add_seq (join_seq
, after_join
);
5543 /* Generate code to implement the REDUCTION clauses. */
5546 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5548 gimple_seq sub_seq
= NULL
;
5553 /* OpenACC loop reductions are handled elsewhere. */
5554 if (is_gimple_omp_oacc (ctx
->stmt
))
5557 /* SIMD reductions are handled in lower_rec_input_clauses. */
5558 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5559 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5562 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5563 update in that case, otherwise use a lock. */
5564 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5565 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5567 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5568 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5570 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5580 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5582 tree var
, ref
, new_var
, orig_var
;
5583 enum tree_code code
;
5584 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5586 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5589 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5590 if (TREE_CODE (var
) == MEM_REF
)
5592 var
= TREE_OPERAND (var
, 0);
5593 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5594 var
= TREE_OPERAND (var
, 0);
5595 if (TREE_CODE (var
) == INDIRECT_REF
5596 || TREE_CODE (var
) == ADDR_EXPR
)
5597 var
= TREE_OPERAND (var
, 0);
5599 if (is_variable_sized (var
))
5601 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5602 var
= DECL_VALUE_EXPR (var
);
5603 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5604 var
= TREE_OPERAND (var
, 0);
5605 gcc_assert (DECL_P (var
));
5608 new_var
= lookup_decl (var
, ctx
);
5609 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5610 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5611 ref
= build_outer_var_ref (var
, ctx
);
5612 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5614 /* reduction(-:var) sums up the partial results, so it acts
5615 identically to reduction(+:var). */
5616 if (code
== MINUS_EXPR
)
5621 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5623 addr
= save_expr (addr
);
5624 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5625 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5626 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5627 gimplify_and_add (x
, stmt_seqp
);
5630 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5632 tree d
= OMP_CLAUSE_DECL (c
);
5633 tree type
= TREE_TYPE (d
);
5634 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5635 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5636 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5637 tree bias
= TREE_OPERAND (d
, 1);
5638 d
= TREE_OPERAND (d
, 0);
5639 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5641 tree b
= TREE_OPERAND (d
, 1);
5642 b
= maybe_lookup_decl (b
, ctx
);
5645 b
= TREE_OPERAND (d
, 1);
5646 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5648 if (integer_zerop (bias
))
5652 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5653 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5654 TREE_TYPE (b
), b
, bias
);
5656 d
= TREE_OPERAND (d
, 0);
5658 /* For ref build_outer_var_ref already performs this, so
5659 only new_var needs a dereference. */
5660 if (TREE_CODE (d
) == INDIRECT_REF
)
5662 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5663 gcc_assert (is_reference (var
) && var
== orig_var
);
5665 else if (TREE_CODE (d
) == ADDR_EXPR
)
5667 if (orig_var
== var
)
5669 new_var
= build_fold_addr_expr (new_var
);
5670 ref
= build_fold_addr_expr (ref
);
5675 gcc_assert (orig_var
== var
);
5676 if (is_reference (var
))
5677 ref
= build_fold_addr_expr (ref
);
5681 tree t
= maybe_lookup_decl (v
, ctx
);
5685 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5686 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5688 if (!integer_zerop (bias
))
5690 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5691 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5692 TREE_TYPE (new_var
), new_var
,
5693 unshare_expr (bias
));
5694 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5695 TREE_TYPE (ref
), ref
, bias
);
5697 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5698 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5699 tree m
= create_tmp_var (ptype
, NULL
);
5700 gimplify_assign (m
, new_var
, stmt_seqp
);
5702 m
= create_tmp_var (ptype
, NULL
);
5703 gimplify_assign (m
, ref
, stmt_seqp
);
5705 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5706 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5707 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5708 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5709 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5710 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5711 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5713 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5714 tree decl_placeholder
5715 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5716 SET_DECL_VALUE_EXPR (placeholder
, out
);
5717 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5718 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5719 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5720 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5721 gimple_seq_add_seq (&sub_seq
,
5722 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5723 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5724 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5725 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5729 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5730 out
= unshare_expr (out
);
5731 gimplify_assign (out
, x
, &sub_seq
);
5733 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5734 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5735 gimple_seq_add_stmt (&sub_seq
, g
);
5736 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5737 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5738 gimple_seq_add_stmt (&sub_seq
, g
);
5739 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5740 build_int_cst (TREE_TYPE (i
), 1));
5741 gimple_seq_add_stmt (&sub_seq
, g
);
5742 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5743 gimple_seq_add_stmt (&sub_seq
, g
);
5744 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5746 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5748 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5750 if (is_reference (var
)
5751 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5753 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5754 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5755 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5756 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5757 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5758 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5759 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5763 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5764 ref
= build_outer_var_ref (var
, ctx
);
5765 gimplify_assign (ref
, x
, &sub_seq
);
5769 if (is_gimple_omp_oacc (ctx
->stmt
))
5772 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5774 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5776 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5778 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5780 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5784 /* Generate code to implement the COPYPRIVATE clauses. */
5787 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5792 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5794 tree var
, new_var
, ref
, x
;
5796 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5798 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5801 var
= OMP_CLAUSE_DECL (c
);
5802 by_ref
= use_pointer_for_field (var
, NULL
);
5804 ref
= build_sender_ref (var
, ctx
);
5805 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5808 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5809 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5811 gimplify_assign (ref
, x
, slist
);
5813 ref
= build_receiver_ref (var
, false, ctx
);
5816 ref
= fold_convert_loc (clause_loc
,
5817 build_pointer_type (TREE_TYPE (new_var
)),
5819 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5821 if (is_reference (var
))
5823 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5824 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5825 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5827 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5828 gimplify_and_add (x
, rlist
);
5833 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5834 and REDUCTION from the sender (aka parent) side. */
5837 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5841 int ignored_looptemp
= 0;
5842 bool is_taskloop
= false;
5844 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5845 by GOMP_taskloop. */
5846 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5848 ignored_looptemp
= 2;
5852 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5854 tree val
, ref
, x
, var
;
5855 bool by_ref
, do_in
= false, do_out
= false;
5856 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5858 switch (OMP_CLAUSE_CODE (c
))
5860 case OMP_CLAUSE_PRIVATE
:
5861 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
5864 case OMP_CLAUSE_FIRSTPRIVATE
:
5865 case OMP_CLAUSE_COPYIN
:
5866 case OMP_CLAUSE_LASTPRIVATE
:
5867 case OMP_CLAUSE_REDUCTION
:
5869 case OMP_CLAUSE_SHARED
:
5870 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5873 case OMP_CLAUSE__LOOPTEMP_
:
5874 if (ignored_looptemp
)
5884 val
= OMP_CLAUSE_DECL (c
);
5885 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5886 && TREE_CODE (val
) == MEM_REF
)
5888 val
= TREE_OPERAND (val
, 0);
5889 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
5890 val
= TREE_OPERAND (val
, 0);
5891 if (TREE_CODE (val
) == INDIRECT_REF
5892 || TREE_CODE (val
) == ADDR_EXPR
)
5893 val
= TREE_OPERAND (val
, 0);
5894 if (is_variable_sized (val
))
5898 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
5899 outer taskloop region. */
5900 omp_context
*ctx_for_o
= ctx
;
5902 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
5903 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5904 ctx_for_o
= ctx
->outer
;
5906 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
5908 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
5909 && is_global_var (var
))
5912 t
= omp_member_access_dummy_var (var
);
5915 var
= DECL_VALUE_EXPR (var
);
5916 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
5918 var
= unshare_and_remap (var
, t
, o
);
5920 var
= unshare_expr (var
);
5923 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
5925 /* Handle taskloop firstprivate/lastprivate, where the
5926 lastprivate on GIMPLE_OMP_TASK is represented as
5927 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
5928 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
5929 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
5930 if (use_pointer_for_field (val
, ctx
))
5931 var
= build_fold_addr_expr (var
);
5932 gimplify_assign (x
, var
, ilist
);
5933 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
5937 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
5938 || val
== OMP_CLAUSE_DECL (c
))
5939 && is_variable_sized (val
))
5941 by_ref
= use_pointer_for_field (val
, NULL
);
5943 switch (OMP_CLAUSE_CODE (c
))
5945 case OMP_CLAUSE_PRIVATE
:
5946 case OMP_CLAUSE_FIRSTPRIVATE
:
5947 case OMP_CLAUSE_COPYIN
:
5948 case OMP_CLAUSE__LOOPTEMP_
:
5952 case OMP_CLAUSE_LASTPRIVATE
:
5953 if (by_ref
|| is_reference (val
))
5955 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
5962 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
5967 case OMP_CLAUSE_REDUCTION
:
5969 if (val
== OMP_CLAUSE_DECL (c
))
5970 do_out
= !(by_ref
|| is_reference (val
));
5972 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
5981 ref
= build_sender_ref (val
, ctx
);
5982 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
5983 gimplify_assign (ref
, x
, ilist
);
5984 if (is_task_ctx (ctx
))
5985 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
5990 ref
= build_sender_ref (val
, ctx
);
5991 gimplify_assign (var
, ref
, olist
);
5996 /* Generate code to implement SHARED from the sender (aka parent)
5997 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
5998 list things that got automatically shared. */
6001 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6003 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6005 if (ctx
->record_type
== NULL
)
6008 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6009 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6011 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6012 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6015 nvar
= maybe_lookup_decl (ovar
, ctx
);
6016 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6019 /* If CTX is a nested parallel directive. Find the immediately
6020 enclosing parallel or workshare construct that contains a
6021 mapping for OVAR. */
6022 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6024 t
= omp_member_access_dummy_var (var
);
6027 var
= DECL_VALUE_EXPR (var
);
6028 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6030 var
= unshare_and_remap (var
, t
, o
);
6032 var
= unshare_expr (var
);
6035 if (use_pointer_for_field (ovar
, ctx
))
6037 x
= build_sender_ref (ovar
, ctx
);
6038 var
= build_fold_addr_expr (var
);
6039 gimplify_assign (x
, var
, ilist
);
6043 x
= build_sender_ref (ovar
, ctx
);
6044 gimplify_assign (x
, var
, ilist
);
6046 if (!TREE_READONLY (var
)
6047 /* We don't need to receive a new reference to a result
6048 or parm decl. In fact we may not store to it as we will
6049 invalidate any pending RSO and generate wrong gimple
6051 && !((TREE_CODE (var
) == RESULT_DECL
6052 || TREE_CODE (var
) == PARM_DECL
)
6053 && DECL_BY_REFERENCE (var
)))
6055 x
= build_sender_ref (ovar
, ctx
);
6056 gimplify_assign (var
, x
, olist
);
6062 /* Emit an OpenACC head marker call, encapulating the partitioning and
6063 other information that must be processed by the target compiler.
6064 Return the maximum number of dimensions the associated loop might
6065 be partitioned over. */
6068 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6069 gimple_seq
*seq
, omp_context
*ctx
)
6071 unsigned levels
= 0;
6073 tree gang_static
= NULL_TREE
;
6074 auto_vec
<tree
, 5> args
;
6076 args
.quick_push (build_int_cst
6077 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6078 args
.quick_push (ddvar
);
6079 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6081 switch (OMP_CLAUSE_CODE (c
))
6083 case OMP_CLAUSE_GANG
:
6084 tag
|= OLF_DIM_GANG
;
6085 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6086 /* static:* is represented by -1, and we can ignore it, as
6087 scheduling is always static. */
6088 if (gang_static
&& integer_minus_onep (gang_static
))
6089 gang_static
= NULL_TREE
;
6093 case OMP_CLAUSE_WORKER
:
6094 tag
|= OLF_DIM_WORKER
;
6098 case OMP_CLAUSE_VECTOR
:
6099 tag
|= OLF_DIM_VECTOR
;
6103 case OMP_CLAUSE_SEQ
:
6107 case OMP_CLAUSE_AUTO
:
6111 case OMP_CLAUSE_INDEPENDENT
:
6112 tag
|= OLF_INDEPENDENT
;
6122 if (DECL_P (gang_static
))
6123 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6124 tag
|= OLF_GANG_STATIC
;
6127 /* In a parallel region, loops are implicitly INDEPENDENT. */
6128 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6129 if (!tgt
|| is_oacc_parallel (tgt
))
6130 tag
|= OLF_INDEPENDENT
;
6132 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6133 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6137 /* Ensure at least one level. */
6141 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6142 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6144 args
.quick_push (gang_static
);
6146 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6147 gimple_set_location (call
, loc
);
6148 gimple_set_lhs (call
, ddvar
);
6149 gimple_seq_add_stmt (seq
, call
);
6154 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6155 partitioning level of the enclosed region. */
6158 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6159 tree tofollow
, gimple_seq
*seq
)
6161 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6162 : IFN_UNIQUE_OACC_TAIL_MARK
);
6163 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6164 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6165 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6166 marker
, ddvar
, tofollow
);
6167 gimple_set_location (call
, loc
);
6168 gimple_set_lhs (call
, ddvar
);
6169 gimple_seq_add_stmt (seq
, call
);
6172 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6173 the loop clauses, from which we extract reductions. Initialize
6177 lower_oacc_head_tail (location_t loc
, tree clauses
,
6178 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6181 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6182 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6184 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6186 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6188 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6189 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6191 for (unsigned done
= 1; count
; count
--, done
++)
6193 gimple_seq fork_seq
= NULL
;
6194 gimple_seq join_seq
= NULL
;
6196 tree place
= build_int_cst (integer_type_node
, -1);
6197 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6198 fork_kind
, ddvar
, place
);
6199 gimple_set_location (fork
, loc
);
6200 gimple_set_lhs (fork
, ddvar
);
6202 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6203 join_kind
, ddvar
, place
);
6204 gimple_set_location (join
, loc
);
6205 gimple_set_lhs (join
, ddvar
);
6207 /* Mark the beginning of this level sequence. */
6209 lower_oacc_loop_marker (loc
, ddvar
, true,
6210 build_int_cst (integer_type_node
, count
),
6212 lower_oacc_loop_marker (loc
, ddvar
, false,
6213 build_int_cst (integer_type_node
, done
),
6216 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6217 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6219 /* Append this level to head. */
6220 gimple_seq_add_seq (head
, fork_seq
);
6221 /* Prepend it to tail. */
6222 gimple_seq_add_seq (&join_seq
, *tail
);
6228 /* Mark the end of the sequence. */
6229 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6230 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6233 /* A convenience function to build an empty GIMPLE_COND with just the
6237 gimple_build_cond_empty (tree cond
)
6239 enum tree_code pred_code
;
6242 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6243 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6246 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6249 /* Build the function calls to GOMP_parallel_start etc to actually
6250 generate the parallel operation. REGION is the parallel region
6251 being expanded. BB is the block where to insert the code. WS_ARGS
6252 will be set if this is a call to a combined parallel+workshare
6253 construct, it contains the list of additional arguments needed by
6254 the workshare construct. */
6257 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6258 gomp_parallel
*entry_stmt
,
6259 vec
<tree
, va_gc
> *ws_args
)
6261 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6262 gimple_stmt_iterator gsi
;
6264 enum built_in_function start_ix
;
6266 location_t clause_loc
;
6267 vec
<tree
, va_gc
> *args
;
6269 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6271 /* Determine what flavor of GOMP_parallel we will be
6273 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6274 if (is_combined_parallel (region
))
6276 switch (region
->inner
->type
)
6278 case GIMPLE_OMP_FOR
:
6279 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6280 switch (region
->inner
->sched_kind
)
6282 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6285 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6286 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6287 if (region
->inner
->sched_modifiers
6288 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6290 start_ix2
= 3 + region
->inner
->sched_kind
;
6295 start_ix2
= region
->inner
->sched_kind
;
6298 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6299 start_ix
= (enum built_in_function
) start_ix2
;
6301 case GIMPLE_OMP_SECTIONS
:
6302 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6309 /* By default, the value of NUM_THREADS is zero (selected at run time)
6310 and there is no conditional. */
6312 val
= build_int_cst (unsigned_type_node
, 0);
6313 flags
= build_int_cst (unsigned_type_node
, 0);
6315 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6317 cond
= OMP_CLAUSE_IF_EXPR (c
);
6319 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6322 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6323 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6326 clause_loc
= gimple_location (entry_stmt
);
6328 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6330 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6332 /* Ensure 'val' is of the correct type. */
6333 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6335 /* If we found the clause 'if (cond)', build either
6336 (cond != 0) or (cond ? val : 1u). */
6339 cond
= gimple_boolify (cond
);
6341 if (integer_zerop (val
))
6342 val
= fold_build2_loc (clause_loc
,
6343 EQ_EXPR
, unsigned_type_node
, cond
,
6344 build_int_cst (TREE_TYPE (cond
), 0));
6347 basic_block cond_bb
, then_bb
, else_bb
;
6348 edge e
, e_then
, e_else
;
6349 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6351 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6352 if (gimple_in_ssa_p (cfun
))
6354 tmp_then
= make_ssa_name (tmp_var
);
6355 tmp_else
= make_ssa_name (tmp_var
);
6356 tmp_join
= make_ssa_name (tmp_var
);
6365 e
= split_block_after_labels (bb
);
6370 then_bb
= create_empty_bb (cond_bb
);
6371 else_bb
= create_empty_bb (then_bb
);
6372 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6373 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6375 stmt
= gimple_build_cond_empty (cond
);
6376 gsi
= gsi_start_bb (cond_bb
);
6377 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6379 gsi
= gsi_start_bb (then_bb
);
6380 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6382 gsi
= gsi_start_bb (else_bb
);
6383 expand_omp_build_assign (&gsi
, tmp_else
,
6384 build_int_cst (unsigned_type_node
, 1),
6387 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6388 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6389 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6390 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6391 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6392 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6394 if (gimple_in_ssa_p (cfun
))
6396 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6397 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6398 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6404 gsi
= gsi_start_bb (bb
);
6405 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6406 false, GSI_CONTINUE_LINKING
);
6409 gsi
= gsi_last_bb (bb
);
6410 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6412 t1
= null_pointer_node
;
6414 t1
= build_fold_addr_expr (t
);
6415 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6417 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6418 args
->quick_push (t2
);
6419 args
->quick_push (t1
);
6420 args
->quick_push (val
);
6422 args
->splice (*ws_args
);
6423 args
->quick_push (flags
);
6425 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6426 builtin_decl_explicit (start_ix
), args
);
6428 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6429 false, GSI_CONTINUE_LINKING
);
6432 /* Insert a function call whose name is FUNC_NAME with the information from
6433 ENTRY_STMT into the basic_block BB. */
6436 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6437 vec
<tree
, va_gc
> *ws_args
)
6440 gimple_stmt_iterator gsi
;
6441 vec
<tree
, va_gc
> *args
;
6443 gcc_assert (vec_safe_length (ws_args
) == 2);
6444 tree func_name
= (*ws_args
)[0];
6445 tree grain
= (*ws_args
)[1];
6447 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6448 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6449 gcc_assert (count
!= NULL_TREE
);
6450 count
= OMP_CLAUSE_OPERAND (count
, 0);
6452 gsi
= gsi_last_bb (bb
);
6453 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6455 t1
= null_pointer_node
;
6457 t1
= build_fold_addr_expr (t
);
6458 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6460 vec_alloc (args
, 4);
6461 args
->quick_push (t2
);
6462 args
->quick_push (t1
);
6463 args
->quick_push (count
);
6464 args
->quick_push (grain
);
6465 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6467 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6468 GSI_CONTINUE_LINKING
);
6471 /* Build the function call to GOMP_task to actually
6472 generate the task operation. BB is the block where to insert the code. */
6475 expand_task_call (struct omp_region
*region
, basic_block bb
,
6476 gomp_task
*entry_stmt
)
6479 gimple_stmt_iterator gsi
;
6480 location_t loc
= gimple_location (entry_stmt
);
6482 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6484 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6485 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6486 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6487 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6488 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6489 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6492 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6493 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6494 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6496 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6497 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6498 tree num_tasks
= NULL_TREE
;
6502 gimple
*g
= last_stmt (region
->outer
->entry
);
6503 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6504 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6505 struct omp_for_data fd
;
6506 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6507 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6508 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6509 OMP_CLAUSE__LOOPTEMP_
);
6510 startvar
= OMP_CLAUSE_DECL (startvar
);
6511 endvar
= OMP_CLAUSE_DECL (endvar
);
6512 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6513 if (fd
.loop
.cond_code
== LT_EXPR
)
6514 iflags
|= GOMP_TASK_FLAG_UP
;
6515 tree tclauses
= gimple_omp_for_clauses (g
);
6516 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6518 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6521 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6524 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6525 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6528 num_tasks
= integer_zero_node
;
6530 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6531 if (ifc
== NULL_TREE
)
6532 iflags
|= GOMP_TASK_FLAG_IF
;
6533 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6534 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6535 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6538 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6540 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6542 tree cond
= boolean_true_node
;
6547 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6548 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6549 build_int_cst (unsigned_type_node
,
6551 build_int_cst (unsigned_type_node
, 0));
6552 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6556 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6561 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6562 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6563 build_int_cst (unsigned_type_node
,
6564 GOMP_TASK_FLAG_FINAL
),
6565 build_int_cst (unsigned_type_node
, 0));
6566 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6569 depend
= OMP_CLAUSE_DECL (depend
);
6571 depend
= build_int_cst (ptr_type_node
, 0);
6573 priority
= fold_convert (integer_type_node
,
6574 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6576 priority
= integer_zero_node
;
6578 gsi
= gsi_last_bb (bb
);
6579 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6581 t2
= null_pointer_node
;
6583 t2
= build_fold_addr_expr_loc (loc
, t
);
6584 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6585 t
= gimple_omp_task_copy_fn (entry_stmt
);
6587 t3
= null_pointer_node
;
6589 t3
= build_fold_addr_expr_loc (loc
, t
);
6592 t
= build_call_expr (ull
6593 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6594 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6596 gimple_omp_task_arg_size (entry_stmt
),
6597 gimple_omp_task_arg_align (entry_stmt
), flags
,
6598 num_tasks
, priority
, startvar
, endvar
, step
);
6600 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6602 gimple_omp_task_arg_size (entry_stmt
),
6603 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6606 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6607 false, GSI_CONTINUE_LINKING
);
6611 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6612 catch handler and return it. This prevents programs from violating the
6613 structured block semantics with throws. */
6616 maybe_catch_exception (gimple_seq body
)
6621 if (!flag_exceptions
)
6624 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6625 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6627 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6629 g
= gimple_build_eh_must_not_throw (decl
);
6630 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6633 return gimple_seq_alloc_with_stmt (g
);
6636 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6639 vec2chain (vec
<tree
, va_gc
> *v
)
6641 tree chain
= NULL_TREE
, t
;
6644 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6646 DECL_CHAIN (t
) = chain
;
6654 /* Remove barriers in REGION->EXIT's block. Note that this is only
6655 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6656 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6657 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6661 remove_exit_barrier (struct omp_region
*region
)
6663 gimple_stmt_iterator gsi
;
6664 basic_block exit_bb
;
6668 int any_addressable_vars
= -1;
6670 exit_bb
= region
->exit
;
6672 /* If the parallel region doesn't return, we don't have REGION->EXIT
6677 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6678 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6679 statements that can appear in between are extremely limited -- no
6680 memory operations at all. Here, we allow nothing at all, so the
6681 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6682 gsi
= gsi_last_bb (exit_bb
);
6683 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6685 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6688 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6690 gsi
= gsi_last_bb (e
->src
);
6691 if (gsi_end_p (gsi
))
6693 stmt
= gsi_stmt (gsi
);
6694 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6695 && !gimple_omp_return_nowait_p (stmt
))
6697 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6698 in many cases. If there could be tasks queued, the barrier
6699 might be needed to let the tasks run before some local
6700 variable of the parallel that the task uses as shared
6701 runs out of scope. The task can be spawned either
6702 from within current function (this would be easy to check)
6703 or from some function it calls and gets passed an address
6704 of such a variable. */
6705 if (any_addressable_vars
< 0)
6707 gomp_parallel
*parallel_stmt
6708 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6709 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6710 tree local_decls
, block
, decl
;
6713 any_addressable_vars
= 0;
6714 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6715 if (TREE_ADDRESSABLE (decl
))
6717 any_addressable_vars
= 1;
6720 for (block
= gimple_block (stmt
);
6721 !any_addressable_vars
6723 && TREE_CODE (block
) == BLOCK
;
6724 block
= BLOCK_SUPERCONTEXT (block
))
6726 for (local_decls
= BLOCK_VARS (block
);
6728 local_decls
= DECL_CHAIN (local_decls
))
6729 if (TREE_ADDRESSABLE (local_decls
))
6731 any_addressable_vars
= 1;
6734 if (block
== gimple_block (parallel_stmt
))
6738 if (!any_addressable_vars
)
6739 gimple_omp_return_set_nowait (stmt
);
6745 remove_exit_barriers (struct omp_region
*region
)
6747 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6748 remove_exit_barrier (region
);
6752 region
= region
->inner
;
6753 remove_exit_barriers (region
);
6754 while (region
->next
)
6756 region
= region
->next
;
6757 remove_exit_barriers (region
);
6762 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6763 calls. These can't be declared as const functions, but
6764 within one parallel body they are constant, so they can be
6765 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6766 which are declared const. Similarly for task body, except
6767 that in untied task omp_get_thread_num () can change at any task
6768 scheduling point. */
6771 optimize_omp_library_calls (gimple
*entry_stmt
)
6774 gimple_stmt_iterator gsi
;
6775 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6776 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6777 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6778 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6779 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6780 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6781 OMP_CLAUSE_UNTIED
) != NULL
);
6783 FOR_EACH_BB_FN (bb
, cfun
)
6784 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6786 gimple
*call
= gsi_stmt (gsi
);
6789 if (is_gimple_call (call
)
6790 && (decl
= gimple_call_fndecl (call
))
6791 && DECL_EXTERNAL (decl
)
6792 && TREE_PUBLIC (decl
)
6793 && DECL_INITIAL (decl
) == NULL
)
6797 if (DECL_NAME (decl
) == thr_num_id
)
6799 /* In #pragma omp task untied omp_get_thread_num () can change
6800 during the execution of the task region. */
6803 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6805 else if (DECL_NAME (decl
) == num_thr_id
)
6806 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6810 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6811 || gimple_call_num_args (call
) != 0)
6814 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
6817 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
6818 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
6819 TREE_TYPE (TREE_TYPE (built_in
))))
6822 gimple_call_set_fndecl (call
, built_in
);
6827 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
6831 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
6835 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
6836 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
6839 if (TREE_CODE (t
) == ADDR_EXPR
)
6840 recompute_tree_invariant_for_addr_expr (t
);
6842 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
6846 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
6849 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
6852 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
6853 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
6854 !after
, after
? GSI_CONTINUE_LINKING
6856 gimple
*stmt
= gimple_build_assign (to
, from
);
6858 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
6860 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
6861 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
6862 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
6864 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
6865 gimple_regimplify_operands (stmt
, &gsi
);
6869 /* Expand the OpenMP parallel or task directive starting at REGION. */
6872 expand_omp_taskreg (struct omp_region
*region
)
6874 basic_block entry_bb
, exit_bb
, new_bb
;
6875 struct function
*child_cfun
;
6876 tree child_fn
, block
, t
;
6877 gimple_stmt_iterator gsi
;
6878 gimple
*entry_stmt
, *stmt
;
6880 vec
<tree
, va_gc
> *ws_args
;
6882 entry_stmt
= last_stmt (region
->entry
);
6883 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
6884 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
6886 entry_bb
= region
->entry
;
6887 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
6888 exit_bb
= region
->cont
;
6890 exit_bb
= region
->exit
;
6894 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
6895 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
6896 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
6899 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
6900 and the inner statement contains the name of the built-in function
6902 ws_args
= region
->inner
->ws_args
;
6903 else if (is_combined_parallel (region
))
6904 ws_args
= region
->ws_args
;
6908 if (child_cfun
->cfg
)
6910 /* Due to inlining, it may happen that we have already outlined
6911 the region, in which case all we need to do is make the
6912 sub-graph unreachable and emit the parallel call. */
6913 edge entry_succ_e
, exit_succ_e
;
6915 entry_succ_e
= single_succ_edge (entry_bb
);
6917 gsi
= gsi_last_bb (entry_bb
);
6918 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
6919 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
6920 gsi_remove (&gsi
, true);
6925 exit_succ_e
= single_succ_edge (exit_bb
);
6926 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
6928 remove_edge_and_dominated_blocks (entry_succ_e
);
6932 unsigned srcidx
, dstidx
, num
;
6934 /* If the parallel region needs data sent from the parent
6935 function, then the very first statement (except possible
6936 tree profile counter updates) of the parallel body
6937 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
6938 &.OMP_DATA_O is passed as an argument to the child function,
6939 we need to replace it with the argument as seen by the child
6942 In most cases, this will end up being the identity assignment
6943 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
6944 a function call that has been inlined, the original PARM_DECL
6945 .OMP_DATA_I may have been converted into a different local
6946 variable. In which case, we need to keep the assignment. */
6947 if (gimple_omp_taskreg_data_arg (entry_stmt
))
6949 basic_block entry_succ_bb
6950 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
6951 : FALLTHRU_EDGE (entry_bb
)->dest
;
6953 gimple
*parcopy_stmt
= NULL
;
6955 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
6959 gcc_assert (!gsi_end_p (gsi
));
6960 stmt
= gsi_stmt (gsi
);
6961 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
6964 if (gimple_num_ops (stmt
) == 2)
6966 tree arg
= gimple_assign_rhs1 (stmt
);
6968 /* We're ignore the subcode because we're
6969 effectively doing a STRIP_NOPS. */
6971 if (TREE_CODE (arg
) == ADDR_EXPR
6972 && TREE_OPERAND (arg
, 0)
6973 == gimple_omp_taskreg_data_arg (entry_stmt
))
6975 parcopy_stmt
= stmt
;
6981 gcc_assert (parcopy_stmt
!= NULL
);
6982 arg
= DECL_ARGUMENTS (child_fn
);
6984 if (!gimple_in_ssa_p (cfun
))
6986 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
6987 gsi_remove (&gsi
, true);
6990 /* ?? Is setting the subcode really necessary ?? */
6991 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
6992 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6997 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
6998 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
6999 /* We'd like to set the rhs to the default def in the child_fn,
7000 but it's too early to create ssa names in the child_fn.
7001 Instead, we set the rhs to the parm. In
7002 move_sese_region_to_fn, we introduce a default def for the
7003 parm, map the parm to it's default def, and once we encounter
7004 this stmt, replace the parm with the default def. */
7005 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7006 update_stmt (parcopy_stmt
);
7010 /* Declare local variables needed in CHILD_CFUN. */
7011 block
= DECL_INITIAL (child_fn
);
7012 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7013 /* The gimplifier could record temporaries in parallel/task block
7014 rather than in containing function's local_decls chain,
7015 which would mean cgraph missed finalizing them. Do it now. */
7016 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7017 if (TREE_CODE (t
) == VAR_DECL
7019 && !DECL_EXTERNAL (t
))
7020 varpool_node::finalize_decl (t
);
7021 DECL_SAVED_TREE (child_fn
) = NULL
;
7022 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7023 gimple_set_body (child_fn
, NULL
);
7024 TREE_USED (block
) = 1;
7026 /* Reset DECL_CONTEXT on function arguments. */
7027 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7028 DECL_CONTEXT (t
) = child_fn
;
7030 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7031 so that it can be moved to the child function. */
7032 gsi
= gsi_last_bb (entry_bb
);
7033 stmt
= gsi_stmt (gsi
);
7034 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7035 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7036 e
= split_block (entry_bb
, stmt
);
7037 gsi_remove (&gsi
, true);
7040 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7041 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7044 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7045 gcc_assert (e2
->dest
== region
->exit
);
7046 remove_edge (BRANCH_EDGE (entry_bb
));
7047 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7048 gsi
= gsi_last_bb (region
->exit
);
7049 gcc_assert (!gsi_end_p (gsi
)
7050 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7051 gsi_remove (&gsi
, true);
7054 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7057 gsi
= gsi_last_bb (exit_bb
);
7058 gcc_assert (!gsi_end_p (gsi
)
7059 && (gimple_code (gsi_stmt (gsi
))
7060 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7061 stmt
= gimple_build_return (NULL
);
7062 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7063 gsi_remove (&gsi
, true);
7066 /* Move the parallel region into CHILD_CFUN. */
7068 if (gimple_in_ssa_p (cfun
))
7070 init_tree_ssa (child_cfun
);
7071 init_ssa_operands (child_cfun
);
7072 child_cfun
->gimple_df
->in_ssa_p
= true;
7076 block
= gimple_block (entry_stmt
);
7078 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7080 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7083 basic_block dest_bb
= e2
->dest
;
7085 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7087 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7089 /* When the OMP expansion process cannot guarantee an up-to-date
7090 loop tree arrange for the child function to fixup loops. */
7091 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7092 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7094 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7095 num
= vec_safe_length (child_cfun
->local_decls
);
7096 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7098 t
= (*child_cfun
->local_decls
)[srcidx
];
7099 if (DECL_CONTEXT (t
) == cfun
->decl
)
7101 if (srcidx
!= dstidx
)
7102 (*child_cfun
->local_decls
)[dstidx
] = t
;
7106 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7108 /* Inform the callgraph about the new function. */
7109 child_cfun
->curr_properties
= cfun
->curr_properties
;
7110 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7111 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7112 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7113 node
->parallelized_function
= 1;
7114 cgraph_node::add_new_function (child_fn
, true);
7116 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7117 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7119 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7120 fixed in a following pass. */
7121 push_cfun (child_cfun
);
7123 assign_assembler_name_if_neeeded (child_fn
);
7126 optimize_omp_library_calls (entry_stmt
);
7127 cgraph_edge::rebuild_edges ();
7129 /* Some EH regions might become dead, see PR34608. If
7130 pass_cleanup_cfg isn't the first pass to happen with the
7131 new child, these dead EH edges might cause problems.
7132 Clean them up now. */
7133 if (flag_exceptions
)
7136 bool changed
= false;
7138 FOR_EACH_BB_FN (bb
, cfun
)
7139 changed
|= gimple_purge_dead_eh_edges (bb
);
7141 cleanup_tree_cfg ();
7143 if (gimple_in_ssa_p (cfun
))
7144 update_ssa (TODO_update_ssa
);
7145 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7146 verify_loop_structure ();
7149 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7151 omp_any_child_fn_dumped
= true;
7152 dump_function_header (dump_file
, child_fn
, dump_flags
);
7153 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7157 /* Emit a library call to launch the children threads. */
7159 expand_cilk_for_call (new_bb
,
7160 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7161 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7162 expand_parallel_call (region
, new_bb
,
7163 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7165 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7166 if (gimple_in_ssa_p (cfun
))
7167 update_ssa (TODO_update_ssa_only_virtuals
);
7170 /* Information about members of an OpenACC collapsed loop nest. */
7172 struct oacc_collapse
7174 tree base
; /* Base value. */
7175 tree iters
; /* Number of steps. */
7176 tree step
; /* step size. */
7179 /* Helper for expand_oacc_for. Determine collapsed loop information.
7180 Fill in COUNTS array. Emit any initialization code before GSI.
7181 Return the calculated outer loop bound of BOUND_TYPE. */
7184 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7185 gimple_stmt_iterator
*gsi
,
7186 oacc_collapse
*counts
, tree bound_type
)
7188 tree total
= build_int_cst (bound_type
, 1);
7191 gcc_assert (integer_onep (fd
->loop
.step
));
7192 gcc_assert (integer_zerop (fd
->loop
.n1
));
7194 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7196 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7198 tree iter_type
= TREE_TYPE (loop
->v
);
7199 tree diff_type
= iter_type
;
7200 tree plus_type
= iter_type
;
7202 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7204 if (POINTER_TYPE_P (iter_type
))
7205 plus_type
= sizetype
;
7206 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7207 diff_type
= signed_type_for (diff_type
);
7211 tree s
= loop
->step
;
7212 bool up
= loop
->cond_code
== LT_EXPR
;
7213 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7217 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7218 true, GSI_SAME_STMT
);
7219 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7220 true, GSI_SAME_STMT
);
7222 /* Convert the step, avoiding possible unsigned->signed overflow. */
7223 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7225 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7226 s
= fold_convert (diff_type
, s
);
7228 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7229 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7230 true, GSI_SAME_STMT
);
7232 /* Determine the range, avoiding possible unsigned->signed overflow. */
7233 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7234 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7235 fold_convert (plus_type
, negating
? b
: e
),
7236 fold_convert (plus_type
, negating
? e
: b
));
7237 expr
= fold_convert (diff_type
, expr
);
7239 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7240 tree range
= force_gimple_operand_gsi
7241 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7243 /* Determine number of iterations. */
7244 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7245 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7246 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7248 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7249 true, GSI_SAME_STMT
);
7251 counts
[ix
].base
= b
;
7252 counts
[ix
].iters
= iters
;
7253 counts
[ix
].step
= s
;
7255 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7256 fold_convert (bound_type
, iters
));
7262 /* Emit initializers for collapsed loop members. IVAR is the outer
7263 loop iteration variable, from which collapsed loop iteration values
7264 are calculated. COUNTS array has been initialized by
7265 expand_oacc_collapse_inits. */
7268 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7269 gimple_stmt_iterator
*gsi
,
7270 const oacc_collapse
*counts
, tree ivar
)
7272 tree ivar_type
= TREE_TYPE (ivar
);
7274 /* The most rapidly changing iteration variable is the innermost
7276 for (int ix
= fd
->collapse
; ix
--;)
7278 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7279 const oacc_collapse
*collapse
= &counts
[ix
];
7280 tree iter_type
= TREE_TYPE (loop
->v
);
7281 tree diff_type
= TREE_TYPE (collapse
->step
);
7282 tree plus_type
= iter_type
;
7283 enum tree_code plus_code
= PLUS_EXPR
;
7286 if (POINTER_TYPE_P (iter_type
))
7288 plus_code
= POINTER_PLUS_EXPR
;
7289 plus_type
= sizetype
;
7292 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7293 fold_convert (ivar_type
, collapse
->iters
));
7294 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7296 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7297 fold_convert (plus_type
, expr
));
7298 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7299 true, GSI_SAME_STMT
);
7300 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7301 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7305 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7306 fold_convert (ivar_type
, collapse
->iters
));
7307 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7308 true, GSI_SAME_STMT
);
7314 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7315 of the combined collapse > 1 loop constructs, generate code like:
7316 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7321 count3 = (adj + N32 - N31) / STEP3;
7322 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7327 count2 = (adj + N22 - N21) / STEP2;
7328 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7333 count1 = (adj + N12 - N11) / STEP1;
7334 count = count1 * count2 * count3;
7335 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7337 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7338 of the combined loop constructs, just initialize COUNTS array
7339 from the _looptemp_ clauses. */
7341 /* NOTE: It *could* be better to moosh all of the BBs together,
7342 creating one larger BB with all the computation and the unexpected
7343 jump at the end. I.e.
7345 bool zero3, zero2, zero1, zero;
7348 count3 = (N32 - N31) /[cl] STEP3;
7350 count2 = (N22 - N21) /[cl] STEP2;
7352 count1 = (N12 - N11) /[cl] STEP1;
7353 zero = zero3 || zero2 || zero1;
7354 count = count1 * count2 * count3;
7355 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7357 After all, we expect the zero=false, and thus we expect to have to
7358 evaluate all of the comparison expressions, so short-circuiting
7359 oughtn't be a win. Since the condition isn't protecting a
7360 denominator, we're not concerned about divide-by-zero, so we can
7361 fully evaluate count even if a numerator turned out to be wrong.
7363 It seems like putting this all together would create much better
7364 scheduling opportunities, and less pressure on the chip's branch
7368 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7369 basic_block
&entry_bb
, tree
*counts
,
7370 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7371 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7372 basic_block
&l2_dom_bb
)
7374 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7378 /* Collapsed loops need work for expansion into SSA form. */
7379 gcc_assert (!gimple_in_ssa_p (cfun
));
7381 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7382 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7384 gcc_assert (fd
->ordered
== 0);
7385 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7386 isn't supposed to be handled, as the inner loop doesn't
7388 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7389 OMP_CLAUSE__LOOPTEMP_
);
7390 gcc_assert (innerc
);
7391 for (i
= 0; i
< fd
->collapse
; i
++)
7393 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7394 OMP_CLAUSE__LOOPTEMP_
);
7395 gcc_assert (innerc
);
7397 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7399 counts
[0] = NULL_TREE
;
7404 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7406 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7407 counts
[i
] = NULL_TREE
;
7408 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7409 fold_convert (itype
, fd
->loops
[i
].n1
),
7410 fold_convert (itype
, fd
->loops
[i
].n2
));
7411 if (t
&& integer_zerop (t
))
7413 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7414 counts
[i
] = build_int_cst (type
, 0);
7418 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7420 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7422 if (i
>= fd
->collapse
&& counts
[i
])
7424 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7425 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7426 fold_convert (itype
, fd
->loops
[i
].n1
),
7427 fold_convert (itype
, fd
->loops
[i
].n2
)))
7428 == NULL_TREE
|| !integer_onep (t
)))
7432 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7433 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7434 true, GSI_SAME_STMT
);
7435 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7436 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7437 true, GSI_SAME_STMT
);
7438 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7439 NULL_TREE
, NULL_TREE
);
7440 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7441 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7442 expand_omp_regimplify_p
, NULL
, NULL
)
7443 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7444 expand_omp_regimplify_p
, NULL
, NULL
))
7446 *gsi
= gsi_for_stmt (cond_stmt
);
7447 gimple_regimplify_operands (cond_stmt
, gsi
);
7449 e
= split_block (entry_bb
, cond_stmt
);
7450 basic_block
&zero_iter_bb
7451 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7452 int &first_zero_iter
7453 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7454 if (zero_iter_bb
== NULL
)
7456 gassign
*assign_stmt
;
7457 first_zero_iter
= i
;
7458 zero_iter_bb
= create_empty_bb (entry_bb
);
7459 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7460 *gsi
= gsi_after_labels (zero_iter_bb
);
7461 if (i
< fd
->collapse
)
7462 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7463 build_zero_cst (type
));
7466 counts
[i
] = create_tmp_reg (type
, ".count");
7468 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7470 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7471 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7474 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7475 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7476 e
->flags
= EDGE_TRUE_VALUE
;
7477 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7478 if (l2_dom_bb
== NULL
)
7479 l2_dom_bb
= entry_bb
;
7481 *gsi
= gsi_last_bb (entry_bb
);
7484 if (POINTER_TYPE_P (itype
))
7485 itype
= signed_type_for (itype
);
7486 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7488 t
= fold_build2 (PLUS_EXPR
, itype
,
7489 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7490 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7491 fold_convert (itype
, fd
->loops
[i
].n2
));
7492 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7493 fold_convert (itype
, fd
->loops
[i
].n1
));
7494 /* ?? We could probably use CEIL_DIV_EXPR instead of
7495 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7496 generate the same code in the end because generically we
7497 don't know that the values involved must be negative for
7499 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7500 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7501 fold_build1 (NEGATE_EXPR
, itype
, t
),
7502 fold_build1 (NEGATE_EXPR
, itype
,
7503 fold_convert (itype
,
7504 fd
->loops
[i
].step
)));
7506 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7507 fold_convert (itype
, fd
->loops
[i
].step
));
7508 t
= fold_convert (type
, t
);
7509 if (TREE_CODE (t
) == INTEGER_CST
)
7513 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7514 counts
[i
] = create_tmp_reg (type
, ".count");
7515 expand_omp_build_assign (gsi
, counts
[i
], t
);
7517 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7522 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7523 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7529 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7531 V3 = N31 + (T % count3) * STEP3;
7533 V2 = N21 + (T % count2) * STEP2;
7535 V1 = N11 + T * STEP1;
7536 if this loop doesn't have an inner loop construct combined with it.
7537 If it does have an inner loop construct combined with it and the
7538 iteration count isn't known constant, store values from counts array
7539 into its _looptemp_ temporaries instead. */
7542 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7543 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7546 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7548 /* If fd->loop.n2 is constant, then no propagation of the counts
7549 is needed, they are constant. */
7550 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7553 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7554 ? gimple_omp_taskreg_clauses (inner_stmt
)
7555 : gimple_omp_for_clauses (inner_stmt
);
7556 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7557 isn't supposed to be handled, as the inner loop doesn't
7559 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7560 gcc_assert (innerc
);
7561 for (i
= 0; i
< fd
->collapse
; i
++)
7563 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7564 OMP_CLAUSE__LOOPTEMP_
);
7565 gcc_assert (innerc
);
7568 tree tem
= OMP_CLAUSE_DECL (innerc
);
7569 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7570 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7571 false, GSI_CONTINUE_LINKING
);
7572 gassign
*stmt
= gimple_build_assign (tem
, t
);
7573 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7579 tree type
= TREE_TYPE (fd
->loop
.v
);
7580 tree tem
= create_tmp_reg (type
, ".tem");
7581 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7582 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7584 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7586 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7588 if (POINTER_TYPE_P (vtype
))
7589 itype
= signed_type_for (vtype
);
7591 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7594 t
= fold_convert (itype
, t
);
7595 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7596 fold_convert (itype
, fd
->loops
[i
].step
));
7597 if (POINTER_TYPE_P (vtype
))
7598 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7600 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7601 t
= force_gimple_operand_gsi (gsi
, t
,
7602 DECL_P (fd
->loops
[i
].v
)
7603 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7605 GSI_CONTINUE_LINKING
);
7606 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7607 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7610 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7611 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7612 false, GSI_CONTINUE_LINKING
);
7613 stmt
= gimple_build_assign (tem
, t
);
7614 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7620 /* Helper function for expand_omp_for_*. Generate code like:
7623 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7627 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7634 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7635 basic_block body_bb
)
7637 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7639 gimple_stmt_iterator gsi
;
7645 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7647 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7649 bb
= create_empty_bb (last_bb
);
7650 add_bb_to_loop (bb
, last_bb
->loop_father
);
7651 gsi
= gsi_start_bb (bb
);
7653 if (i
< fd
->collapse
- 1)
7655 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7656 e
->probability
= REG_BR_PROB_BASE
/ 8;
7658 t
= fd
->loops
[i
+ 1].n1
;
7659 t
= force_gimple_operand_gsi (&gsi
, t
,
7660 DECL_P (fd
->loops
[i
+ 1].v
)
7661 && TREE_ADDRESSABLE (fd
->loops
[i
7664 GSI_CONTINUE_LINKING
);
7665 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7666 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7671 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7673 if (POINTER_TYPE_P (vtype
))
7674 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7676 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7677 t
= force_gimple_operand_gsi (&gsi
, t
,
7678 DECL_P (fd
->loops
[i
].v
)
7679 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7680 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7681 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7682 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7686 t
= fd
->loops
[i
].n2
;
7687 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7688 false, GSI_CONTINUE_LINKING
);
7689 tree v
= fd
->loops
[i
].v
;
7690 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7691 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7692 false, GSI_CONTINUE_LINKING
);
7693 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7694 stmt
= gimple_build_cond_empty (t
);
7695 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7696 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7697 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7700 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7708 /* Expand #pragma omp ordered depend(source). */
7711 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7712 tree
*counts
, location_t loc
)
7714 enum built_in_function source_ix
7715 = fd
->iter_type
== long_integer_type_node
7716 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7718 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7719 build_fold_addr_expr (counts
[fd
->ordered
]));
7720 gimple_set_location (g
, loc
);
7721 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7724 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7727 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7728 tree
*counts
, tree c
, location_t loc
)
7730 auto_vec
<tree
, 10> args
;
7731 enum built_in_function sink_ix
7732 = fd
->iter_type
== long_integer_type_node
7733 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7734 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7736 gimple_stmt_iterator gsi2
= *gsi
;
7737 bool warned_step
= false;
7739 for (i
= 0; i
< fd
->ordered
; i
++)
7741 off
= TREE_PURPOSE (deps
);
7742 if (!integer_zerop (off
))
7744 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7745 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7746 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7747 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7748 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7749 "lexically later iteration");
7752 deps
= TREE_CHAIN (deps
);
7754 /* If all offsets corresponding to the collapsed loops are zero,
7755 this depend clause can be ignored. FIXME: but there is still a
7756 flush needed. We need to emit one __sync_synchronize () for it
7757 though (perhaps conditionally)? Solve this together with the
7758 conservative dependence folding optimization.
7759 if (i >= fd->collapse)
7762 deps
= OMP_CLAUSE_DECL (c
);
7764 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7765 edge e2
= split_block_after_labels (e1
->dest
);
7767 *gsi
= gsi_after_labels (e1
->dest
);
7768 for (i
= 0; i
< fd
->ordered
; i
++)
7770 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7771 if (POINTER_TYPE_P (itype
))
7774 deps
= TREE_CHAIN (deps
);
7775 off
= TREE_PURPOSE (deps
);
7776 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7778 if (integer_zerop (off
))
7779 t
= boolean_true_node
;
7783 tree co
= fold_convert_loc (loc
, itype
, off
);
7784 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7786 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7787 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7788 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7789 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7792 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7793 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7794 fd
->loops
[i
].v
, co
);
7796 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7797 fd
->loops
[i
].v
, co
);
7798 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7800 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7801 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7804 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7807 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7808 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7811 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
7815 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
7819 off
= fold_convert_loc (loc
, itype
, off
);
7821 if (fd
->loops
[i
].cond_code
== LT_EXPR
7822 ? !integer_onep (fd
->loops
[i
].step
)
7823 : !integer_minus_onep (fd
->loops
[i
].step
))
7825 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7826 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
7827 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7830 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
7831 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
7832 build_int_cst (itype
, 0));
7833 if (integer_zerop (t
) && !warned_step
)
7835 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
7836 "in the iteration space");
7839 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
7843 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7849 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7850 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7851 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
7853 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7854 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
7855 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7858 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
7859 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7860 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
7861 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
7862 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7865 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
7867 if (i
< fd
->collapse
- 1)
7869 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
7874 off
= unshare_expr (off
);
7875 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
7876 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
7877 true, GSI_SAME_STMT
);
7880 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
7881 gimple_set_location (g
, loc
);
7882 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7884 *gsi
= gsi_last_bb (e1
->src
);
7885 cond
= unshare_expr (cond
);
7886 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
7887 GSI_CONTINUE_LINKING
);
7888 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
7889 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
7890 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7891 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7892 e1
->flags
= EDGE_TRUE_VALUE
;
7893 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
7895 *gsi
= gsi_after_labels (e2
->dest
);
7898 /* Expand all #pragma omp ordered depend(source) and
7899 #pragma omp ordered depend(sink:...) constructs in the current
7900 #pragma omp for ordered(n) region. */
7903 expand_omp_ordered_source_sink (struct omp_region
*region
,
7904 struct omp_for_data
*fd
, tree
*counts
,
7905 basic_block cont_bb
)
7907 struct omp_region
*inner
;
7909 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
7910 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
7911 counts
[i
] = NULL_TREE
;
7912 else if (i
>= fd
->collapse
&& !cont_bb
)
7913 counts
[i
] = build_zero_cst (fd
->iter_type
);
7914 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
7915 && integer_onep (fd
->loops
[i
].step
))
7916 counts
[i
] = NULL_TREE
;
7918 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
7920 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
7921 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
7922 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
7924 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
7925 if (inner
->type
== GIMPLE_OMP_ORDERED
)
7927 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
7928 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
7929 location_t loc
= gimple_location (ord_stmt
);
7931 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7932 c
; c
= OMP_CLAUSE_CHAIN (c
))
7933 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
7936 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
7937 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7938 c
; c
= OMP_CLAUSE_CHAIN (c
))
7939 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
7940 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
7941 gsi_remove (&gsi
, true);
7945 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
7949 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
7950 basic_block cont_bb
, basic_block body_bb
,
7951 bool ordered_lastprivate
)
7953 if (fd
->ordered
== fd
->collapse
)
7958 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7959 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7961 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7962 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
7963 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
7964 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7965 size_int (i
- fd
->collapse
+ 1),
7966 NULL_TREE
, NULL_TREE
);
7967 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7972 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
7974 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
7975 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7976 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7977 fold_convert (type
, fd
->loops
[i
].n1
));
7979 expand_omp_build_assign (&gsi
, counts
[i
],
7980 build_zero_cst (fd
->iter_type
));
7981 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7982 size_int (i
- fd
->collapse
+ 1),
7983 NULL_TREE
, NULL_TREE
);
7984 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7985 if (!gsi_end_p (gsi
))
7988 gsi
= gsi_last_bb (body_bb
);
7989 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
7990 basic_block new_body
= e1
->dest
;
7991 if (body_bb
== cont_bb
)
7994 basic_block new_header
;
7995 if (EDGE_COUNT (cont_bb
->preds
) > 0)
7997 gsi
= gsi_last_bb (cont_bb
);
7998 if (POINTER_TYPE_P (type
))
7999 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8000 fold_convert (sizetype
,
8001 fd
->loops
[i
].step
));
8003 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8004 fold_convert (type
, fd
->loops
[i
].step
));
8005 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8008 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8009 build_int_cst (fd
->iter_type
, 1));
8010 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8015 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8016 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8017 t
= fold_convert (fd
->iter_type
, t
);
8018 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8019 true, GSI_SAME_STMT
);
8021 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8022 size_int (i
- fd
->collapse
+ 1),
8023 NULL_TREE
, NULL_TREE
);
8024 expand_omp_build_assign (&gsi
, aref
, t
);
8026 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8027 new_header
= e2
->dest
;
8030 new_header
= cont_bb
;
8031 gsi
= gsi_after_labels (new_header
);
8032 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8033 true, GSI_SAME_STMT
);
8035 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8036 true, NULL_TREE
, true, GSI_SAME_STMT
);
8037 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8038 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8039 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8042 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8043 e3
->flags
= EDGE_FALSE_VALUE
;
8044 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8045 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8046 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8048 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8049 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8053 struct loop
*loop
= alloc_loop ();
8054 loop
->header
= new_header
;
8055 loop
->latch
= e2
->src
;
8056 add_loop (loop
, body_bb
->loop_father
);
8060 /* If there are any lastprivate clauses and it is possible some loops
8061 might have zero iterations, ensure all the decls are initialized,
8062 otherwise we could crash evaluating C++ class iterators with lastprivate
8064 bool need_inits
= false;
8065 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8068 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8069 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8070 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8071 fold_convert (type
, fd
->loops
[i
].n1
));
8075 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8076 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8078 fold_convert (type
, fd
->loops
[i
].n1
),
8079 fold_convert (type
, fd
->loops
[i
].n2
));
8080 if (!integer_onep (this_cond
))
8088 /* A subroutine of expand_omp_for. Generate code for a parallel
8089 loop with any schedule. Given parameters:
8091 for (V = N1; V cond N2; V += STEP) BODY;
8093 where COND is "<" or ">", we generate pseudocode
8095 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8096 if (more) goto L0; else goto L3;
8103 if (V cond iend) goto L1; else goto L2;
8105 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8108 If this is a combined omp parallel loop, instead of the call to
8109 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8110 If this is gimple_omp_for_combined_p loop, then instead of assigning
8111 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8112 inner GIMPLE_OMP_FOR and V += STEP; and
8113 if (V cond iend) goto L1; else goto L2; are removed.
8115 For collapsed loops, given parameters:
8117 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8118 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8119 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8122 we generate pseudocode
8124 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8129 count3 = (adj + N32 - N31) / STEP3;
8130 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8135 count2 = (adj + N22 - N21) / STEP2;
8136 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8141 count1 = (adj + N12 - N11) / STEP1;
8142 count = count1 * count2 * count3;
8147 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8148 if (more) goto L0; else goto L3;
8152 V3 = N31 + (T % count3) * STEP3;
8154 V2 = N21 + (T % count2) * STEP2;
8156 V1 = N11 + T * STEP1;
8161 if (V < iend) goto L10; else goto L2;
8164 if (V3 cond3 N32) goto L1; else goto L11;
8168 if (V2 cond2 N22) goto L1; else goto L12;
8174 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8180 expand_omp_for_generic (struct omp_region
*region
,
8181 struct omp_for_data
*fd
,
8182 enum built_in_function start_fn
,
8183 enum built_in_function next_fn
,
8186 tree type
, istart0
, iend0
, iend
;
8187 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8188 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8189 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8190 gimple_stmt_iterator gsi
;
8191 gassign
*assign_stmt
;
8192 bool in_combined_parallel
= is_combined_parallel (region
);
8193 bool broken_loop
= region
->cont
== NULL
;
8195 tree
*counts
= NULL
;
8197 bool ordered_lastprivate
= false;
8199 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8200 gcc_assert (fd
->iter_type
== long_integer_type_node
8201 || !in_combined_parallel
);
8203 entry_bb
= region
->entry
;
8204 cont_bb
= region
->cont
;
8206 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8207 gcc_assert (broken_loop
8208 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8209 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8210 l1_bb
= single_succ (l0_bb
);
8213 l2_bb
= create_empty_bb (cont_bb
);
8214 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8215 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8217 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8221 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8222 exit_bb
= region
->exit
;
8224 gsi
= gsi_last_bb (entry_bb
);
8226 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8228 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8229 OMP_CLAUSE_LASTPRIVATE
))
8230 ordered_lastprivate
= false;
8231 if (fd
->collapse
> 1 || fd
->ordered
)
8233 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8234 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8236 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8237 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8238 zero_iter1_bb
, first_zero_iter1
,
8239 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8243 /* Some counts[i] vars might be uninitialized if
8244 some loop has zero iterations. But the body shouldn't
8245 be executed in that case, so just avoid uninit warnings. */
8246 for (i
= first_zero_iter1
;
8247 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8248 if (SSA_VAR_P (counts
[i
]))
8249 TREE_NO_WARNING (counts
[i
]) = 1;
8251 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8253 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8254 gsi
= gsi_last_bb (entry_bb
);
8255 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8256 get_immediate_dominator (CDI_DOMINATORS
,
8261 /* Some counts[i] vars might be uninitialized if
8262 some loop has zero iterations. But the body shouldn't
8263 be executed in that case, so just avoid uninit warnings. */
8264 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8265 if (SSA_VAR_P (counts
[i
]))
8266 TREE_NO_WARNING (counts
[i
]) = 1;
8268 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8272 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8274 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8275 gsi
= gsi_last_bb (entry_bb
);
8276 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8277 get_immediate_dominator
8278 (CDI_DOMINATORS
, zero_iter2_bb
));
8281 if (fd
->collapse
== 1)
8283 counts
[0] = fd
->loop
.n2
;
8284 fd
->loop
= fd
->loops
[0];
8288 type
= TREE_TYPE (fd
->loop
.v
);
8289 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8290 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8291 TREE_ADDRESSABLE (istart0
) = 1;
8292 TREE_ADDRESSABLE (iend0
) = 1;
8294 /* See if we need to bias by LLONG_MIN. */
8295 if (fd
->iter_type
== long_long_unsigned_type_node
8296 && TREE_CODE (type
) == INTEGER_TYPE
8297 && !TYPE_UNSIGNED (type
)
8298 && fd
->ordered
== 0)
8302 if (fd
->loop
.cond_code
== LT_EXPR
)
8305 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8309 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8312 if (TREE_CODE (n1
) != INTEGER_CST
8313 || TREE_CODE (n2
) != INTEGER_CST
8314 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8315 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8318 gimple_stmt_iterator gsif
= gsi
;
8321 tree arr
= NULL_TREE
;
8322 if (in_combined_parallel
)
8324 gcc_assert (fd
->ordered
== 0);
8325 /* In a combined parallel loop, emit a call to
8326 GOMP_loop_foo_next. */
8327 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8328 build_fold_addr_expr (istart0
),
8329 build_fold_addr_expr (iend0
));
8333 tree t0
, t1
, t2
, t3
, t4
;
8334 /* If this is not a combined parallel loop, emit a call to
8335 GOMP_loop_foo_start in ENTRY_BB. */
8336 t4
= build_fold_addr_expr (iend0
);
8337 t3
= build_fold_addr_expr (istart0
);
8340 t0
= build_int_cst (unsigned_type_node
,
8341 fd
->ordered
- fd
->collapse
+ 1);
8342 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8344 - fd
->collapse
+ 1),
8346 DECL_NAMELESS (arr
) = 1;
8347 TREE_ADDRESSABLE (arr
) = 1;
8348 TREE_STATIC (arr
) = 1;
8349 vec
<constructor_elt
, va_gc
> *v
;
8350 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8353 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8356 if (idx
== 0 && fd
->collapse
> 1)
8359 c
= counts
[idx
+ fd
->collapse
- 1];
8360 tree purpose
= size_int (idx
);
8361 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8362 if (TREE_CODE (c
) != INTEGER_CST
)
8363 TREE_STATIC (arr
) = 0;
8366 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8367 if (!TREE_STATIC (arr
))
8368 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8369 void_type_node
, arr
),
8370 true, NULL_TREE
, true, GSI_SAME_STMT
);
8371 t1
= build_fold_addr_expr (arr
);
8376 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8379 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8382 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8383 OMP_CLAUSE__LOOPTEMP_
);
8384 gcc_assert (innerc
);
8385 t0
= OMP_CLAUSE_DECL (innerc
);
8386 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8387 OMP_CLAUSE__LOOPTEMP_
);
8388 gcc_assert (innerc
);
8389 t1
= OMP_CLAUSE_DECL (innerc
);
8391 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8392 && TYPE_PRECISION (TREE_TYPE (t0
))
8393 != TYPE_PRECISION (fd
->iter_type
))
8395 /* Avoid casting pointers to integer of a different size. */
8396 tree itype
= signed_type_for (type
);
8397 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8398 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8402 t1
= fold_convert (fd
->iter_type
, t1
);
8403 t0
= fold_convert (fd
->iter_type
, t0
);
8407 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8408 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8411 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8415 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8416 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8418 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8419 5, t0
, t1
, t
, t3
, t4
);
8421 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8422 6, t0
, t1
, t2
, t
, t3
, t4
);
8424 else if (fd
->ordered
)
8425 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8428 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8429 5, t0
, t1
, t2
, t3
, t4
);
8437 /* The GOMP_loop_ull_*start functions have additional boolean
8438 argument, true for < loops and false for > loops.
8439 In Fortran, the C bool type can be different from
8440 boolean_type_node. */
8441 bfn_decl
= builtin_decl_explicit (start_fn
);
8442 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8443 t5
= build_int_cst (c_bool_type
,
8444 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8447 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8448 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8449 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8450 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8453 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8454 6, t5
, t0
, t1
, t2
, t3
, t4
);
8457 if (TREE_TYPE (t
) != boolean_type_node
)
8458 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8459 t
, build_int_cst (TREE_TYPE (t
), 0));
8460 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8461 true, GSI_SAME_STMT
);
8462 if (arr
&& !TREE_STATIC (arr
))
8464 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8465 TREE_THIS_VOLATILE (clobber
) = 1;
8466 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8469 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8471 /* Remove the GIMPLE_OMP_FOR statement. */
8472 gsi_remove (&gsi
, true);
8474 if (gsi_end_p (gsif
))
8475 gsif
= gsi_after_labels (gsi_bb (gsif
));
8478 /* Iteration setup for sequential loop goes in L0_BB. */
8479 tree startvar
= fd
->loop
.v
;
8480 tree endvar
= NULL_TREE
;
8482 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8484 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8485 && gimple_omp_for_kind (inner_stmt
)
8486 == GF_OMP_FOR_KIND_SIMD
);
8487 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8488 OMP_CLAUSE__LOOPTEMP_
);
8489 gcc_assert (innerc
);
8490 startvar
= OMP_CLAUSE_DECL (innerc
);
8491 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8492 OMP_CLAUSE__LOOPTEMP_
);
8493 gcc_assert (innerc
);
8494 endvar
= OMP_CLAUSE_DECL (innerc
);
8497 gsi
= gsi_start_bb (l0_bb
);
8499 if (fd
->ordered
&& fd
->collapse
== 1)
8500 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8501 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8503 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8504 if (fd
->ordered
&& fd
->collapse
== 1)
8506 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8507 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8508 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8511 t
= fold_convert (TREE_TYPE (startvar
), t
);
8512 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8518 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8519 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8520 t
= fold_convert (TREE_TYPE (startvar
), t
);
8522 t
= force_gimple_operand_gsi (&gsi
, t
,
8524 && TREE_ADDRESSABLE (startvar
),
8525 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8526 assign_stmt
= gimple_build_assign (startvar
, t
);
8527 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8530 if (fd
->ordered
&& fd
->collapse
== 1)
8531 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8532 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8534 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8535 if (fd
->ordered
&& fd
->collapse
== 1)
8537 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8538 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8539 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8542 t
= fold_convert (TREE_TYPE (startvar
), t
);
8543 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8549 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8550 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8551 t
= fold_convert (TREE_TYPE (startvar
), t
);
8553 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8554 false, GSI_CONTINUE_LINKING
);
8557 assign_stmt
= gimple_build_assign (endvar
, iend
);
8558 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8559 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8560 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8562 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8563 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8565 /* Handle linear clause adjustments. */
8566 tree itercnt
= NULL_TREE
;
8567 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8568 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8569 c
; c
= OMP_CLAUSE_CHAIN (c
))
8570 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8571 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8573 tree d
= OMP_CLAUSE_DECL (c
);
8574 bool is_ref
= is_reference (d
);
8575 tree t
= d
, a
, dest
;
8577 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8578 tree type
= TREE_TYPE (t
);
8579 if (POINTER_TYPE_P (type
))
8581 dest
= unshare_expr (t
);
8582 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8583 expand_omp_build_assign (&gsif
, v
, t
);
8584 if (itercnt
== NULL_TREE
)
8587 tree n1
= fd
->loop
.n1
;
8588 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8591 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8593 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8595 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8597 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8598 itercnt
, fd
->loop
.step
);
8599 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8601 GSI_CONTINUE_LINKING
);
8603 a
= fold_build2 (MULT_EXPR
, type
,
8604 fold_convert (type
, itercnt
),
8605 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8606 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8607 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8608 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8609 false, GSI_CONTINUE_LINKING
);
8610 assign_stmt
= gimple_build_assign (dest
, t
);
8611 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8613 if (fd
->collapse
> 1)
8614 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8618 /* Until now, counts array contained number of iterations or
8619 variable containing it for ith loop. From now on, we need
8620 those counts only for collapsed loops, and only for the 2nd
8621 till the last collapsed one. Move those one element earlier,
8622 we'll use counts[fd->collapse - 1] for the first source/sink
8623 iteration counter and so on and counts[fd->ordered]
8624 as the array holding the current counter values for
8626 if (fd
->collapse
> 1)
8627 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8631 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8633 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8635 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8636 fold_convert (type
, fd
->loops
[i
].n1
),
8637 fold_convert (type
, fd
->loops
[i
].n2
));
8638 if (!integer_onep (this_cond
))
8641 if (i
< fd
->ordered
)
8644 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8645 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8646 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8647 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8648 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8649 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8650 make_edge (cont_bb
, l1_bb
, 0);
8651 l2_bb
= create_empty_bb (cont_bb
);
8652 broken_loop
= false;
8655 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8656 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8657 ordered_lastprivate
);
8658 if (counts
[fd
->collapse
- 1])
8660 gcc_assert (fd
->collapse
== 1);
8661 gsi
= gsi_last_bb (l0_bb
);
8662 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8664 gsi
= gsi_last_bb (cont_bb
);
8665 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8666 build_int_cst (fd
->iter_type
, 1));
8667 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8668 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8669 size_zero_node
, NULL_TREE
, NULL_TREE
);
8670 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8671 t
= counts
[fd
->collapse
- 1];
8673 else if (fd
->collapse
> 1)
8677 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8678 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8679 t
= fold_convert (fd
->iter_type
, t
);
8681 gsi
= gsi_last_bb (l0_bb
);
8682 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8683 size_zero_node
, NULL_TREE
, NULL_TREE
);
8684 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8685 false, GSI_CONTINUE_LINKING
);
8686 expand_omp_build_assign (&gsi
, aref
, t
, true);
8691 /* Code to control the increment and predicate for the sequential
8692 loop goes in the CONT_BB. */
8693 gsi
= gsi_last_bb (cont_bb
);
8694 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8695 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8696 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8697 vback
= gimple_omp_continue_control_def (cont_stmt
);
8699 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8701 if (POINTER_TYPE_P (type
))
8702 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8704 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8705 t
= force_gimple_operand_gsi (&gsi
, t
,
8707 && TREE_ADDRESSABLE (vback
),
8708 NULL_TREE
, true, GSI_SAME_STMT
);
8709 assign_stmt
= gimple_build_assign (vback
, t
);
8710 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8712 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8714 if (fd
->collapse
> 1)
8718 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8719 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8720 t
= fold_convert (fd
->iter_type
, t
);
8722 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8723 counts
[fd
->ordered
], size_zero_node
,
8724 NULL_TREE
, NULL_TREE
);
8725 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8726 true, GSI_SAME_STMT
);
8727 expand_omp_build_assign (&gsi
, aref
, t
);
8730 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8731 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8733 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8734 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8737 /* Remove GIMPLE_OMP_CONTINUE. */
8738 gsi_remove (&gsi
, true);
8740 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8741 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8743 /* Emit code to get the next parallel iteration in L2_BB. */
8744 gsi
= gsi_start_bb (l2_bb
);
8746 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8747 build_fold_addr_expr (istart0
),
8748 build_fold_addr_expr (iend0
));
8749 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8750 false, GSI_CONTINUE_LINKING
);
8751 if (TREE_TYPE (t
) != boolean_type_node
)
8752 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8753 t
, build_int_cst (TREE_TYPE (t
), 0));
8754 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8755 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8758 /* Add the loop cleanup function. */
8759 gsi
= gsi_last_bb (exit_bb
);
8760 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8761 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8762 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8763 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8765 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8766 gcall
*call_stmt
= gimple_build_call (t
, 0);
8767 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8768 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8769 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8772 tree arr
= counts
[fd
->ordered
];
8773 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8774 TREE_THIS_VOLATILE (clobber
) = 1;
8775 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8778 gsi_remove (&gsi
, true);
8780 /* Connect the new blocks. */
8781 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8782 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8788 e
= find_edge (cont_bb
, l3_bb
);
8789 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8791 phis
= phi_nodes (l3_bb
);
8792 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8794 gimple
*phi
= gsi_stmt (gsi
);
8795 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8796 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8800 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8801 e
= find_edge (cont_bb
, l1_bb
);
8804 e
= BRANCH_EDGE (cont_bb
);
8805 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8807 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8812 else if (fd
->collapse
> 1)
8815 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
8818 e
->flags
= EDGE_TRUE_VALUE
;
8821 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
8822 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
8826 e
= find_edge (cont_bb
, l2_bb
);
8827 e
->flags
= EDGE_FALLTHRU
;
8829 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
8831 if (gimple_in_ssa_p (cfun
))
8833 /* Add phis to the outer loop that connect to the phis in the inner,
8834 original loop, and move the loop entry value of the inner phi to
8835 the loop entry value of the outer phi. */
8837 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
8839 source_location locus
;
8841 gphi
*exit_phi
= psi
.phi ();
8843 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
8844 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
8846 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
8847 edge latch_to_l1
= find_edge (latch
, l1_bb
);
8849 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
8851 tree t
= gimple_phi_result (exit_phi
);
8852 tree new_res
= copy_ssa_name (t
, NULL
);
8853 nphi
= create_phi_node (new_res
, l0_bb
);
8855 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
8856 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
8857 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
8858 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
8859 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
8861 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
8862 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
8864 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
8868 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
8869 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
8870 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
8871 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
8872 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
8873 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
8874 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
8875 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
8877 /* We enter expand_omp_for_generic with a loop. This original loop may
8878 have its own loop struct, or it may be part of an outer loop struct
8879 (which may be the fake loop). */
8880 struct loop
*outer_loop
= entry_bb
->loop_father
;
8881 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
8883 add_bb_to_loop (l2_bb
, outer_loop
);
8885 /* We've added a new loop around the original loop. Allocate the
8886 corresponding loop struct. */
8887 struct loop
*new_loop
= alloc_loop ();
8888 new_loop
->header
= l0_bb
;
8889 new_loop
->latch
= l2_bb
;
8890 add_loop (new_loop
, outer_loop
);
8892 /* Allocate a loop structure for the original loop unless we already
8894 if (!orig_loop_has_loop_struct
8895 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8897 struct loop
*orig_loop
= alloc_loop ();
8898 orig_loop
->header
= l1_bb
;
8899 /* The loop may have multiple latches. */
8900 add_loop (orig_loop
, new_loop
);
8906 /* A subroutine of expand_omp_for. Generate code for a parallel
8907 loop with static schedule and no specified chunk size. Given
8910 for (V = N1; V cond N2; V += STEP) BODY;
8912 where COND is "<" or ">", we generate pseudocode
8914 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
8919 if ((__typeof (V)) -1 > 0 && cond is >)
8920 n = -(adj + N2 - N1) / -STEP;
8922 n = (adj + N2 - N1) / STEP;
8925 if (threadid < tt) goto L3; else goto L4;
8930 s0 = q * threadid + tt;
8933 if (s0 >= e0) goto L2; else goto L0;
8939 if (V cond e) goto L1;
8944 expand_omp_for_static_nochunk (struct omp_region
*region
,
8945 struct omp_for_data
*fd
,
8948 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
8949 tree type
, itype
, vmain
, vback
;
8950 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
8951 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
8953 gimple_stmt_iterator gsi
;
8955 bool broken_loop
= region
->cont
== NULL
;
8956 tree
*counts
= NULL
;
8959 itype
= type
= TREE_TYPE (fd
->loop
.v
);
8960 if (POINTER_TYPE_P (type
))
8961 itype
= signed_type_for (type
);
8963 entry_bb
= region
->entry
;
8964 cont_bb
= region
->cont
;
8965 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8966 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8967 gcc_assert (broken_loop
8968 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
8969 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8970 body_bb
= single_succ (seq_start_bb
);
8973 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
8974 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
8975 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8977 exit_bb
= region
->exit
;
8979 /* Iteration space partitioning goes in ENTRY_BB. */
8980 gsi
= gsi_last_bb (entry_bb
);
8981 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8983 if (fd
->collapse
> 1)
8985 int first_zero_iter
= -1, dummy
= -1;
8986 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
8988 counts
= XALLOCAVEC (tree
, fd
->collapse
);
8989 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8990 fin_bb
, first_zero_iter
,
8991 dummy_bb
, dummy
, l2_dom_bb
);
8994 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8995 t
= integer_one_node
;
8997 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
8998 fold_convert (type
, fd
->loop
.n1
),
8999 fold_convert (type
, fd
->loop
.n2
));
9000 if (fd
->collapse
== 1
9001 && TYPE_UNSIGNED (type
)
9002 && (t
== NULL_TREE
|| !integer_onep (t
)))
9004 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9005 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9006 true, GSI_SAME_STMT
);
9007 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9008 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9009 true, GSI_SAME_STMT
);
9010 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9011 NULL_TREE
, NULL_TREE
);
9012 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9013 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9014 expand_omp_regimplify_p
, NULL
, NULL
)
9015 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9016 expand_omp_regimplify_p
, NULL
, NULL
))
9018 gsi
= gsi_for_stmt (cond_stmt
);
9019 gimple_regimplify_operands (cond_stmt
, &gsi
);
9021 ep
= split_block (entry_bb
, cond_stmt
);
9022 ep
->flags
= EDGE_TRUE_VALUE
;
9023 entry_bb
= ep
->dest
;
9024 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9025 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9026 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9027 if (gimple_in_ssa_p (cfun
))
9029 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9030 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9031 !gsi_end_p (gpi
); gsi_next (&gpi
))
9033 gphi
*phi
= gpi
.phi ();
9034 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9035 ep
, UNKNOWN_LOCATION
);
9038 gsi
= gsi_last_bb (entry_bb
);
9041 switch (gimple_omp_for_kind (fd
->for_stmt
))
9043 case GF_OMP_FOR_KIND_FOR
:
9044 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9045 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9047 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9048 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9049 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9054 nthreads
= build_call_expr (nthreads
, 0);
9055 nthreads
= fold_convert (itype
, nthreads
);
9056 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9057 true, GSI_SAME_STMT
);
9058 threadid
= build_call_expr (threadid
, 0);
9059 threadid
= fold_convert (itype
, threadid
);
9060 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9061 true, GSI_SAME_STMT
);
9065 step
= fd
->loop
.step
;
9066 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9068 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9069 OMP_CLAUSE__LOOPTEMP_
);
9070 gcc_assert (innerc
);
9071 n1
= OMP_CLAUSE_DECL (innerc
);
9072 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9073 OMP_CLAUSE__LOOPTEMP_
);
9074 gcc_assert (innerc
);
9075 n2
= OMP_CLAUSE_DECL (innerc
);
9077 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9078 true, NULL_TREE
, true, GSI_SAME_STMT
);
9079 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9080 true, NULL_TREE
, true, GSI_SAME_STMT
);
9081 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9082 true, NULL_TREE
, true, GSI_SAME_STMT
);
9084 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9085 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9086 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9087 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9088 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9089 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9090 fold_build1 (NEGATE_EXPR
, itype
, t
),
9091 fold_build1 (NEGATE_EXPR
, itype
, step
));
9093 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9094 t
= fold_convert (itype
, t
);
9095 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9097 q
= create_tmp_reg (itype
, "q");
9098 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9099 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9100 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9102 tt
= create_tmp_reg (itype
, "tt");
9103 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9104 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9105 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9107 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9108 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9109 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9111 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9112 gsi
= gsi_last_bb (second_bb
);
9113 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9115 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9117 gassign
*assign_stmt
9118 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9119 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9121 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9122 gsi
= gsi_last_bb (third_bb
);
9123 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9125 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9126 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9127 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9129 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9130 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9132 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9133 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9135 /* Remove the GIMPLE_OMP_FOR statement. */
9136 gsi_remove (&gsi
, true);
9138 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9139 gsi
= gsi_start_bb (seq_start_bb
);
9141 tree startvar
= fd
->loop
.v
;
9142 tree endvar
= NULL_TREE
;
9144 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9146 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9147 ? gimple_omp_parallel_clauses (inner_stmt
)
9148 : gimple_omp_for_clauses (inner_stmt
);
9149 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9150 gcc_assert (innerc
);
9151 startvar
= OMP_CLAUSE_DECL (innerc
);
9152 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9153 OMP_CLAUSE__LOOPTEMP_
);
9154 gcc_assert (innerc
);
9155 endvar
= OMP_CLAUSE_DECL (innerc
);
9156 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9157 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9160 for (i
= 1; i
< fd
->collapse
; i
++)
9162 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9163 OMP_CLAUSE__LOOPTEMP_
);
9164 gcc_assert (innerc
);
9166 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9167 OMP_CLAUSE__LOOPTEMP_
);
9170 /* If needed (distribute parallel for with lastprivate),
9171 propagate down the total number of iterations. */
9172 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9174 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9175 GSI_CONTINUE_LINKING
);
9176 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9177 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9181 t
= fold_convert (itype
, s0
);
9182 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9183 if (POINTER_TYPE_P (type
))
9184 t
= fold_build_pointer_plus (n1
, t
);
9186 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9187 t
= fold_convert (TREE_TYPE (startvar
), t
);
9188 t
= force_gimple_operand_gsi (&gsi
, t
,
9190 && TREE_ADDRESSABLE (startvar
),
9191 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9192 assign_stmt
= gimple_build_assign (startvar
, t
);
9193 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9195 t
= fold_convert (itype
, e0
);
9196 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9197 if (POINTER_TYPE_P (type
))
9198 t
= fold_build_pointer_plus (n1
, t
);
9200 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9201 t
= fold_convert (TREE_TYPE (startvar
), t
);
9202 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9203 false, GSI_CONTINUE_LINKING
);
9206 assign_stmt
= gimple_build_assign (endvar
, e
);
9207 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9208 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9209 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9211 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9212 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9214 /* Handle linear clause adjustments. */
9215 tree itercnt
= NULL_TREE
;
9216 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9217 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9218 c
; c
= OMP_CLAUSE_CHAIN (c
))
9219 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9220 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9222 tree d
= OMP_CLAUSE_DECL (c
);
9223 bool is_ref
= is_reference (d
);
9224 tree t
= d
, a
, dest
;
9226 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9227 if (itercnt
== NULL_TREE
)
9229 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9231 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9232 fold_convert (itype
, n1
),
9233 fold_convert (itype
, fd
->loop
.n1
));
9234 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9235 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9236 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9238 GSI_CONTINUE_LINKING
);
9243 tree type
= TREE_TYPE (t
);
9244 if (POINTER_TYPE_P (type
))
9246 a
= fold_build2 (MULT_EXPR
, type
,
9247 fold_convert (type
, itercnt
),
9248 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9249 dest
= unshare_expr (t
);
9250 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9251 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9252 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9253 false, GSI_CONTINUE_LINKING
);
9254 assign_stmt
= gimple_build_assign (dest
, t
);
9255 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9257 if (fd
->collapse
> 1)
9258 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9262 /* The code controlling the sequential loop replaces the
9263 GIMPLE_OMP_CONTINUE. */
9264 gsi
= gsi_last_bb (cont_bb
);
9265 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9266 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9267 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9268 vback
= gimple_omp_continue_control_def (cont_stmt
);
9270 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9272 if (POINTER_TYPE_P (type
))
9273 t
= fold_build_pointer_plus (vmain
, step
);
9275 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9276 t
= force_gimple_operand_gsi (&gsi
, t
,
9278 && TREE_ADDRESSABLE (vback
),
9279 NULL_TREE
, true, GSI_SAME_STMT
);
9280 assign_stmt
= gimple_build_assign (vback
, t
);
9281 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9283 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9284 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9286 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9289 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9290 gsi_remove (&gsi
, true);
9292 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9293 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9296 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9297 gsi
= gsi_last_bb (exit_bb
);
9298 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9300 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9301 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9303 gsi_remove (&gsi
, true);
9305 /* Connect all the blocks. */
9306 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9307 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9308 ep
= find_edge (entry_bb
, second_bb
);
9309 ep
->flags
= EDGE_TRUE_VALUE
;
9310 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9311 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9312 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9316 ep
= find_edge (cont_bb
, body_bb
);
9319 ep
= BRANCH_EDGE (cont_bb
);
9320 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9322 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9327 else if (fd
->collapse
> 1)
9330 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9333 ep
->flags
= EDGE_TRUE_VALUE
;
9334 find_edge (cont_bb
, fin_bb
)->flags
9335 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9338 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9339 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9340 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9342 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9343 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9344 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9345 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9347 struct loop
*loop
= body_bb
->loop_father
;
9348 if (loop
!= entry_bb
->loop_father
)
9350 gcc_assert (loop
->header
== body_bb
);
9351 gcc_assert (broken_loop
9352 || loop
->latch
== region
->cont
9353 || single_pred (loop
->latch
) == region
->cont
);
9357 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9359 loop
= alloc_loop ();
9360 loop
->header
= body_bb
;
9361 if (collapse_bb
== NULL
)
9362 loop
->latch
= cont_bb
;
9363 add_loop (loop
, body_bb
->loop_father
);
9367 /* Return phi in E->DEST with ARG on edge E. */
9370 find_phi_with_arg_on_edge (tree arg
, edge e
)
9372 basic_block bb
= e
->dest
;
9374 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9378 gphi
*phi
= gpi
.phi ();
9379 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9386 /* A subroutine of expand_omp_for. Generate code for a parallel
9387 loop with static schedule and a specified chunk size. Given
9390 for (V = N1; V cond N2; V += STEP) BODY;
9392 where COND is "<" or ">", we generate pseudocode
9394 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9399 if ((__typeof (V)) -1 > 0 && cond is >)
9400 n = -(adj + N2 - N1) / -STEP;
9402 n = (adj + N2 - N1) / STEP;
9404 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9405 here so that V is defined
9406 if the loop is not entered
9408 s0 = (trip * nthreads + threadid) * CHUNK;
9409 e0 = min(s0 + CHUNK, n);
9410 if (s0 < n) goto L1; else goto L4;
9417 if (V cond e) goto L2; else goto L3;
9425 expand_omp_for_static_chunk (struct omp_region
*region
,
9426 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9428 tree n
, s0
, e0
, e
, t
;
9429 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9430 tree type
, itype
, vmain
, vback
, vextra
;
9431 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9432 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9433 gimple_stmt_iterator gsi
;
9435 bool broken_loop
= region
->cont
== NULL
;
9436 tree
*counts
= NULL
;
9439 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9440 if (POINTER_TYPE_P (type
))
9441 itype
= signed_type_for (type
);
9443 entry_bb
= region
->entry
;
9444 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9446 iter_part_bb
= se
->dest
;
9447 cont_bb
= region
->cont
;
9448 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9449 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9450 gcc_assert (broken_loop
9451 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9452 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9453 body_bb
= single_succ (seq_start_bb
);
9456 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9457 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9458 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9459 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9461 exit_bb
= region
->exit
;
9463 /* Trip and adjustment setup goes in ENTRY_BB. */
9464 gsi
= gsi_last_bb (entry_bb
);
9465 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9467 if (fd
->collapse
> 1)
9469 int first_zero_iter
= -1, dummy
= -1;
9470 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9472 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9473 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9474 fin_bb
, first_zero_iter
,
9475 dummy_bb
, dummy
, l2_dom_bb
);
9478 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9479 t
= integer_one_node
;
9481 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9482 fold_convert (type
, fd
->loop
.n1
),
9483 fold_convert (type
, fd
->loop
.n2
));
9484 if (fd
->collapse
== 1
9485 && TYPE_UNSIGNED (type
)
9486 && (t
== NULL_TREE
|| !integer_onep (t
)))
9488 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9489 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9490 true, GSI_SAME_STMT
);
9491 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9492 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9493 true, GSI_SAME_STMT
);
9494 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9495 NULL_TREE
, NULL_TREE
);
9496 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9497 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9498 expand_omp_regimplify_p
, NULL
, NULL
)
9499 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9500 expand_omp_regimplify_p
, NULL
, NULL
))
9502 gsi
= gsi_for_stmt (cond_stmt
);
9503 gimple_regimplify_operands (cond_stmt
, &gsi
);
9505 se
= split_block (entry_bb
, cond_stmt
);
9506 se
->flags
= EDGE_TRUE_VALUE
;
9507 entry_bb
= se
->dest
;
9508 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9509 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9510 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9511 if (gimple_in_ssa_p (cfun
))
9513 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9514 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9515 !gsi_end_p (gpi
); gsi_next (&gpi
))
9517 gphi
*phi
= gpi
.phi ();
9518 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9519 se
, UNKNOWN_LOCATION
);
9522 gsi
= gsi_last_bb (entry_bb
);
9525 switch (gimple_omp_for_kind (fd
->for_stmt
))
9527 case GF_OMP_FOR_KIND_FOR
:
9528 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9529 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9531 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9532 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9533 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9538 nthreads
= build_call_expr (nthreads
, 0);
9539 nthreads
= fold_convert (itype
, nthreads
);
9540 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9541 true, GSI_SAME_STMT
);
9542 threadid
= build_call_expr (threadid
, 0);
9543 threadid
= fold_convert (itype
, threadid
);
9544 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9545 true, GSI_SAME_STMT
);
9549 step
= fd
->loop
.step
;
9550 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9552 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9553 OMP_CLAUSE__LOOPTEMP_
);
9554 gcc_assert (innerc
);
9555 n1
= OMP_CLAUSE_DECL (innerc
);
9556 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9557 OMP_CLAUSE__LOOPTEMP_
);
9558 gcc_assert (innerc
);
9559 n2
= OMP_CLAUSE_DECL (innerc
);
9561 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9562 true, NULL_TREE
, true, GSI_SAME_STMT
);
9563 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9564 true, NULL_TREE
, true, GSI_SAME_STMT
);
9565 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9566 true, NULL_TREE
, true, GSI_SAME_STMT
);
9567 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9568 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9570 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9573 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9574 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9575 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9576 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9577 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9578 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9579 fold_build1 (NEGATE_EXPR
, itype
, t
),
9580 fold_build1 (NEGATE_EXPR
, itype
, step
));
9582 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9583 t
= fold_convert (itype
, t
);
9584 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9585 true, GSI_SAME_STMT
);
9587 trip_var
= create_tmp_reg (itype
, ".trip");
9588 if (gimple_in_ssa_p (cfun
))
9590 trip_init
= make_ssa_name (trip_var
);
9591 trip_main
= make_ssa_name (trip_var
);
9592 trip_back
= make_ssa_name (trip_var
);
9596 trip_init
= trip_var
;
9597 trip_main
= trip_var
;
9598 trip_back
= trip_var
;
9601 gassign
*assign_stmt
9602 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9603 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9605 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9606 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9607 if (POINTER_TYPE_P (type
))
9608 t
= fold_build_pointer_plus (n1
, t
);
9610 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9611 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9612 true, GSI_SAME_STMT
);
9614 /* Remove the GIMPLE_OMP_FOR. */
9615 gsi_remove (&gsi
, true);
9617 gimple_stmt_iterator gsif
= gsi
;
9619 /* Iteration space partitioning goes in ITER_PART_BB. */
9620 gsi
= gsi_last_bb (iter_part_bb
);
9622 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9623 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9624 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9625 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9626 false, GSI_CONTINUE_LINKING
);
9628 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9629 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9630 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9631 false, GSI_CONTINUE_LINKING
);
9633 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9634 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9636 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9637 gsi
= gsi_start_bb (seq_start_bb
);
9639 tree startvar
= fd
->loop
.v
;
9640 tree endvar
= NULL_TREE
;
9642 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9644 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9645 ? gimple_omp_parallel_clauses (inner_stmt
)
9646 : gimple_omp_for_clauses (inner_stmt
);
9647 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9648 gcc_assert (innerc
);
9649 startvar
= OMP_CLAUSE_DECL (innerc
);
9650 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9651 OMP_CLAUSE__LOOPTEMP_
);
9652 gcc_assert (innerc
);
9653 endvar
= OMP_CLAUSE_DECL (innerc
);
9654 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9655 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9658 for (i
= 1; i
< fd
->collapse
; i
++)
9660 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9661 OMP_CLAUSE__LOOPTEMP_
);
9662 gcc_assert (innerc
);
9664 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9665 OMP_CLAUSE__LOOPTEMP_
);
9668 /* If needed (distribute parallel for with lastprivate),
9669 propagate down the total number of iterations. */
9670 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9672 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9673 GSI_CONTINUE_LINKING
);
9674 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9675 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9680 t
= fold_convert (itype
, s0
);
9681 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9682 if (POINTER_TYPE_P (type
))
9683 t
= fold_build_pointer_plus (n1
, t
);
9685 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9686 t
= fold_convert (TREE_TYPE (startvar
), t
);
9687 t
= force_gimple_operand_gsi (&gsi
, t
,
9689 && TREE_ADDRESSABLE (startvar
),
9690 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9691 assign_stmt
= gimple_build_assign (startvar
, t
);
9692 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9694 t
= fold_convert (itype
, e0
);
9695 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9696 if (POINTER_TYPE_P (type
))
9697 t
= fold_build_pointer_plus (n1
, t
);
9699 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9700 t
= fold_convert (TREE_TYPE (startvar
), t
);
9701 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9702 false, GSI_CONTINUE_LINKING
);
9705 assign_stmt
= gimple_build_assign (endvar
, e
);
9706 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9707 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9708 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9710 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9711 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9713 /* Handle linear clause adjustments. */
9714 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9715 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9716 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9717 c
; c
= OMP_CLAUSE_CHAIN (c
))
9718 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9719 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9721 tree d
= OMP_CLAUSE_DECL (c
);
9722 bool is_ref
= is_reference (d
);
9723 tree t
= d
, a
, dest
;
9725 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9726 tree type
= TREE_TYPE (t
);
9727 if (POINTER_TYPE_P (type
))
9729 dest
= unshare_expr (t
);
9730 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9731 expand_omp_build_assign (&gsif
, v
, t
);
9732 if (itercnt
== NULL_TREE
)
9734 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9737 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9738 fold_convert (itype
, fd
->loop
.n1
));
9739 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9742 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9745 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9746 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9748 GSI_CONTINUE_LINKING
);
9753 a
= fold_build2 (MULT_EXPR
, type
,
9754 fold_convert (type
, itercnt
),
9755 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9756 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9757 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9758 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9759 false, GSI_CONTINUE_LINKING
);
9760 assign_stmt
= gimple_build_assign (dest
, t
);
9761 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9763 if (fd
->collapse
> 1)
9764 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9768 /* The code controlling the sequential loop goes in CONT_BB,
9769 replacing the GIMPLE_OMP_CONTINUE. */
9770 gsi
= gsi_last_bb (cont_bb
);
9771 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9772 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9773 vback
= gimple_omp_continue_control_def (cont_stmt
);
9775 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9777 if (POINTER_TYPE_P (type
))
9778 t
= fold_build_pointer_plus (vmain
, step
);
9780 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9781 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9782 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9783 true, GSI_SAME_STMT
);
9784 assign_stmt
= gimple_build_assign (vback
, t
);
9785 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9787 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9788 t
= build2 (EQ_EXPR
, boolean_type_node
,
9789 build_int_cst (itype
, 0),
9790 build_int_cst (itype
, 1));
9792 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9793 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9795 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9798 /* Remove GIMPLE_OMP_CONTINUE. */
9799 gsi_remove (&gsi
, true);
9801 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9802 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9804 /* Trip update code goes into TRIP_UPDATE_BB. */
9805 gsi
= gsi_start_bb (trip_update_bb
);
9807 t
= build_int_cst (itype
, 1);
9808 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9809 assign_stmt
= gimple_build_assign (trip_back
, t
);
9810 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9813 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9814 gsi
= gsi_last_bb (exit_bb
);
9815 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9817 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9818 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9820 gsi_remove (&gsi
, true);
9822 /* Connect the new blocks. */
9823 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
9824 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
9828 se
= find_edge (cont_bb
, body_bb
);
9831 se
= BRANCH_EDGE (cont_bb
);
9832 gcc_assert (single_succ (se
->dest
) == body_bb
);
9834 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9839 else if (fd
->collapse
> 1)
9842 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9845 se
->flags
= EDGE_TRUE_VALUE
;
9846 find_edge (cont_bb
, trip_update_bb
)->flags
9847 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9849 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
9852 if (gimple_in_ssa_p (cfun
))
9860 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
9862 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
9863 remove arguments of the phi nodes in fin_bb. We need to create
9864 appropriate phi nodes in iter_part_bb instead. */
9865 se
= find_edge (iter_part_bb
, fin_bb
);
9866 re
= single_succ_edge (trip_update_bb
);
9867 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
9868 ene
= single_succ_edge (entry_bb
);
9870 psi
= gsi_start_phis (fin_bb
);
9871 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
9872 gsi_next (&psi
), ++i
)
9875 source_location locus
;
9878 t
= gimple_phi_result (phi
);
9879 gcc_assert (t
== redirect_edge_var_map_result (vm
));
9881 if (!single_pred_p (fin_bb
))
9882 t
= copy_ssa_name (t
, phi
);
9884 nphi
= create_phi_node (t
, iter_part_bb
);
9886 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
9887 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
9889 /* A special case -- fd->loop.v is not yet computed in
9890 iter_part_bb, we need to use vextra instead. */
9891 if (t
== fd
->loop
.v
)
9893 add_phi_arg (nphi
, t
, ene
, locus
);
9894 locus
= redirect_edge_var_map_location (vm
);
9895 tree back_arg
= redirect_edge_var_map_def (vm
);
9896 add_phi_arg (nphi
, back_arg
, re
, locus
);
9897 edge ce
= find_edge (cont_bb
, body_bb
);
9900 ce
= BRANCH_EDGE (cont_bb
);
9901 gcc_assert (single_succ (ce
->dest
) == body_bb
);
9902 ce
= single_succ_edge (ce
->dest
);
9904 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
9905 gcc_assert (inner_loop_phi
!= NULL
);
9906 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
9907 find_edge (seq_start_bb
, body_bb
), locus
);
9909 if (!single_pred_p (fin_bb
))
9910 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
9912 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
9913 redirect_edge_var_map_clear (re
);
9914 if (single_pred_p (fin_bb
))
9917 psi
= gsi_start_phis (fin_bb
);
9918 if (gsi_end_p (psi
))
9920 remove_phi_node (&psi
, false);
9923 /* Make phi node for trip. */
9924 phi
= create_phi_node (trip_main
, iter_part_bb
);
9925 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
9927 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
9932 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
9933 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
9934 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
9935 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9936 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9937 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
9938 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
9939 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9940 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9944 struct loop
*loop
= body_bb
->loop_father
;
9945 struct loop
*trip_loop
= alloc_loop ();
9946 trip_loop
->header
= iter_part_bb
;
9947 trip_loop
->latch
= trip_update_bb
;
9948 add_loop (trip_loop
, iter_part_bb
->loop_father
);
9950 if (loop
!= entry_bb
->loop_father
)
9952 gcc_assert (loop
->header
== body_bb
);
9953 gcc_assert (loop
->latch
== region
->cont
9954 || single_pred (loop
->latch
) == region
->cont
);
9955 trip_loop
->inner
= loop
;
9959 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9961 loop
= alloc_loop ();
9962 loop
->header
= body_bb
;
9963 if (collapse_bb
== NULL
)
9964 loop
->latch
= cont_bb
;
9965 add_loop (loop
, trip_loop
);
9970 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
9972 for (V = N1; V cond N2; V += STEP) BODY;
9974 where COND is "<" or ">" or "!=", we generate pseudocode
9976 for (ind_var = low; ind_var < high; ind_var++)
9978 V = n1 + (ind_var * STEP)
9983 In the above pseudocode, low and high are function parameters of the
9984 child function. In the function below, we are inserting a temp.
9985 variable that will be making a call to two OMP functions that will not be
9986 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
9987 with _Cilk_for). These functions are replaced with low and high
9988 by the function that handles taskreg. */
9992 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
9994 bool broken_loop
= region
->cont
== NULL
;
9995 basic_block entry_bb
= region
->entry
;
9996 basic_block cont_bb
= region
->cont
;
9998 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9999 gcc_assert (broken_loop
10000 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10001 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10002 basic_block l1_bb
, l2_bb
;
10006 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10007 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10008 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10009 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10013 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10014 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10015 l2_bb
= single_succ (l1_bb
);
10017 basic_block exit_bb
= region
->exit
;
10018 basic_block l2_dom_bb
= NULL
;
10020 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10022 /* Below statements until the "tree high_val = ..." are pseudo statements
10023 used to pass information to be used by expand_omp_taskreg.
10024 low_val and high_val will be replaced by the __low and __high
10025 parameter from the child function.
10027 The call_exprs part is a place-holder, it is mainly used
10028 to distinctly identify to the top-level part that this is
10029 where we should put low and high (reasoning given in header
10033 = gimple_omp_parallel_child_fn (
10034 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10035 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10036 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10038 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10040 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10043 gcc_assert (low_val
&& high_val
);
10045 tree type
= TREE_TYPE (low_val
);
10046 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10047 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10049 /* Not needed in SSA form right now. */
10050 gcc_assert (!gimple_in_ssa_p (cfun
));
10051 if (l2_dom_bb
== NULL
)
10055 tree n2
= high_val
;
10057 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10059 /* Replace the GIMPLE_OMP_FOR statement. */
10060 gsi_replace (&gsi
, stmt
, true);
10064 /* Code to control the increment goes in the CONT_BB. */
10065 gsi
= gsi_last_bb (cont_bb
);
10066 stmt
= gsi_stmt (gsi
);
10067 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10068 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10069 build_one_cst (type
));
10071 /* Replace GIMPLE_OMP_CONTINUE. */
10072 gsi_replace (&gsi
, stmt
, true);
10075 /* Emit the condition in L1_BB. */
10076 gsi
= gsi_after_labels (l1_bb
);
10077 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10078 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10080 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10081 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10082 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10084 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10085 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10086 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10087 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10089 /* The condition is always '<' since the runtime will fill in the low
10090 and high values. */
10091 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10092 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10094 /* Remove GIMPLE_OMP_RETURN. */
10095 gsi
= gsi_last_bb (exit_bb
);
10096 gsi_remove (&gsi
, true);
10098 /* Connect the new blocks. */
10099 remove_edge (FALLTHRU_EDGE (entry_bb
));
10104 remove_edge (BRANCH_EDGE (entry_bb
));
10105 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10107 e
= BRANCH_EDGE (l1_bb
);
10108 ne
= FALLTHRU_EDGE (l1_bb
);
10109 e
->flags
= EDGE_TRUE_VALUE
;
10113 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10115 ne
= single_succ_edge (l1_bb
);
10116 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10119 ne
->flags
= EDGE_FALSE_VALUE
;
10120 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10121 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10123 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10124 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10125 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10129 struct loop
*loop
= alloc_loop ();
10130 loop
->header
= l1_bb
;
10131 loop
->latch
= cont_bb
;
10132 add_loop (loop
, l1_bb
->loop_father
);
10133 loop
->safelen
= INT_MAX
;
10136 /* Pick the correct library function based on the precision of the
10137 induction variable type. */
10138 tree lib_fun
= NULL_TREE
;
10139 if (TYPE_PRECISION (type
) == 32)
10140 lib_fun
= cilk_for_32_fndecl
;
10141 else if (TYPE_PRECISION (type
) == 64)
10142 lib_fun
= cilk_for_64_fndecl
;
10144 gcc_unreachable ();
10146 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10148 /* WS_ARGS contains the library function flavor to call:
10149 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10150 user-defined grain value. If the user does not define one, then zero
10151 is passed in by the parser. */
10152 vec_alloc (region
->ws_args
, 2);
10153 region
->ws_args
->quick_push (lib_fun
);
10154 region
->ws_args
->quick_push (fd
->chunk_size
);
10157 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10158 loop. Given parameters:
10160 for (V = N1; V cond N2; V += STEP) BODY;
10162 where COND is "<" or ">", we generate pseudocode
10170 if (V cond N2) goto L0; else goto L2;
10173 For collapsed loops, given parameters:
10175 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10176 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10177 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10180 we generate pseudocode
10186 count3 = (adj + N32 - N31) / STEP3;
10191 count2 = (adj + N22 - N21) / STEP2;
10196 count1 = (adj + N12 - N11) / STEP1;
10197 count = count1 * count2 * count3;
10207 V2 += (V3 cond3 N32) ? 0 : STEP2;
10208 V3 = (V3 cond3 N32) ? V3 : N31;
10209 V1 += (V2 cond2 N22) ? 0 : STEP1;
10210 V2 = (V2 cond2 N22) ? V2 : N21;
10212 if (V < count) goto L0; else goto L2;
10218 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10221 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10222 gimple_stmt_iterator gsi
;
10225 bool broken_loop
= region
->cont
== NULL
;
10227 tree
*counts
= NULL
;
10229 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10230 OMP_CLAUSE_SAFELEN
);
10231 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10232 OMP_CLAUSE__SIMDUID_
);
10235 type
= TREE_TYPE (fd
->loop
.v
);
10236 entry_bb
= region
->entry
;
10237 cont_bb
= region
->cont
;
10238 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10239 gcc_assert (broken_loop
10240 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10241 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10244 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10245 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10246 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10247 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10251 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10252 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10253 l2_bb
= single_succ (l1_bb
);
10255 exit_bb
= region
->exit
;
10258 gsi
= gsi_last_bb (entry_bb
);
10260 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10261 /* Not needed in SSA form right now. */
10262 gcc_assert (!gimple_in_ssa_p (cfun
));
10263 if (fd
->collapse
> 1)
10265 int first_zero_iter
= -1, dummy
= -1;
10266 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10268 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10269 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10270 zero_iter_bb
, first_zero_iter
,
10271 dummy_bb
, dummy
, l2_dom_bb
);
10273 if (l2_dom_bb
== NULL
)
10278 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10280 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10281 OMP_CLAUSE__LOOPTEMP_
);
10282 gcc_assert (innerc
);
10283 n1
= OMP_CLAUSE_DECL (innerc
);
10284 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10285 OMP_CLAUSE__LOOPTEMP_
);
10286 gcc_assert (innerc
);
10287 n2
= OMP_CLAUSE_DECL (innerc
);
10288 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10289 fold_convert (type
, n1
));
10290 if (fd
->collapse
> 1)
10293 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10299 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10300 fold_convert (type
, fd
->loop
.n1
));
10301 if (fd
->collapse
> 1)
10302 for (i
= 0; i
< fd
->collapse
; i
++)
10304 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10305 if (POINTER_TYPE_P (itype
))
10306 itype
= signed_type_for (itype
);
10307 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10308 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10312 /* Remove the GIMPLE_OMP_FOR statement. */
10313 gsi_remove (&gsi
, true);
10317 /* Code to control the increment goes in the CONT_BB. */
10318 gsi
= gsi_last_bb (cont_bb
);
10319 stmt
= gsi_stmt (gsi
);
10320 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10322 if (POINTER_TYPE_P (type
))
10323 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10325 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10326 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10328 if (fd
->collapse
> 1)
10330 i
= fd
->collapse
- 1;
10331 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10333 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10334 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10338 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10339 fd
->loops
[i
].step
);
10340 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10341 fd
->loops
[i
].v
, t
);
10343 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10345 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10347 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10348 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10349 if (POINTER_TYPE_P (itype2
))
10350 itype2
= signed_type_for (itype2
);
10351 t
= build3 (COND_EXPR
, itype2
,
10352 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10354 fold_convert (itype
, fd
->loops
[i
].n2
)),
10355 build_int_cst (itype2
, 0),
10356 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10357 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10358 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10360 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10361 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10363 t
= build3 (COND_EXPR
, itype
,
10364 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10366 fold_convert (itype
, fd
->loops
[i
].n2
)),
10368 fold_convert (itype
, fd
->loops
[i
].n1
));
10369 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10373 /* Remove GIMPLE_OMP_CONTINUE. */
10374 gsi_remove (&gsi
, true);
10377 /* Emit the condition in L1_BB. */
10378 gsi
= gsi_start_bb (l1_bb
);
10380 t
= fold_convert (type
, n2
);
10381 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10382 false, GSI_CONTINUE_LINKING
);
10383 tree v
= fd
->loop
.v
;
10384 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10385 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10386 false, GSI_CONTINUE_LINKING
);
10387 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10388 cond_stmt
= gimple_build_cond_empty (t
);
10389 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10390 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10392 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10395 gsi
= gsi_for_stmt (cond_stmt
);
10396 gimple_regimplify_operands (cond_stmt
, &gsi
);
10399 /* Remove GIMPLE_OMP_RETURN. */
10400 gsi
= gsi_last_bb (exit_bb
);
10401 gsi_remove (&gsi
, true);
10403 /* Connect the new blocks. */
10404 remove_edge (FALLTHRU_EDGE (entry_bb
));
10408 remove_edge (BRANCH_EDGE (entry_bb
));
10409 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10411 e
= BRANCH_EDGE (l1_bb
);
10412 ne
= FALLTHRU_EDGE (l1_bb
);
10413 e
->flags
= EDGE_TRUE_VALUE
;
10417 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10419 ne
= single_succ_edge (l1_bb
);
10420 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10423 ne
->flags
= EDGE_FALSE_VALUE
;
10424 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10425 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10427 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10428 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10429 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10433 struct loop
*loop
= alloc_loop ();
10434 loop
->header
= l1_bb
;
10435 loop
->latch
= cont_bb
;
10436 add_loop (loop
, l1_bb
->loop_father
);
10437 if (safelen
== NULL_TREE
)
10438 loop
->safelen
= INT_MAX
;
10441 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10442 if (TREE_CODE (safelen
) != INTEGER_CST
)
10444 else if (!tree_fits_uhwi_p (safelen
)
10445 || tree_to_uhwi (safelen
) > INT_MAX
)
10446 loop
->safelen
= INT_MAX
;
10448 loop
->safelen
= tree_to_uhwi (safelen
);
10449 if (loop
->safelen
== 1)
10454 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10455 cfun
->has_simduid_loops
= true;
10457 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10459 if ((flag_tree_loop_vectorize
10460 || (!global_options_set
.x_flag_tree_loop_vectorize
10461 && !global_options_set
.x_flag_tree_vectorize
))
10462 && flag_tree_loop_optimize
10463 && loop
->safelen
> 1)
10465 loop
->force_vectorize
= true;
10466 cfun
->has_force_vectorize_loops
= true;
10470 cfun
->has_simduid_loops
= true;
10473 /* Taskloop construct is represented after gimplification with
10474 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10475 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10476 which should just compute all the needed loop temporaries
10477 for GIMPLE_OMP_TASK. */
10480 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10481 struct omp_for_data
*fd
,
10482 gimple
*inner_stmt
)
10484 tree type
, bias
= NULL_TREE
;
10485 basic_block entry_bb
, cont_bb
, exit_bb
;
10486 gimple_stmt_iterator gsi
;
10487 gassign
*assign_stmt
;
10488 tree
*counts
= NULL
;
10491 gcc_assert (inner_stmt
);
10492 gcc_assert (region
->cont
);
10493 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10494 && gimple_omp_task_taskloop_p (inner_stmt
));
10495 type
= TREE_TYPE (fd
->loop
.v
);
10497 /* See if we need to bias by LLONG_MIN. */
10498 if (fd
->iter_type
== long_long_unsigned_type_node
10499 && TREE_CODE (type
) == INTEGER_TYPE
10500 && !TYPE_UNSIGNED (type
))
10504 if (fd
->loop
.cond_code
== LT_EXPR
)
10507 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10511 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10514 if (TREE_CODE (n1
) != INTEGER_CST
10515 || TREE_CODE (n2
) != INTEGER_CST
10516 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10517 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10520 entry_bb
= region
->entry
;
10521 cont_bb
= region
->cont
;
10522 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10523 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10524 exit_bb
= region
->exit
;
10526 gsi
= gsi_last_bb (entry_bb
);
10527 gimple
*for_stmt
= gsi_stmt (gsi
);
10528 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10529 if (fd
->collapse
> 1)
10531 int first_zero_iter
= -1, dummy
= -1;
10532 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10534 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10535 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10536 zero_iter_bb
, first_zero_iter
,
10537 dummy_bb
, dummy
, l2_dom_bb
);
10541 /* Some counts[i] vars might be uninitialized if
10542 some loop has zero iterations. But the body shouldn't
10543 be executed in that case, so just avoid uninit warnings. */
10544 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10545 if (SSA_VAR_P (counts
[i
]))
10546 TREE_NO_WARNING (counts
[i
]) = 1;
10548 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10549 entry_bb
= e
->dest
;
10550 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10551 gsi
= gsi_last_bb (entry_bb
);
10552 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10553 get_immediate_dominator (CDI_DOMINATORS
,
10561 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10562 && TYPE_PRECISION (TREE_TYPE (t0
))
10563 != TYPE_PRECISION (fd
->iter_type
))
10565 /* Avoid casting pointers to integer of a different size. */
10566 tree itype
= signed_type_for (type
);
10567 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10568 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10572 t1
= fold_convert (fd
->iter_type
, t1
);
10573 t0
= fold_convert (fd
->iter_type
, t0
);
10577 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10578 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10581 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10582 OMP_CLAUSE__LOOPTEMP_
);
10583 gcc_assert (innerc
);
10584 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10585 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10586 gcc_assert (innerc
);
10587 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10588 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10590 gcc_assert (innerc
);
10591 for (i
= 1; i
< fd
->collapse
; i
++)
10593 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10594 OMP_CLAUSE__LOOPTEMP_
);
10595 gcc_assert (innerc
);
10597 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10598 OMP_CLAUSE__LOOPTEMP_
);
10601 /* If needed (inner taskloop has lastprivate clause), propagate
10602 down the total number of iterations. */
10603 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10605 GSI_CONTINUE_LINKING
);
10606 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10607 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10611 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10612 GSI_CONTINUE_LINKING
);
10613 assign_stmt
= gimple_build_assign (startvar
, t0
);
10614 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10616 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10617 GSI_CONTINUE_LINKING
);
10618 assign_stmt
= gimple_build_assign (endvar
, t1
);
10619 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10620 if (fd
->collapse
> 1)
10621 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10623 /* Remove the GIMPLE_OMP_FOR statement. */
10624 gsi
= gsi_for_stmt (for_stmt
);
10625 gsi_remove (&gsi
, true);
10627 gsi
= gsi_last_bb (cont_bb
);
10628 gsi_remove (&gsi
, true);
10630 gsi
= gsi_last_bb (exit_bb
);
10631 gsi_remove (&gsi
, true);
10633 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10634 remove_edge (BRANCH_EDGE (entry_bb
));
10635 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10636 remove_edge (BRANCH_EDGE (cont_bb
));
10637 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10638 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10639 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10642 /* Taskloop construct is represented after gimplification with
10643 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10644 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10645 GOMP_taskloop{,_ull} function arranges for each task to be given just
10646 a single range of iterations. */
10649 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10650 struct omp_for_data
*fd
,
10651 gimple
*inner_stmt
)
10653 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10654 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10655 basic_block fin_bb
;
10656 gimple_stmt_iterator gsi
;
10658 bool broken_loop
= region
->cont
== NULL
;
10659 tree
*counts
= NULL
;
10662 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10663 if (POINTER_TYPE_P (type
))
10664 itype
= signed_type_for (type
);
10666 /* See if we need to bias by LLONG_MIN. */
10667 if (fd
->iter_type
== long_long_unsigned_type_node
10668 && TREE_CODE (type
) == INTEGER_TYPE
10669 && !TYPE_UNSIGNED (type
))
10673 if (fd
->loop
.cond_code
== LT_EXPR
)
10676 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10680 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10683 if (TREE_CODE (n1
) != INTEGER_CST
10684 || TREE_CODE (n2
) != INTEGER_CST
10685 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10686 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10689 entry_bb
= region
->entry
;
10690 cont_bb
= region
->cont
;
10691 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10692 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10693 gcc_assert (broken_loop
10694 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10695 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10698 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10699 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10701 exit_bb
= region
->exit
;
10703 /* Iteration space partitioning goes in ENTRY_BB. */
10704 gsi
= gsi_last_bb (entry_bb
);
10705 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10707 if (fd
->collapse
> 1)
10709 int first_zero_iter
= -1, dummy
= -1;
10710 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10712 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10713 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10714 fin_bb
, first_zero_iter
,
10715 dummy_bb
, dummy
, l2_dom_bb
);
10719 t
= integer_one_node
;
10721 step
= fd
->loop
.step
;
10722 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10723 OMP_CLAUSE__LOOPTEMP_
);
10724 gcc_assert (innerc
);
10725 n1
= OMP_CLAUSE_DECL (innerc
);
10726 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10727 gcc_assert (innerc
);
10728 n2
= OMP_CLAUSE_DECL (innerc
);
10731 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10732 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10734 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10735 true, NULL_TREE
, true, GSI_SAME_STMT
);
10736 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10737 true, NULL_TREE
, true, GSI_SAME_STMT
);
10738 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10739 true, NULL_TREE
, true, GSI_SAME_STMT
);
10741 tree startvar
= fd
->loop
.v
;
10742 tree endvar
= NULL_TREE
;
10744 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10746 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10747 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10748 gcc_assert (innerc
);
10749 startvar
= OMP_CLAUSE_DECL (innerc
);
10750 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10751 OMP_CLAUSE__LOOPTEMP_
);
10752 gcc_assert (innerc
);
10753 endvar
= OMP_CLAUSE_DECL (innerc
);
10755 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10756 t
= force_gimple_operand_gsi (&gsi
, t
,
10758 && TREE_ADDRESSABLE (startvar
),
10759 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10760 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10761 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10763 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10764 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10765 false, GSI_CONTINUE_LINKING
);
10768 assign_stmt
= gimple_build_assign (endvar
, e
);
10769 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10770 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10771 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10773 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10774 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10776 if (fd
->collapse
> 1)
10777 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10781 /* The code controlling the sequential loop replaces the
10782 GIMPLE_OMP_CONTINUE. */
10783 gsi
= gsi_last_bb (cont_bb
);
10784 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10785 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10786 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10787 vback
= gimple_omp_continue_control_def (cont_stmt
);
10789 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10791 if (POINTER_TYPE_P (type
))
10792 t
= fold_build_pointer_plus (vmain
, step
);
10794 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10795 t
= force_gimple_operand_gsi (&gsi
, t
,
10797 && TREE_ADDRESSABLE (vback
),
10798 NULL_TREE
, true, GSI_SAME_STMT
);
10799 assign_stmt
= gimple_build_assign (vback
, t
);
10800 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10802 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10803 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10805 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10808 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10809 gsi_remove (&gsi
, true);
10811 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10812 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10815 /* Remove the GIMPLE_OMP_FOR statement. */
10816 gsi
= gsi_for_stmt (fd
->for_stmt
);
10817 gsi_remove (&gsi
, true);
10819 /* Remove the GIMPLE_OMP_RETURN statement. */
10820 gsi
= gsi_last_bb (exit_bb
);
10821 gsi_remove (&gsi
, true);
10823 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10825 remove_edge (BRANCH_EDGE (entry_bb
));
10828 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
10829 region
->outer
->cont
= NULL
;
10832 /* Connect all the blocks. */
10835 ep
= find_edge (cont_bb
, body_bb
);
10836 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10841 else if (fd
->collapse
> 1)
10844 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10847 ep
->flags
= EDGE_TRUE_VALUE
;
10848 find_edge (cont_bb
, fin_bb
)->flags
10849 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10852 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10853 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10855 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10856 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10858 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
10860 struct loop
*loop
= alloc_loop ();
10861 loop
->header
= body_bb
;
10862 if (collapse_bb
== NULL
)
10863 loop
->latch
= cont_bb
;
10864 add_loop (loop
, body_bb
->loop_father
);
10868 /* A subroutine of expand_omp_for. Generate code for an OpenACC
10869 partitioned loop. The lowering here is abstracted, in that the
10870 loop parameters are passed through internal functions, which are
10871 further lowered by oacc_device_lower, once we get to the target
10872 compiler. The loop is of the form:
10874 for (V = B; V LTGT E; V += S) {BODY}
10876 where LTGT is < or >. We may have a specified chunking size, CHUNKING
10877 (constant 0 for no chunking) and we will have a GWV partitioning
10878 mask, specifying dimensions over which the loop is to be
10879 partitioned (see note below). We generate code that looks like:
10881 <entry_bb> [incoming FALL->body, BRANCH->exit]
10882 typedef signedintify (typeof (V)) T; // underlying signed integral type
10885 T DIR = LTGT == '<' ? +1 : -1;
10886 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
10887 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
10889 <head_bb> [created by splitting end of entry_bb]
10890 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
10891 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
10892 if (!(offset LTGT bound)) goto bottom_bb;
10894 <body_bb> [incoming]
10898 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
10900 if (offset LTGT bound) goto body_bb; [*]
10902 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
10904 if (chunk < chunk_max) goto head_bb;
10906 <exit_bb> [incoming]
10907 V = B + ((range -/+ 1) / S +/- 1) * S [*]
10909 [*] Needed if V live at end of loop
10911 Note: CHUNKING & GWV mask are specified explicitly here. This is a
10912 transition, and will be specified by a more general mechanism shortly.
10916 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10918 tree v
= fd
->loop
.v
;
10919 enum tree_code cond_code
= fd
->loop
.cond_code
;
10920 enum tree_code plus_code
= PLUS_EXPR
;
10922 tree chunk_size
= integer_minus_one_node
;
10923 tree gwv
= integer_zero_node
;
10924 tree iter_type
= TREE_TYPE (v
);
10925 tree diff_type
= iter_type
;
10926 tree plus_type
= iter_type
;
10927 struct oacc_collapse
*counts
= NULL
;
10929 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
10930 == GF_OMP_FOR_KIND_OACC_LOOP
);
10931 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
10932 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
10934 if (POINTER_TYPE_P (iter_type
))
10936 plus_code
= POINTER_PLUS_EXPR
;
10937 plus_type
= sizetype
;
10939 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
10940 diff_type
= signed_type_for (diff_type
);
10942 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
10943 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
10944 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
10945 basic_block bottom_bb
= NULL
;
10947 /* entry_bb has two sucessors; the branch edge is to the exit
10948 block, fallthrough edge to body. */
10949 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
10950 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
10952 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
10953 body_bb, or to a block whose only successor is the body_bb. Its
10954 fallthrough successor is the final block (same as the branch
10955 successor of the entry_bb). */
10958 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10959 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
10961 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
10962 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
10965 gcc_assert (!gimple_in_ssa_p (cfun
));
10967 /* The exit block only has entry_bb and cont_bb as predecessors. */
10968 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
10971 tree chunk_max
= NULL_TREE
;
10972 tree bound
, offset
;
10973 tree step
= create_tmp_var (diff_type
, ".step");
10974 bool up
= cond_code
== LT_EXPR
;
10975 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
10976 bool chunking
= !gimple_in_ssa_p (cfun
);;
10979 /* SSA instances. */
10980 tree offset_incr
= NULL_TREE
;
10981 tree offset_init
= NULL_TREE
;
10983 gimple_stmt_iterator gsi
;
10989 edge split
, be
, fte
;
10991 /* Split the end of entry_bb to create head_bb. */
10992 split
= split_block (entry_bb
, last_stmt (entry_bb
));
10993 basic_block head_bb
= split
->dest
;
10994 entry_bb
= split
->src
;
10996 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
10997 gsi
= gsi_last_bb (entry_bb
);
10998 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
10999 loc
= gimple_location (for_stmt
);
11001 if (gimple_in_ssa_p (cfun
))
11003 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11004 gcc_assert (integer_zerop (fd
->loop
.n1
));
11005 /* The SSA parallelizer does gang parallelism. */
11006 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11009 if (fd
->collapse
> 1)
11011 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11012 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11013 TREE_TYPE (fd
->loop
.n2
));
11015 if (SSA_VAR_P (fd
->loop
.n2
))
11017 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11018 true, GSI_SAME_STMT
);
11019 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11020 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11025 tree b
= fd
->loop
.n1
;
11026 tree e
= fd
->loop
.n2
;
11027 tree s
= fd
->loop
.step
;
11029 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11030 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11032 /* Convert the step, avoiding possible unsigned->signed overflow. */
11033 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11035 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11036 s
= fold_convert (diff_type
, s
);
11038 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11039 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11042 chunk_size
= integer_zero_node
;
11043 expr
= fold_convert (diff_type
, chunk_size
);
11044 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11045 NULL_TREE
, true, GSI_SAME_STMT
);
11046 /* Determine the range, avoiding possible unsigned->signed overflow. */
11047 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11048 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11049 fold_convert (plus_type
, negating
? b
: e
),
11050 fold_convert (plus_type
, negating
? e
: b
));
11051 expr
= fold_convert (diff_type
, expr
);
11053 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11054 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11055 NULL_TREE
, true, GSI_SAME_STMT
);
11057 chunk_no
= build_int_cst (diff_type
, 0);
11060 gcc_assert (!gimple_in_ssa_p (cfun
));
11063 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11064 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11066 ass
= gimple_build_assign (chunk_no
, expr
);
11067 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11069 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11070 build_int_cst (integer_type_node
,
11071 IFN_GOACC_LOOP_CHUNKS
),
11072 dir
, range
, s
, chunk_size
, gwv
);
11073 gimple_call_set_lhs (call
, chunk_max
);
11074 gimple_set_location (call
, loc
);
11075 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11078 chunk_size
= chunk_no
;
11080 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11081 build_int_cst (integer_type_node
,
11082 IFN_GOACC_LOOP_STEP
),
11083 dir
, range
, s
, chunk_size
, gwv
);
11084 gimple_call_set_lhs (call
, step
);
11085 gimple_set_location (call
, loc
);
11086 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11088 /* Remove the GIMPLE_OMP_FOR. */
11089 gsi_remove (&gsi
, true);
11091 /* Fixup edges from head_bb */
11092 be
= BRANCH_EDGE (head_bb
);
11093 fte
= FALLTHRU_EDGE (head_bb
);
11094 be
->flags
|= EDGE_FALSE_VALUE
;
11095 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11097 basic_block body_bb
= fte
->dest
;
11099 if (gimple_in_ssa_p (cfun
))
11101 gsi
= gsi_last_bb (cont_bb
);
11102 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11104 offset
= gimple_omp_continue_control_use (cont_stmt
);
11105 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11109 offset
= create_tmp_var (diff_type
, ".offset");
11110 offset_init
= offset_incr
= offset
;
11112 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11114 /* Loop offset & bound go into head_bb. */
11115 gsi
= gsi_start_bb (head_bb
);
11117 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11118 build_int_cst (integer_type_node
,
11119 IFN_GOACC_LOOP_OFFSET
),
11121 chunk_size
, gwv
, chunk_no
);
11122 gimple_call_set_lhs (call
, offset_init
);
11123 gimple_set_location (call
, loc
);
11124 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11126 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11127 build_int_cst (integer_type_node
,
11128 IFN_GOACC_LOOP_BOUND
),
11130 chunk_size
, gwv
, offset_init
);
11131 gimple_call_set_lhs (call
, bound
);
11132 gimple_set_location (call
, loc
);
11133 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11135 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11136 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11137 GSI_CONTINUE_LINKING
);
11139 /* V assignment goes into body_bb. */
11140 if (!gimple_in_ssa_p (cfun
))
11142 gsi
= gsi_start_bb (body_bb
);
11144 expr
= build2 (plus_code
, iter_type
, b
,
11145 fold_convert (plus_type
, offset
));
11146 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11147 true, GSI_SAME_STMT
);
11148 ass
= gimple_build_assign (v
, expr
);
11149 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11150 if (fd
->collapse
> 1)
11151 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11154 /* Loop increment goes into cont_bb. If this is not a loop, we
11155 will have spawned threads as if it was, and each one will
11156 execute one iteration. The specification is not explicit about
11157 whether such constructs are ill-formed or not, and they can
11158 occur, especially when noreturn routines are involved. */
11161 gsi
= gsi_last_bb (cont_bb
);
11162 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11163 loc
= gimple_location (cont_stmt
);
11165 /* Increment offset. */
11166 if (gimple_in_ssa_p (cfun
))
11167 expr
= build2 (plus_code
, iter_type
, offset
,
11168 fold_convert (plus_type
, step
));
11170 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11171 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11172 true, GSI_SAME_STMT
);
11173 ass
= gimple_build_assign (offset_incr
, expr
);
11174 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11175 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11176 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11178 /* Remove the GIMPLE_OMP_CONTINUE. */
11179 gsi_remove (&gsi
, true);
11181 /* Fixup edges from cont_bb */
11182 be
= BRANCH_EDGE (cont_bb
);
11183 fte
= FALLTHRU_EDGE (cont_bb
);
11184 be
->flags
|= EDGE_TRUE_VALUE
;
11185 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11189 /* Split the beginning of exit_bb to make bottom_bb. We
11190 need to insert a nop at the start, because splitting is
11191 after a stmt, not before. */
11192 gsi
= gsi_start_bb (exit_bb
);
11193 stmt
= gimple_build_nop ();
11194 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11195 split
= split_block (exit_bb
, stmt
);
11196 bottom_bb
= split
->src
;
11197 exit_bb
= split
->dest
;
11198 gsi
= gsi_last_bb (bottom_bb
);
11200 /* Chunk increment and test goes into bottom_bb. */
11201 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11202 build_int_cst (diff_type
, 1));
11203 ass
= gimple_build_assign (chunk_no
, expr
);
11204 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11206 /* Chunk test at end of bottom_bb. */
11207 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11208 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11209 GSI_CONTINUE_LINKING
);
11211 /* Fixup edges from bottom_bb. */
11212 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11213 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11217 gsi
= gsi_last_bb (exit_bb
);
11218 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11219 loc
= gimple_location (gsi_stmt (gsi
));
11221 if (!gimple_in_ssa_p (cfun
))
11223 /* Insert the final value of V, in case it is live. This is the
11224 value for the only thread that survives past the join. */
11225 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11226 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11227 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11228 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11229 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11230 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11231 true, GSI_SAME_STMT
);
11232 ass
= gimple_build_assign (v
, expr
);
11233 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11236 /* Remove the OMP_RETURN. */
11237 gsi_remove (&gsi
, true);
11241 /* We now have one or two nested loops. Update the loop
11243 struct loop
*parent
= entry_bb
->loop_father
;
11244 struct loop
*body
= body_bb
->loop_father
;
11248 struct loop
*chunk_loop
= alloc_loop ();
11249 chunk_loop
->header
= head_bb
;
11250 chunk_loop
->latch
= bottom_bb
;
11251 add_loop (chunk_loop
, parent
);
11252 parent
= chunk_loop
;
11254 else if (parent
!= body
)
11256 gcc_assert (body
->header
== body_bb
);
11257 gcc_assert (body
->latch
== cont_bb
11258 || single_pred (body
->latch
) == cont_bb
);
11264 struct loop
*body_loop
= alloc_loop ();
11265 body_loop
->header
= body_bb
;
11266 body_loop
->latch
= cont_bb
;
11267 add_loop (body_loop
, parent
);
11272 /* Expand the OMP loop defined by REGION. */
11275 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11277 struct omp_for_data fd
;
11278 struct omp_for_data_loop
*loops
;
11281 = (struct omp_for_data_loop
*)
11282 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11283 * sizeof (struct omp_for_data_loop
));
11284 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11286 region
->sched_kind
= fd
.sched_kind
;
11287 region
->sched_modifiers
= fd
.sched_modifiers
;
11289 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11290 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11291 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11294 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11295 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11296 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11299 /* If there isn't a continue then this is a degerate case where
11300 the introduction of abnormal edges during lowering will prevent
11301 original loops from being detected. Fix that up. */
11302 loops_state_set (LOOPS_NEED_FIXUP
);
11304 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11305 expand_omp_simd (region
, &fd
);
11306 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11307 expand_cilk_for (region
, &fd
);
11308 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11310 gcc_assert (!inner_stmt
);
11311 expand_oacc_for (region
, &fd
);
11313 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11315 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11316 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11318 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11320 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11321 && !fd
.have_ordered
)
11323 if (fd
.chunk_size
== NULL
)
11324 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11326 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11330 int fn_index
, start_ix
, next_ix
;
11332 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11333 == GF_OMP_FOR_KIND_FOR
);
11334 if (fd
.chunk_size
== NULL
11335 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11336 fd
.chunk_size
= integer_zero_node
;
11337 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11338 switch (fd
.sched_kind
)
11340 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11343 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11344 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11345 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11347 && !fd
.have_ordered
)
11349 fn_index
= 3 + fd
.sched_kind
;
11354 fn_index
= fd
.sched_kind
;
11358 fn_index
+= fd
.have_ordered
* 6;
11360 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11362 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11363 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11364 if (fd
.iter_type
== long_long_unsigned_type_node
)
11366 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11367 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11368 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11369 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11371 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11372 (enum built_in_function
) next_ix
, inner_stmt
);
11375 if (gimple_in_ssa_p (cfun
))
11376 update_ssa (TODO_update_ssa_only_virtuals
);
11380 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11382 v = GOMP_sections_start (n);
11399 v = GOMP_sections_next ();
11404 If this is a combined parallel sections, replace the call to
11405 GOMP_sections_start with call to GOMP_sections_next. */
11408 expand_omp_sections (struct omp_region
*region
)
11410 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11412 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11413 gimple_stmt_iterator si
, switch_si
;
11414 gomp_sections
*sections_stmt
;
11416 gomp_continue
*cont
;
11419 struct omp_region
*inner
;
11421 bool exit_reachable
= region
->cont
!= NULL
;
11423 gcc_assert (region
->exit
!= NULL
);
11424 entry_bb
= region
->entry
;
11425 l0_bb
= single_succ (entry_bb
);
11426 l1_bb
= region
->cont
;
11427 l2_bb
= region
->exit
;
11428 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11429 l2
= gimple_block_label (l2_bb
);
11432 /* This can happen if there are reductions. */
11433 len
= EDGE_COUNT (l0_bb
->succs
);
11434 gcc_assert (len
> 0);
11435 e
= EDGE_SUCC (l0_bb
, len
- 1);
11436 si
= gsi_last_bb (e
->dest
);
11439 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11440 l2
= gimple_block_label (e
->dest
);
11442 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11444 si
= gsi_last_bb (e
->dest
);
11446 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11448 l2
= gimple_block_label (e
->dest
);
11453 if (exit_reachable
)
11454 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11456 default_bb
= create_empty_bb (l0_bb
);
11458 /* We will build a switch() with enough cases for all the
11459 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11460 and a default case to abort if something goes wrong. */
11461 len
= EDGE_COUNT (l0_bb
->succs
);
11463 /* Use vec::quick_push on label_vec throughout, since we know the size
11465 auto_vec
<tree
> label_vec (len
);
11467 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11468 GIMPLE_OMP_SECTIONS statement. */
11469 si
= gsi_last_bb (entry_bb
);
11470 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11471 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11472 vin
= gimple_omp_sections_control (sections_stmt
);
11473 if (!is_combined_parallel (region
))
11475 /* If we are not inside a combined parallel+sections region,
11476 call GOMP_sections_start. */
11477 t
= build_int_cst (unsigned_type_node
, len
- 1);
11478 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11479 stmt
= gimple_build_call (u
, 1, t
);
11483 /* Otherwise, call GOMP_sections_next. */
11484 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11485 stmt
= gimple_build_call (u
, 0);
11487 gimple_call_set_lhs (stmt
, vin
);
11488 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11489 gsi_remove (&si
, true);
11491 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11493 switch_si
= gsi_last_bb (l0_bb
);
11494 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11495 if (exit_reachable
)
11497 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11498 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11499 vmain
= gimple_omp_continue_control_use (cont
);
11500 vnext
= gimple_omp_continue_control_def (cont
);
11508 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11509 label_vec
.quick_push (t
);
11512 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11513 for (inner
= region
->inner
, casei
= 1;
11515 inner
= inner
->next
, i
++, casei
++)
11517 basic_block s_entry_bb
, s_exit_bb
;
11519 /* Skip optional reduction region. */
11520 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11527 s_entry_bb
= inner
->entry
;
11528 s_exit_bb
= inner
->exit
;
11530 t
= gimple_block_label (s_entry_bb
);
11531 u
= build_int_cst (unsigned_type_node
, casei
);
11532 u
= build_case_label (u
, NULL
, t
);
11533 label_vec
.quick_push (u
);
11535 si
= gsi_last_bb (s_entry_bb
);
11536 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11537 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11538 gsi_remove (&si
, true);
11539 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11541 if (s_exit_bb
== NULL
)
11544 si
= gsi_last_bb (s_exit_bb
);
11545 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11546 gsi_remove (&si
, true);
11548 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11551 /* Error handling code goes in DEFAULT_BB. */
11552 t
= gimple_block_label (default_bb
);
11553 u
= build_case_label (NULL
, NULL
, t
);
11554 make_edge (l0_bb
, default_bb
, 0);
11555 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11557 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11558 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11559 gsi_remove (&switch_si
, true);
11561 si
= gsi_start_bb (default_bb
);
11562 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11563 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11565 if (exit_reachable
)
11569 /* Code to get the next section goes in L1_BB. */
11570 si
= gsi_last_bb (l1_bb
);
11571 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11573 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11574 stmt
= gimple_build_call (bfn_decl
, 0);
11575 gimple_call_set_lhs (stmt
, vnext
);
11576 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11577 gsi_remove (&si
, true);
11579 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11582 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11583 si
= gsi_last_bb (l2_bb
);
11584 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11585 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11586 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11587 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11589 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11590 stmt
= gimple_build_call (t
, 0);
11591 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11592 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11593 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11594 gsi_remove (&si
, true);
11596 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11600 /* Expand code for an OpenMP single directive. We've already expanded
11601 much of the code, here we simply place the GOMP_barrier call. */
11604 expand_omp_single (struct omp_region
*region
)
11606 basic_block entry_bb
, exit_bb
;
11607 gimple_stmt_iterator si
;
11609 entry_bb
= region
->entry
;
11610 exit_bb
= region
->exit
;
11612 si
= gsi_last_bb (entry_bb
);
11613 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11614 gsi_remove (&si
, true);
11615 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11617 si
= gsi_last_bb (exit_bb
);
11618 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11620 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11621 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11623 gsi_remove (&si
, true);
11624 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11628 /* Generic expansion for OpenMP synchronization directives: master,
11629 ordered and critical. All we need to do here is remove the entry
11630 and exit markers for REGION. */
11633 expand_omp_synch (struct omp_region
*region
)
11635 basic_block entry_bb
, exit_bb
;
11636 gimple_stmt_iterator si
;
11638 entry_bb
= region
->entry
;
11639 exit_bb
= region
->exit
;
11641 si
= gsi_last_bb (entry_bb
);
11642 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11643 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11644 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11645 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11646 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11647 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11648 gsi_remove (&si
, true);
11649 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11653 si
= gsi_last_bb (exit_bb
);
11654 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11655 gsi_remove (&si
, true);
11656 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11660 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11661 operation as a normal volatile load. */
11664 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11665 tree loaded_val
, int index
)
11667 enum built_in_function tmpbase
;
11668 gimple_stmt_iterator gsi
;
11669 basic_block store_bb
;
11672 tree decl
, call
, type
, itype
;
11674 gsi
= gsi_last_bb (load_bb
);
11675 stmt
= gsi_stmt (gsi
);
11676 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11677 loc
= gimple_location (stmt
);
11679 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11680 is smaller than word size, then expand_atomic_load assumes that the load
11681 is atomic. We could avoid the builtin entirely in this case. */
11683 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11684 decl
= builtin_decl_explicit (tmpbase
);
11685 if (decl
== NULL_TREE
)
11688 type
= TREE_TYPE (loaded_val
);
11689 itype
= TREE_TYPE (TREE_TYPE (decl
));
11691 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11692 build_int_cst (NULL
,
11693 gimple_omp_atomic_seq_cst_p (stmt
)
11695 : MEMMODEL_RELAXED
));
11696 if (!useless_type_conversion_p (type
, itype
))
11697 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11698 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11700 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11701 gsi_remove (&gsi
, true);
11703 store_bb
= single_succ (load_bb
);
11704 gsi
= gsi_last_bb (store_bb
);
11705 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11706 gsi_remove (&gsi
, true);
11708 if (gimple_in_ssa_p (cfun
))
11709 update_ssa (TODO_update_ssa_no_phi
);
11714 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11715 operation as a normal volatile store. */
11718 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11719 tree loaded_val
, tree stored_val
, int index
)
11721 enum built_in_function tmpbase
;
11722 gimple_stmt_iterator gsi
;
11723 basic_block store_bb
= single_succ (load_bb
);
11726 tree decl
, call
, type
, itype
;
11727 machine_mode imode
;
11730 gsi
= gsi_last_bb (load_bb
);
11731 stmt
= gsi_stmt (gsi
);
11732 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11734 /* If the load value is needed, then this isn't a store but an exchange. */
11735 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11737 gsi
= gsi_last_bb (store_bb
);
11738 stmt
= gsi_stmt (gsi
);
11739 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11740 loc
= gimple_location (stmt
);
11742 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11743 is smaller than word size, then expand_atomic_store assumes that the store
11744 is atomic. We could avoid the builtin entirely in this case. */
11746 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11747 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
11748 decl
= builtin_decl_explicit (tmpbase
);
11749 if (decl
== NULL_TREE
)
11752 type
= TREE_TYPE (stored_val
);
11754 /* Dig out the type of the function's second argument. */
11755 itype
= TREE_TYPE (decl
);
11756 itype
= TYPE_ARG_TYPES (itype
);
11757 itype
= TREE_CHAIN (itype
);
11758 itype
= TREE_VALUE (itype
);
11759 imode
= TYPE_MODE (itype
);
11761 if (exchange
&& !can_atomic_exchange_p (imode
, true))
11764 if (!useless_type_conversion_p (itype
, type
))
11765 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
11766 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
11767 build_int_cst (NULL
,
11768 gimple_omp_atomic_seq_cst_p (stmt
)
11770 : MEMMODEL_RELAXED
));
11773 if (!useless_type_conversion_p (type
, itype
))
11774 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11775 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11778 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11779 gsi_remove (&gsi
, true);
11781 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
11782 gsi
= gsi_last_bb (load_bb
);
11783 gsi_remove (&gsi
, true);
11785 if (gimple_in_ssa_p (cfun
))
11786 update_ssa (TODO_update_ssa_no_phi
);
11791 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11792 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
11793 size of the data type, and thus usable to find the index of the builtin
11794 decl. Returns false if the expression is not of the proper form. */
11797 expand_omp_atomic_fetch_op (basic_block load_bb
,
11798 tree addr
, tree loaded_val
,
11799 tree stored_val
, int index
)
11801 enum built_in_function oldbase
, newbase
, tmpbase
;
11802 tree decl
, itype
, call
;
11804 basic_block store_bb
= single_succ (load_bb
);
11805 gimple_stmt_iterator gsi
;
11808 enum tree_code code
;
11809 bool need_old
, need_new
;
11810 machine_mode imode
;
11813 /* We expect to find the following sequences:
11816 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
11819 val = tmp OP something; (or: something OP tmp)
11820 GIMPLE_OMP_STORE (val)
11822 ???FIXME: Allow a more flexible sequence.
11823 Perhaps use data flow to pick the statements.
11827 gsi
= gsi_after_labels (store_bb
);
11828 stmt
= gsi_stmt (gsi
);
11829 loc
= gimple_location (stmt
);
11830 if (!is_gimple_assign (stmt
))
11833 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
11835 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
11836 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
11837 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
11838 gcc_checking_assert (!need_old
|| !need_new
);
11840 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
11843 /* Check for one of the supported fetch-op operations. */
11844 code
= gimple_assign_rhs_code (stmt
);
11848 case POINTER_PLUS_EXPR
:
11849 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
11850 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
11853 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
11854 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
11857 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
11858 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
11861 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
11862 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
11865 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
11866 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
11872 /* Make sure the expression is of the proper form. */
11873 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
11874 rhs
= gimple_assign_rhs2 (stmt
);
11875 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
11876 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
11877 rhs
= gimple_assign_rhs1 (stmt
);
11881 tmpbase
= ((enum built_in_function
)
11882 ((need_new
? newbase
: oldbase
) + index
+ 1));
11883 decl
= builtin_decl_explicit (tmpbase
);
11884 if (decl
== NULL_TREE
)
11886 itype
= TREE_TYPE (TREE_TYPE (decl
));
11887 imode
= TYPE_MODE (itype
);
11889 /* We could test all of the various optabs involved, but the fact of the
11890 matter is that (with the exception of i486 vs i586 and xadd) all targets
11891 that support any atomic operaton optab also implements compare-and-swap.
11892 Let optabs.c take care of expanding any compare-and-swap loop. */
11893 if (!can_compare_and_swap_p (imode
, true))
11896 gsi
= gsi_last_bb (load_bb
);
11897 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11899 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
11900 It only requires that the operation happen atomically. Thus we can
11901 use the RELAXED memory model. */
11902 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
11903 fold_convert_loc (loc
, itype
, rhs
),
11904 build_int_cst (NULL
,
11905 seq_cst
? MEMMODEL_SEQ_CST
11906 : MEMMODEL_RELAXED
));
11908 if (need_old
|| need_new
)
11910 lhs
= need_old
? loaded_val
: stored_val
;
11911 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
11912 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
11915 call
= fold_convert_loc (loc
, void_type_node
, call
);
11916 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11917 gsi_remove (&gsi
, true);
11919 gsi
= gsi_last_bb (store_bb
);
11920 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11921 gsi_remove (&gsi
, true);
11922 gsi
= gsi_last_bb (store_bb
);
11923 gsi_remove (&gsi
, true);
11925 if (gimple_in_ssa_p (cfun
))
11926 update_ssa (TODO_update_ssa_no_phi
);
11931 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
11935 newval = rhs; // with oldval replacing *addr in rhs
11936 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
11937 if (oldval != newval)
11940 INDEX is log2 of the size of the data type, and thus usable to find the
11941 index of the builtin decl. */
11944 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
11945 tree addr
, tree loaded_val
, tree stored_val
,
11948 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
11949 tree type
, itype
, cmpxchg
, iaddr
;
11950 gimple_stmt_iterator si
;
11951 basic_block loop_header
= single_succ (load_bb
);
11952 gimple
*phi
, *stmt
;
11954 enum built_in_function fncode
;
11956 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
11957 order to use the RELAXED memory model effectively. */
11958 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
11960 cmpxchg
= builtin_decl_explicit (fncode
);
11961 if (cmpxchg
== NULL_TREE
)
11963 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
11964 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
11966 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
11969 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
11970 si
= gsi_last_bb (load_bb
);
11971 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11973 /* For floating-point values, we'll need to view-convert them to integers
11974 so that we can perform the atomic compare and swap. Simplify the
11975 following code by always setting up the "i"ntegral variables. */
11976 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
11980 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
11983 = force_gimple_operand_gsi (&si
,
11984 fold_convert (TREE_TYPE (iaddr
), addr
),
11985 false, NULL_TREE
, true, GSI_SAME_STMT
);
11986 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
11987 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11988 loadedi
= create_tmp_var (itype
);
11989 if (gimple_in_ssa_p (cfun
))
11990 loadedi
= make_ssa_name (loadedi
);
11995 loadedi
= loaded_val
;
11998 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11999 tree loaddecl
= builtin_decl_explicit (fncode
);
12002 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12003 build_call_expr (loaddecl
, 2, iaddr
,
12004 build_int_cst (NULL_TREE
,
12005 MEMMODEL_RELAXED
)));
12007 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12008 build_int_cst (TREE_TYPE (iaddr
), 0));
12011 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12014 /* Move the value to the LOADEDI temporary. */
12015 if (gimple_in_ssa_p (cfun
))
12017 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12018 phi
= create_phi_node (loadedi
, loop_header
);
12019 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12023 gsi_insert_before (&si
,
12024 gimple_build_assign (loadedi
, initial
),
12026 if (loadedi
!= loaded_val
)
12028 gimple_stmt_iterator gsi2
;
12031 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12032 gsi2
= gsi_start_bb (loop_header
);
12033 if (gimple_in_ssa_p (cfun
))
12036 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12037 true, GSI_SAME_STMT
);
12038 stmt
= gimple_build_assign (loaded_val
, x
);
12039 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12043 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12044 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12045 true, GSI_SAME_STMT
);
12048 gsi_remove (&si
, true);
12050 si
= gsi_last_bb (store_bb
);
12051 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12054 storedi
= stored_val
;
12057 force_gimple_operand_gsi (&si
,
12058 build1 (VIEW_CONVERT_EXPR
, itype
,
12059 stored_val
), true, NULL_TREE
, true,
12062 /* Build the compare&swap statement. */
12063 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12064 new_storedi
= force_gimple_operand_gsi (&si
,
12065 fold_convert (TREE_TYPE (loadedi
),
12068 true, GSI_SAME_STMT
);
12070 if (gimple_in_ssa_p (cfun
))
12071 old_vali
= loadedi
;
12074 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12075 stmt
= gimple_build_assign (old_vali
, loadedi
);
12076 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12078 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12079 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12082 /* Note that we always perform the comparison as an integer, even for
12083 floating point. This allows the atomic operation to properly
12084 succeed even with NaNs and -0.0. */
12085 stmt
= gimple_build_cond_empty
12086 (build2 (NE_EXPR
, boolean_type_node
,
12087 new_storedi
, old_vali
));
12088 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12091 e
= single_succ_edge (store_bb
);
12092 e
->flags
&= ~EDGE_FALLTHRU
;
12093 e
->flags
|= EDGE_FALSE_VALUE
;
12095 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12097 /* Copy the new value to loadedi (we already did that before the condition
12098 if we are not in SSA). */
12099 if (gimple_in_ssa_p (cfun
))
12101 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12102 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12105 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12106 gsi_remove (&si
, true);
12108 struct loop
*loop
= alloc_loop ();
12109 loop
->header
= loop_header
;
12110 loop
->latch
= store_bb
;
12111 add_loop (loop
, loop_header
->loop_father
);
12113 if (gimple_in_ssa_p (cfun
))
12114 update_ssa (TODO_update_ssa_no_phi
);
12119 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12121 GOMP_atomic_start ();
12123 GOMP_atomic_end ();
12125 The result is not globally atomic, but works so long as all parallel
12126 references are within #pragma omp atomic directives. According to
12127 responses received from omp@openmp.org, appears to be within spec.
12128 Which makes sense, since that's how several other compilers handle
12129 this situation as well.
12130 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12131 expanding. STORED_VAL is the operand of the matching
12132 GIMPLE_OMP_ATOMIC_STORE.
12135 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12136 loaded_val = *addr;
12139 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12140 *addr = stored_val;
12144 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12145 tree addr
, tree loaded_val
, tree stored_val
)
12147 gimple_stmt_iterator si
;
12151 si
= gsi_last_bb (load_bb
);
12152 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12154 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12155 t
= build_call_expr (t
, 0);
12156 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12158 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12159 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12160 gsi_remove (&si
, true);
12162 si
= gsi_last_bb (store_bb
);
12163 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12165 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12167 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12169 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12170 t
= build_call_expr (t
, 0);
12171 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12172 gsi_remove (&si
, true);
12174 if (gimple_in_ssa_p (cfun
))
12175 update_ssa (TODO_update_ssa_no_phi
);
12179 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12180 using expand_omp_atomic_fetch_op. If it failed, we try to
12181 call expand_omp_atomic_pipeline, and if it fails too, the
12182 ultimate fallback is wrapping the operation in a mutex
12183 (expand_omp_atomic_mutex). REGION is the atomic region built
12184 by build_omp_regions_1(). */
12187 expand_omp_atomic (struct omp_region
*region
)
12189 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12190 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12191 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12192 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12193 tree addr
= gimple_omp_atomic_load_rhs (load
);
12194 tree stored_val
= gimple_omp_atomic_store_val (store
);
12195 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12196 HOST_WIDE_INT index
;
12198 /* Make sure the type is one of the supported sizes. */
12199 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12200 index
= exact_log2 (index
);
12201 if (index
>= 0 && index
<= 4)
12203 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12205 /* __sync builtins require strict data alignment. */
12206 if (exact_log2 (align
) >= index
)
12209 if (loaded_val
== stored_val
12210 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12211 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12212 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12213 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12216 /* Atomic store. */
12217 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12218 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12219 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12220 && store_bb
== single_succ (load_bb
)
12221 && first_stmt (store_bb
) == store
12222 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12223 stored_val
, index
))
12226 /* When possible, use specialized atomic update functions. */
12227 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12228 && store_bb
== single_succ (load_bb
)
12229 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12230 loaded_val
, stored_val
, index
))
12233 /* If we don't have specialized __sync builtins, try and implement
12234 as a compare and swap loop. */
12235 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12236 loaded_val
, stored_val
, index
))
12241 /* The ultimate fallback is wrapping the operation in a mutex. */
12242 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12246 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12247 macro on gomp-constants.h. We do not check for overflow. */
12250 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12254 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12257 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12258 device
, build_int_cst (unsigned_type_node
,
12259 GOMP_LAUNCH_DEVICE_SHIFT
));
12260 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12265 /* Look for compute grid dimension clauses and convert to an attribute
12266 attached to FN. This permits the target-side code to (a) massage
12267 the dimensions, (b) emit that data and (c) optimize. Non-constant
12268 dimensions are pushed onto ARGS.
12270 The attribute value is a TREE_LIST. A set of dimensions is
12271 represented as a list of INTEGER_CST. Those that are runtime
12272 exprs are represented as an INTEGER_CST of zero.
12274 TOOO. Normally the attribute will just contain a single such list. If
12275 however it contains a list of lists, this will represent the use of
12276 device_type. Each member of the outer list is an assoc list of
12277 dimensions, keyed by the device type. The first entry will be the
12278 default. Well, that's the plan. */
12280 #define OACC_FN_ATTRIB "oacc function"
12282 /* Replace any existing oacc fn attribute with updated dimensions. */
12285 replace_oacc_fn_attrib (tree fn
, tree dims
)
12287 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12288 tree attribs
= DECL_ATTRIBUTES (fn
);
12290 /* If we happen to be present as the first attrib, drop it. */
12291 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12292 attribs
= TREE_CHAIN (attribs
);
12293 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12296 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12297 function attribute. Push any that are non-constant onto the ARGS
12298 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
12301 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
12303 /* Must match GOMP_DIM ordering. */
12304 static const omp_clause_code ids
[]
12305 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12306 OMP_CLAUSE_VECTOR_LENGTH
};
12308 tree dims
[GOMP_DIM_MAX
];
12309 tree attr
= NULL_TREE
;
12310 unsigned non_const
= 0;
12312 for (ix
= GOMP_DIM_MAX
; ix
--;)
12314 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12315 tree dim
= NULL_TREE
;
12318 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12320 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12322 dim
= integer_zero_node
;
12323 non_const
|= GOMP_DIM_MASK (ix
);
12325 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12328 replace_oacc_fn_attrib (fn
, attr
);
12332 /* Push a dynamic argument set. */
12333 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12334 NULL_TREE
, non_const
));
12335 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12336 if (non_const
& GOMP_DIM_MASK (ix
))
12337 args
->safe_push (dims
[ix
]);
12341 /* Process the routine's dimension clauess to generate an attribute
12342 value. Issue diagnostics as appropriate. We default to SEQ
12343 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12344 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12345 can have a loop partitioned on it. non-zero indicates
12346 yes, zero indicates no. By construction once a non-zero has been
12347 reached, further inner dimensions must also be non-zero. We set
12348 TREE_VALUE to zero for the dimensions that may be partitioned and
12349 1 for the other ones -- if a loop is (erroneously) spawned at
12350 an outer level, we don't want to try and partition it. */
12353 build_oacc_routine_dims (tree clauses
)
12355 /* Must match GOMP_DIM ordering. */
12356 static const omp_clause_code ids
[] =
12357 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12361 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12362 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12363 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12366 error_at (OMP_CLAUSE_LOCATION (clauses
),
12367 "multiple loop axes specified for routine");
12372 /* Default to SEQ. */
12374 level
= GOMP_DIM_MAX
;
12376 tree dims
= NULL_TREE
;
12378 for (ix
= GOMP_DIM_MAX
; ix
--;)
12379 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12380 build_int_cst (integer_type_node
, ix
< level
), dims
);
12385 /* Retrieve the oacc function attrib and return it. Non-oacc
12386 functions will return NULL. */
12389 get_oacc_fn_attrib (tree fn
)
12391 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12394 /* Extract an oacc execution dimension from FN. FN must be an
12395 offloaded function or routine that has already had its execution
12396 dimensions lowered to the target-specific values. */
12399 get_oacc_fn_dim_size (tree fn
, int axis
)
12401 tree attrs
= get_oacc_fn_attrib (fn
);
12403 gcc_assert (axis
< GOMP_DIM_MAX
);
12405 tree dims
= TREE_VALUE (attrs
);
12407 dims
= TREE_CHAIN (dims
);
12409 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12414 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12415 IFN_GOACC_DIM_SIZE call. */
12418 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12420 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12421 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12422 tree arg
= gimple_call_arg (stmt
, 0);
12423 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12425 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12429 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12432 expand_omp_target (struct omp_region
*region
)
12434 basic_block entry_bb
, exit_bb
, new_bb
;
12435 struct function
*child_cfun
;
12436 tree child_fn
, block
, t
;
12437 gimple_stmt_iterator gsi
;
12438 gomp_target
*entry_stmt
;
12441 bool offloaded
, data_region
;
12443 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
12444 new_bb
= region
->entry
;
12446 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
12447 switch (gimple_omp_target_kind (entry_stmt
))
12449 case GF_OMP_TARGET_KIND_REGION
:
12450 case GF_OMP_TARGET_KIND_UPDATE
:
12451 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12452 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12453 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12454 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12455 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12456 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12457 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
12458 data_region
= false;
12460 case GF_OMP_TARGET_KIND_DATA
:
12461 case GF_OMP_TARGET_KIND_OACC_DATA
:
12462 data_region
= true;
12465 gcc_unreachable ();
12468 child_fn
= NULL_TREE
;
12472 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
12473 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
12476 /* Supported by expand_omp_taskreg, but not here. */
12477 if (child_cfun
!= NULL
)
12478 gcc_checking_assert (!child_cfun
->cfg
);
12479 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
12481 entry_bb
= region
->entry
;
12482 exit_bb
= region
->exit
;
12486 unsigned srcidx
, dstidx
, num
;
12488 /* If the offloading region needs data sent from the parent
12489 function, then the very first statement (except possible
12490 tree profile counter updates) of the offloading body
12491 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
12492 &.OMP_DATA_O is passed as an argument to the child function,
12493 we need to replace it with the argument as seen by the child
12496 In most cases, this will end up being the identity assignment
12497 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
12498 a function call that has been inlined, the original PARM_DECL
12499 .OMP_DATA_I may have been converted into a different local
12500 variable. In which case, we need to keep the assignment. */
12501 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
12504 basic_block entry_succ_bb
= single_succ (entry_bb
);
12505 gimple_stmt_iterator gsi
;
12507 gimple
*tgtcopy_stmt
= NULL
;
12508 tree sender
= TREE_VEC_ELT (data_arg
, 0);
12510 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
12512 gcc_assert (!gsi_end_p (gsi
));
12513 stmt
= gsi_stmt (gsi
);
12514 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
12517 if (gimple_num_ops (stmt
) == 2)
12519 tree arg
= gimple_assign_rhs1 (stmt
);
12521 /* We're ignoring the subcode because we're
12522 effectively doing a STRIP_NOPS. */
12524 if (TREE_CODE (arg
) == ADDR_EXPR
12525 && TREE_OPERAND (arg
, 0) == sender
)
12527 tgtcopy_stmt
= stmt
;
12533 gcc_assert (tgtcopy_stmt
!= NULL
);
12534 arg
= DECL_ARGUMENTS (child_fn
);
12536 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
12537 gsi_remove (&gsi
, true);
12540 /* Declare local variables needed in CHILD_CFUN. */
12541 block
= DECL_INITIAL (child_fn
);
12542 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
12543 /* The gimplifier could record temporaries in the offloading block
12544 rather than in containing function's local_decls chain,
12545 which would mean cgraph missed finalizing them. Do it now. */
12546 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
12547 if (TREE_CODE (t
) == VAR_DECL
12549 && !DECL_EXTERNAL (t
))
12550 varpool_node::finalize_decl (t
);
12551 DECL_SAVED_TREE (child_fn
) = NULL
;
12552 /* We'll create a CFG for child_fn, so no gimple body is needed. */
12553 gimple_set_body (child_fn
, NULL
);
12554 TREE_USED (block
) = 1;
12556 /* Reset DECL_CONTEXT on function arguments. */
12557 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
12558 DECL_CONTEXT (t
) = child_fn
;
12560 /* Split ENTRY_BB at GIMPLE_*,
12561 so that it can be moved to the child function. */
12562 gsi
= gsi_last_bb (entry_bb
);
12563 stmt
= gsi_stmt (gsi
);
12565 && gimple_code (stmt
) == gimple_code (entry_stmt
));
12566 e
= split_block (entry_bb
, stmt
);
12567 gsi_remove (&gsi
, true);
12568 entry_bb
= e
->dest
;
12569 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
12571 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
12574 gsi
= gsi_last_bb (exit_bb
);
12575 gcc_assert (!gsi_end_p (gsi
)
12576 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
12577 stmt
= gimple_build_return (NULL
);
12578 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
12579 gsi_remove (&gsi
, true);
12582 /* Move the offloading region into CHILD_CFUN. */
12584 block
= gimple_block (entry_stmt
);
12586 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
12588 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
12589 /* When the OMP expansion process cannot guarantee an up-to-date
12590 loop tree arrange for the child function to fixup loops. */
12591 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12592 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
12594 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
12595 num
= vec_safe_length (child_cfun
->local_decls
);
12596 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
12598 t
= (*child_cfun
->local_decls
)[srcidx
];
12599 if (DECL_CONTEXT (t
) == cfun
->decl
)
12601 if (srcidx
!= dstidx
)
12602 (*child_cfun
->local_decls
)[dstidx
] = t
;
12606 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
12608 /* Inform the callgraph about the new function. */
12609 child_cfun
->curr_properties
= cfun
->curr_properties
;
12610 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
12611 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
12612 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
12613 node
->parallelized_function
= 1;
12614 cgraph_node::add_new_function (child_fn
, true);
12616 #ifdef ENABLE_OFFLOADING
12617 /* Add the new function to the offload table. */
12618 vec_safe_push (offload_funcs
, child_fn
);
12621 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
12622 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
12624 /* Fix the callgraph edges for child_cfun. Those for cfun will be
12625 fixed in a following pass. */
12626 push_cfun (child_cfun
);
12628 assign_assembler_name_if_neeeded (child_fn
);
12629 cgraph_edge::rebuild_edges ();
12631 #ifdef ENABLE_OFFLOADING
12632 /* Prevent IPA from removing child_fn as unreachable, since there are no
12633 refs from the parent function to child_fn in offload LTO mode. */
12634 cgraph_node::get (child_fn
)->mark_force_output ();
12637 /* Some EH regions might become dead, see PR34608. If
12638 pass_cleanup_cfg isn't the first pass to happen with the
12639 new child, these dead EH edges might cause problems.
12640 Clean them up now. */
12641 if (flag_exceptions
)
12644 bool changed
= false;
12646 FOR_EACH_BB_FN (bb
, cfun
)
12647 changed
|= gimple_purge_dead_eh_edges (bb
);
12649 cleanup_tree_cfg ();
12651 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12652 verify_loop_structure ();
12655 if (dump_file
&& !gimple_in_ssa_p (cfun
))
12657 omp_any_child_fn_dumped
= true;
12658 dump_function_header (dump_file
, child_fn
, dump_flags
);
12659 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
12663 /* Emit a library call to launch the offloading region, or do data
12665 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
12666 enum built_in_function start_ix
;
12667 location_t clause_loc
;
12668 unsigned int flags_i
= 0;
12670 switch (gimple_omp_target_kind (entry_stmt
))
12672 case GF_OMP_TARGET_KIND_REGION
:
12673 start_ix
= BUILT_IN_GOMP_TARGET
;
12675 case GF_OMP_TARGET_KIND_DATA
:
12676 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
12678 case GF_OMP_TARGET_KIND_UPDATE
:
12679 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
12681 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12682 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12684 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12685 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12686 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
12688 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12689 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12690 start_ix
= BUILT_IN_GOACC_PARALLEL
;
12692 case GF_OMP_TARGET_KIND_OACC_DATA
:
12693 start_ix
= BUILT_IN_GOACC_DATA_START
;
12695 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12696 start_ix
= BUILT_IN_GOACC_UPDATE
;
12698 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12699 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
12701 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
12702 start_ix
= BUILT_IN_GOACC_DECLARE
;
12705 gcc_unreachable ();
12708 clauses
= gimple_omp_target_clauses (entry_stmt
);
12710 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
12711 library choose) and there is no conditional. */
12713 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
12715 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
12717 cond
= OMP_CLAUSE_IF_EXPR (c
);
12719 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
12722 /* Even if we pass it to all library function calls, it is currently only
12723 defined/used for the OpenMP target ones. */
12724 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
12725 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
12726 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
12727 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
12729 device
= OMP_CLAUSE_DEVICE_ID (c
);
12730 clause_loc
= OMP_CLAUSE_LOCATION (c
);
12733 clause_loc
= gimple_location (entry_stmt
);
12735 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
12737 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
12739 /* Ensure 'device' is of the correct type. */
12740 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
12742 /* If we found the clause 'if (cond)', build
12743 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
12746 cond
= gimple_boolify (cond
);
12748 basic_block cond_bb
, then_bb
, else_bb
;
12752 tmp_var
= create_tmp_var (TREE_TYPE (device
));
12754 e
= split_block_after_labels (new_bb
);
12757 gsi
= gsi_last_bb (new_bb
);
12759 e
= split_block (new_bb
, gsi_stmt (gsi
));
12765 then_bb
= create_empty_bb (cond_bb
);
12766 else_bb
= create_empty_bb (then_bb
);
12767 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
12768 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
12770 stmt
= gimple_build_cond_empty (cond
);
12771 gsi
= gsi_last_bb (cond_bb
);
12772 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12774 gsi
= gsi_start_bb (then_bb
);
12775 stmt
= gimple_build_assign (tmp_var
, device
);
12776 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12778 gsi
= gsi_start_bb (else_bb
);
12779 stmt
= gimple_build_assign (tmp_var
,
12780 build_int_cst (integer_type_node
,
12781 GOMP_DEVICE_HOST_FALLBACK
));
12782 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12784 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
12785 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
12786 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
12787 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
12788 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
12789 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
12794 gsi
= gsi_last_bb (new_bb
);
12795 t
= gimple_omp_target_data_arg (entry_stmt
);
12798 t1
= size_zero_node
;
12799 t2
= build_zero_cst (ptr_type_node
);
12805 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
12806 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
12807 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
12808 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
12809 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
12813 bool tagging
= false;
12814 /* The maximum number used by any start_ix, without varargs. */
12815 auto_vec
<tree
, 11> args
;
12816 args
.quick_push (device
);
12818 args
.quick_push (build_fold_addr_expr (child_fn
));
12819 args
.quick_push (t1
);
12820 args
.quick_push (t2
);
12821 args
.quick_push (t3
);
12822 args
.quick_push (t4
);
12825 case BUILT_IN_GOACC_DATA_START
:
12826 case BUILT_IN_GOACC_DECLARE
:
12827 case BUILT_IN_GOMP_TARGET_DATA
:
12829 case BUILT_IN_GOMP_TARGET
:
12830 case BUILT_IN_GOMP_TARGET_UPDATE
:
12831 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
12832 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
12833 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
12835 depend
= OMP_CLAUSE_DECL (c
);
12837 depend
= build_int_cst (ptr_type_node
, 0);
12838 args
.quick_push (depend
);
12839 if (start_ix
== BUILT_IN_GOMP_TARGET
)
12841 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12844 t
= fold_convert (integer_type_node
,
12845 OMP_CLAUSE_NUM_TEAMS_EXPR (c
));
12846 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL
,
12847 true, GSI_SAME_STMT
);
12850 t
= integer_minus_one_node
;
12851 args
.quick_push (t
);
12852 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12855 t
= fold_convert (integer_type_node
,
12856 OMP_CLAUSE_THREAD_LIMIT_EXPR (c
));
12857 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL
,
12858 true, GSI_SAME_STMT
);
12861 t
= integer_minus_one_node
;
12862 args
.quick_push (t
);
12865 case BUILT_IN_GOACC_PARALLEL
:
12867 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
12871 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
12872 case BUILT_IN_GOACC_UPDATE
:
12874 tree t_async
= NULL_TREE
;
12876 /* If present, use the value specified by the respective
12877 clause, making sure that is of the correct type. */
12878 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
12880 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12882 OMP_CLAUSE_ASYNC_EXPR (c
));
12884 /* Default values for t_async. */
12885 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
12887 build_int_cst (integer_type_node
,
12889 if (tagging
&& t_async
)
12891 unsigned HOST_WIDE_INT i_async
;
12893 if (TREE_CODE (t_async
) == INTEGER_CST
)
12895 /* See if we can pack the async arg in to the tag's
12897 i_async
= TREE_INT_CST_LOW (t_async
);
12899 if (i_async
< GOMP_LAUNCH_OP_MAX
)
12900 t_async
= NULL_TREE
;
12903 i_async
= GOMP_LAUNCH_OP_MAX
;
12904 args
.safe_push (oacc_launch_pack
12905 (GOMP_LAUNCH_ASYNC
, NULL_TREE
, i_async
));
12908 args
.safe_push (t_async
);
12910 /* Save the argument index, and ... */
12911 unsigned t_wait_idx
= args
.length ();
12912 unsigned num_waits
= 0;
12913 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
12915 /* ... push a placeholder. */
12916 args
.safe_push (integer_zero_node
);
12918 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
12919 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
12921 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12923 OMP_CLAUSE_WAIT_EXPR (c
)));
12927 if (!tagging
|| num_waits
)
12931 /* Now that we know the number, update the placeholder. */
12933 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
12935 len
= build_int_cst (integer_type_node
, num_waits
);
12936 len
= fold_convert_loc (gimple_location (entry_stmt
),
12937 unsigned_type_node
, len
);
12938 args
[t_wait_idx
] = len
;
12943 gcc_unreachable ();
12946 /* Push terminal marker - zero. */
12947 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
12949 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
12950 gimple_set_location (g
, gimple_location (entry_stmt
));
12951 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12954 g
= gsi_stmt (gsi
);
12955 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
12956 gsi_remove (&gsi
, true);
12958 if (data_region
&& region
->exit
)
12960 gsi
= gsi_last_bb (region
->exit
);
12961 g
= gsi_stmt (gsi
);
12962 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
12963 gsi_remove (&gsi
, true);
12968 /* Expand the parallel region tree rooted at REGION. Expansion
12969 proceeds in depth-first order. Innermost regions are expanded
12970 first. This way, parallel regions that require a new function to
12971 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
12972 internal dependencies in their body. */
12975 expand_omp (struct omp_region
*region
)
12977 omp_any_child_fn_dumped
= false;
12980 location_t saved_location
;
12981 gimple
*inner_stmt
= NULL
;
12983 /* First, determine whether this is a combined parallel+workshare
12985 if (region
->type
== GIMPLE_OMP_PARALLEL
)
12986 determine_parallel_type (region
);
12988 if (region
->type
== GIMPLE_OMP_FOR
12989 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
12990 inner_stmt
= last_stmt (region
->inner
->entry
);
12993 expand_omp (region
->inner
);
12995 saved_location
= input_location
;
12996 if (gimple_has_location (last_stmt (region
->entry
)))
12997 input_location
= gimple_location (last_stmt (region
->entry
));
12999 switch (region
->type
)
13001 case GIMPLE_OMP_PARALLEL
:
13002 case GIMPLE_OMP_TASK
:
13003 expand_omp_taskreg (region
);
13006 case GIMPLE_OMP_FOR
:
13007 expand_omp_for (region
, inner_stmt
);
13010 case GIMPLE_OMP_SECTIONS
:
13011 expand_omp_sections (region
);
13014 case GIMPLE_OMP_SECTION
:
13015 /* Individual omp sections are handled together with their
13016 parent GIMPLE_OMP_SECTIONS region. */
13019 case GIMPLE_OMP_SINGLE
:
13020 expand_omp_single (region
);
13023 case GIMPLE_OMP_ORDERED
:
13025 gomp_ordered
*ord_stmt
13026 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13027 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13028 OMP_CLAUSE_DEPEND
))
13030 /* We'll expand these when expanding corresponding
13031 worksharing region with ordered(n) clause. */
13032 gcc_assert (region
->outer
13033 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13034 region
->ord_stmt
= ord_stmt
;
13039 case GIMPLE_OMP_MASTER
:
13040 case GIMPLE_OMP_TASKGROUP
:
13041 case GIMPLE_OMP_CRITICAL
:
13042 case GIMPLE_OMP_TEAMS
:
13043 expand_omp_synch (region
);
13046 case GIMPLE_OMP_ATOMIC_LOAD
:
13047 expand_omp_atomic (region
);
13050 case GIMPLE_OMP_TARGET
:
13051 expand_omp_target (region
);
13055 gcc_unreachable ();
13058 input_location
= saved_location
;
13059 region
= region
->next
;
13061 if (omp_any_child_fn_dumped
)
13064 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13065 omp_any_child_fn_dumped
= false;
13070 /* Helper for build_omp_regions. Scan the dominator tree starting at
13071 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13072 true, the function ends once a single tree is built (otherwise, whole
13073 forest of OMP constructs may be built). */
13076 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13079 gimple_stmt_iterator gsi
;
13083 gsi
= gsi_last_bb (bb
);
13084 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13086 struct omp_region
*region
;
13087 enum gimple_code code
;
13089 stmt
= gsi_stmt (gsi
);
13090 code
= gimple_code (stmt
);
13091 if (code
== GIMPLE_OMP_RETURN
)
13093 /* STMT is the return point out of region PARENT. Mark it
13094 as the exit point and make PARENT the immediately
13095 enclosing region. */
13096 gcc_assert (parent
);
13099 parent
= parent
->outer
;
13101 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13103 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13104 GIMPLE_OMP_RETURN, but matches with
13105 GIMPLE_OMP_ATOMIC_LOAD. */
13106 gcc_assert (parent
);
13107 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13110 parent
= parent
->outer
;
13112 else if (code
== GIMPLE_OMP_CONTINUE
)
13114 gcc_assert (parent
);
13117 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13119 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13120 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13124 region
= new_omp_region (bb
, code
, parent
);
13126 if (code
== GIMPLE_OMP_TARGET
)
13128 switch (gimple_omp_target_kind (stmt
))
13130 case GF_OMP_TARGET_KIND_REGION
:
13131 case GF_OMP_TARGET_KIND_DATA
:
13132 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13133 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13134 case GF_OMP_TARGET_KIND_OACC_DATA
:
13136 case GF_OMP_TARGET_KIND_UPDATE
:
13137 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13138 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13139 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13140 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13141 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13142 /* ..., other than for those stand-alone directives... */
13146 gcc_unreachable ();
13149 else if (code
== GIMPLE_OMP_ORDERED
13150 && find_omp_clause (gimple_omp_ordered_clauses
13151 (as_a
<gomp_ordered
*> (stmt
)),
13152 OMP_CLAUSE_DEPEND
))
13153 /* #pragma omp ordered depend is also just a stand-alone
13156 /* ..., this directive becomes the parent for a new region. */
13162 if (single_tree
&& !parent
)
13165 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13167 son
= next_dom_son (CDI_DOMINATORS
, son
))
13168 build_omp_regions_1 (son
, parent
, single_tree
);
13171 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13172 root_omp_region. */
13175 build_omp_regions_root (basic_block root
)
13177 gcc_assert (root_omp_region
== NULL
);
13178 build_omp_regions_1 (root
, NULL
, true);
13179 gcc_assert (root_omp_region
!= NULL
);
13182 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13185 omp_expand_local (basic_block head
)
13187 build_omp_regions_root (head
);
13188 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13190 fprintf (dump_file
, "\nOMP region tree\n\n");
13191 dump_omp_region (dump_file
, root_omp_region
, 0);
13192 fprintf (dump_file
, "\n");
13195 remove_exit_barriers (root_omp_region
);
13196 expand_omp (root_omp_region
);
13198 free_omp_regions ();
13201 /* Scan the CFG and build a tree of OMP regions. Return the root of
13202 the OMP region tree. */
13205 build_omp_regions (void)
13207 gcc_assert (root_omp_region
== NULL
);
13208 calculate_dominance_info (CDI_DOMINATORS
);
13209 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
13212 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
13214 static unsigned int
13215 execute_expand_omp (void)
13217 build_omp_regions ();
13219 if (!root_omp_region
)
13224 fprintf (dump_file
, "\nOMP region tree\n\n");
13225 dump_omp_region (dump_file
, root_omp_region
, 0);
13226 fprintf (dump_file
, "\n");
13229 remove_exit_barriers (root_omp_region
);
13231 expand_omp (root_omp_region
);
13233 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13234 verify_loop_structure ();
13235 cleanup_tree_cfg ();
13237 free_omp_regions ();
13242 /* OMP expansion -- the default pass, run before creation of SSA form. */
13246 const pass_data pass_data_expand_omp
=
13248 GIMPLE_PASS
, /* type */
13249 "ompexp", /* name */
13250 OPTGROUP_NONE
, /* optinfo_flags */
13251 TV_NONE
, /* tv_id */
13252 PROP_gimple_any
, /* properties_required */
13253 PROP_gimple_eomp
, /* properties_provided */
13254 0, /* properties_destroyed */
13255 0, /* todo_flags_start */
13256 0, /* todo_flags_finish */
13259 class pass_expand_omp
: public gimple_opt_pass
13262 pass_expand_omp (gcc::context
*ctxt
)
13263 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
13266 /* opt_pass methods: */
13267 virtual unsigned int execute (function
*)
13269 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
13270 || flag_openmp_simd
!= 0)
13271 && !seen_error ());
13273 /* This pass always runs, to provide PROP_gimple_eomp.
13274 But often, there is nothing to do. */
13278 return execute_expand_omp ();
13281 }; // class pass_expand_omp
13283 } // anon namespace
13286 make_pass_expand_omp (gcc::context
*ctxt
)
13288 return new pass_expand_omp (ctxt
);
13293 const pass_data pass_data_expand_omp_ssa
=
13295 GIMPLE_PASS
, /* type */
13296 "ompexpssa", /* name */
13297 OPTGROUP_NONE
, /* optinfo_flags */
13298 TV_NONE
, /* tv_id */
13299 PROP_cfg
| PROP_ssa
, /* properties_required */
13300 PROP_gimple_eomp
, /* properties_provided */
13301 0, /* properties_destroyed */
13302 0, /* todo_flags_start */
13303 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
13306 class pass_expand_omp_ssa
: public gimple_opt_pass
13309 pass_expand_omp_ssa (gcc::context
*ctxt
)
13310 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
13313 /* opt_pass methods: */
13314 virtual bool gate (function
*fun
)
13316 return !(fun
->curr_properties
& PROP_gimple_eomp
);
13318 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
13320 }; // class pass_expand_omp_ssa
13322 } // anon namespace
13325 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
13327 return new pass_expand_omp_ssa (ctxt
);
13330 /* Routines to lower OMP directives into OMP-GIMPLE. */
13332 /* If ctx is a worksharing context inside of a cancellable parallel
13333 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
13334 and conditional branch to parallel's cancel_label to handle
13335 cancellation in the implicit barrier. */
13338 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
13340 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
13341 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
13342 if (gimple_omp_return_nowait_p (omp_return
))
13345 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
13346 && ctx
->outer
->cancellable
)
13348 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
13349 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
13350 tree lhs
= create_tmp_var (c_bool_type
);
13351 gimple_omp_return_set_lhs (omp_return
, lhs
);
13352 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
13353 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
13354 fold_convert (c_bool_type
,
13355 boolean_false_node
),
13356 ctx
->outer
->cancel_label
, fallthru_label
);
13357 gimple_seq_add_stmt (body
, g
);
13358 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
13362 /* Lower the OpenMP sections directive in the current statement in GSI_P.
13363 CTX is the enclosing OMP context for the current statement. */
13366 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13368 tree block
, control
;
13369 gimple_stmt_iterator tgsi
;
13370 gomp_sections
*stmt
;
13372 gbind
*new_stmt
, *bind
;
13373 gimple_seq ilist
, dlist
, olist
, new_body
;
13375 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
13377 push_gimplify_context ();
13381 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
13382 &ilist
, &dlist
, ctx
, NULL
);
13384 new_body
= gimple_omp_body (stmt
);
13385 gimple_omp_set_body (stmt
, NULL
);
13386 tgsi
= gsi_start (new_body
);
13387 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
13392 sec_start
= gsi_stmt (tgsi
);
13393 sctx
= maybe_lookup_ctx (sec_start
);
13396 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
13397 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
13398 GSI_CONTINUE_LINKING
);
13399 gimple_omp_set_body (sec_start
, NULL
);
13401 if (gsi_one_before_end_p (tgsi
))
13403 gimple_seq l
= NULL
;
13404 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
13406 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
13407 gimple_omp_section_set_last (sec_start
);
13410 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
13411 GSI_CONTINUE_LINKING
);
13414 block
= make_node (BLOCK
);
13415 bind
= gimple_build_bind (NULL
, new_body
, block
);
13418 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
13420 block
= make_node (BLOCK
);
13421 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
13422 gsi_replace (gsi_p
, new_stmt
, true);
13424 pop_gimplify_context (new_stmt
);
13425 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
13426 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13427 if (BLOCK_VARS (block
))
13428 TREE_USED (block
) = 1;
13431 gimple_seq_add_seq (&new_body
, ilist
);
13432 gimple_seq_add_stmt (&new_body
, stmt
);
13433 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
13434 gimple_seq_add_stmt (&new_body
, bind
);
13436 control
= create_tmp_var (unsigned_type_node
, ".section");
13437 t
= gimple_build_omp_continue (control
, control
);
13438 gimple_omp_sections_set_control (stmt
, control
);
13439 gimple_seq_add_stmt (&new_body
, t
);
13441 gimple_seq_add_seq (&new_body
, olist
);
13442 if (ctx
->cancellable
)
13443 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
13444 gimple_seq_add_seq (&new_body
, dlist
);
13446 new_body
= maybe_catch_exception (new_body
);
13448 t
= gimple_build_omp_return
13449 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
13450 OMP_CLAUSE_NOWAIT
));
13451 gimple_seq_add_stmt (&new_body
, t
);
13452 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
13454 gimple_bind_set_body (new_stmt
, new_body
);
13458 /* A subroutine of lower_omp_single. Expand the simple form of
13459 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
13461 if (GOMP_single_start ())
13463 [ GOMP_barrier (); ] -> unless 'nowait' is present.
13465 FIXME. It may be better to delay expanding the logic of this until
13466 pass_expand_omp. The expanded logic may make the job more difficult
13467 to a synchronization analysis pass. */
13470 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
13472 location_t loc
= gimple_location (single_stmt
);
13473 tree tlabel
= create_artificial_label (loc
);
13474 tree flabel
= create_artificial_label (loc
);
13475 gimple
*call
, *cond
;
13478 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
13479 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
13480 call
= gimple_build_call (decl
, 0);
13481 gimple_call_set_lhs (call
, lhs
);
13482 gimple_seq_add_stmt (pre_p
, call
);
13484 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
13485 fold_convert_loc (loc
, TREE_TYPE (lhs
),
13486 boolean_true_node
),
13488 gimple_seq_add_stmt (pre_p
, cond
);
13489 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
13490 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13491 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
13495 /* A subroutine of lower_omp_single. Expand the simple form of
13496 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
13498 #pragma omp single copyprivate (a, b, c)
13500 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
13503 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
13509 GOMP_single_copy_end (©out);
13520 FIXME. It may be better to delay expanding the logic of this until
13521 pass_expand_omp. The expanded logic may make the job more difficult
13522 to a synchronization analysis pass. */
13525 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
13528 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
13529 gimple_seq copyin_seq
;
13530 location_t loc
= gimple_location (single_stmt
);
13532 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
13534 ptr_type
= build_pointer_type (ctx
->record_type
);
13535 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
13537 l0
= create_artificial_label (loc
);
13538 l1
= create_artificial_label (loc
);
13539 l2
= create_artificial_label (loc
);
13541 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
13542 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
13543 t
= fold_convert_loc (loc
, ptr_type
, t
);
13544 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
13546 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
13547 build_int_cst (ptr_type
, 0));
13548 t
= build3 (COND_EXPR
, void_type_node
, t
,
13549 build_and_jump (&l0
), build_and_jump (&l1
));
13550 gimplify_and_add (t
, pre_p
);
13552 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
13554 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13557 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
13560 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
13561 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
13562 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
13563 gimplify_and_add (t
, pre_p
);
13565 t
= build_and_jump (&l2
);
13566 gimplify_and_add (t
, pre_p
);
13568 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
13570 gimple_seq_add_seq (pre_p
, copyin_seq
);
13572 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
13576 /* Expand code for an OpenMP single directive. */
13579 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13583 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
13585 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
13587 push_gimplify_context ();
13589 block
= make_node (BLOCK
);
13590 bind
= gimple_build_bind (NULL
, NULL
, block
);
13591 gsi_replace (gsi_p
, bind
, true);
13594 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
13595 &bind_body
, &dlist
, ctx
, NULL
);
13596 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
13598 gimple_seq_add_stmt (&bind_body
, single_stmt
);
13600 if (ctx
->record_type
)
13601 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
13603 lower_omp_single_simple (single_stmt
, &bind_body
);
13605 gimple_omp_set_body (single_stmt
, NULL
);
13607 gimple_seq_add_seq (&bind_body
, dlist
);
13609 bind_body
= maybe_catch_exception (bind_body
);
13611 t
= gimple_build_omp_return
13612 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
13613 OMP_CLAUSE_NOWAIT
));
13614 gimple_seq_add_stmt (&bind_body_tail
, t
);
13615 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
13616 if (ctx
->record_type
)
13618 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
13619 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
13620 TREE_THIS_VOLATILE (clobber
) = 1;
13621 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
13622 clobber
), GSI_SAME_STMT
);
13624 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
13625 gimple_bind_set_body (bind
, bind_body
);
13627 pop_gimplify_context (bind
);
13629 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13630 BLOCK_VARS (block
) = ctx
->block_vars
;
13631 if (BLOCK_VARS (block
))
13632 TREE_USED (block
) = 1;
13636 /* Expand code for an OpenMP master directive. */
13639 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13641 tree block
, lab
= NULL
, x
, bfn_decl
;
13642 gimple
*stmt
= gsi_stmt (*gsi_p
);
13644 location_t loc
= gimple_location (stmt
);
13647 push_gimplify_context ();
13649 block
= make_node (BLOCK
);
13650 bind
= gimple_build_bind (NULL
, NULL
, block
);
13651 gsi_replace (gsi_p
, bind
, true);
13652 gimple_bind_add_stmt (bind
, stmt
);
13654 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
13655 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
13656 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
13657 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
13659 gimplify_and_add (x
, &tseq
);
13660 gimple_bind_add_seq (bind
, tseq
);
13662 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13663 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13664 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13665 gimple_omp_set_body (stmt
, NULL
);
13667 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
13669 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13671 pop_gimplify_context (bind
);
13673 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13674 BLOCK_VARS (block
) = ctx
->block_vars
;
13678 /* Expand code for an OpenMP taskgroup directive. */
13681 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13683 gimple
*stmt
= gsi_stmt (*gsi_p
);
13686 tree block
= make_node (BLOCK
);
13688 bind
= gimple_build_bind (NULL
, NULL
, block
);
13689 gsi_replace (gsi_p
, bind
, true);
13690 gimple_bind_add_stmt (bind
, stmt
);
13692 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
13694 gimple_bind_add_stmt (bind
, x
);
13696 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13697 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13698 gimple_omp_set_body (stmt
, NULL
);
13700 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13702 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13703 BLOCK_VARS (block
) = ctx
->block_vars
;
13707 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
13710 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
13713 struct omp_for_data fd
;
13714 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
13717 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
13718 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
13719 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
13723 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13724 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
13725 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
13726 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
13728 /* Merge depend clauses from multiple adjacent
13729 #pragma omp ordered depend(sink:...) constructs
13730 into one #pragma omp ordered depend(sink:...), so that
13731 we can optimize them together. */
13732 gimple_stmt_iterator gsi
= *gsi_p
;
13734 while (!gsi_end_p (gsi
))
13736 gimple
*stmt
= gsi_stmt (gsi
);
13737 if (is_gimple_debug (stmt
)
13738 || gimple_code (stmt
) == GIMPLE_NOP
)
13743 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
13745 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
13746 c
= gimple_omp_ordered_clauses (ord_stmt2
);
13748 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
13749 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13752 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
13754 gsi_remove (&gsi
, true);
13758 /* Canonicalize sink dependence clauses into one folded clause if
13761 The basic algorithm is to create a sink vector whose first
13762 element is the GCD of all the first elements, and whose remaining
13763 elements are the minimum of the subsequent columns.
13765 We ignore dependence vectors whose first element is zero because
13766 such dependencies are known to be executed by the same thread.
13768 We take into account the direction of the loop, so a minimum
13769 becomes a maximum if the loop is iterating forwards. We also
13770 ignore sink clauses where the loop direction is unknown, or where
13771 the offsets are clearly invalid because they are not a multiple
13772 of the loop increment.
13776 #pragma omp for ordered(2)
13777 for (i=0; i < N; ++i)
13778 for (j=0; j < M; ++j)
13780 #pragma omp ordered \
13781 depend(sink:i-8,j-2) \
13782 depend(sink:i,j-1) \ // Completely ignored because i+0.
13783 depend(sink:i-4,j-3) \
13784 depend(sink:i-6,j-4)
13785 #pragma omp ordered depend(source)
13790 depend(sink:-gcd(8,4,6),-min(2,3,4))
13795 /* FIXME: Computing GCD's where the first element is zero is
13796 non-trivial in the presence of collapsed loops. Do this later. */
13797 if (fd
.collapse
> 1)
13800 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
13801 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
13802 tree folded_dep
= NULL_TREE
;
13803 /* TRUE if the first dimension's offset is negative. */
13804 bool neg_offset_p
= false;
13806 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13808 while ((c
= *list_p
) != NULL
)
13810 bool remove
= false;
13812 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
13813 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13814 goto next_ordered_clause
;
13817 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
13818 vec
&& TREE_CODE (vec
) == TREE_LIST
;
13819 vec
= TREE_CHAIN (vec
), ++i
)
13821 gcc_assert (i
< len
);
13823 /* extract_omp_for_data has canonicalized the condition. */
13824 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
13825 || fd
.loops
[i
].cond_code
== GT_EXPR
);
13826 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
13827 bool maybe_lexically_later
= true;
13829 /* While the committee makes up its mind, bail if we have any
13830 non-constant steps. */
13831 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
13832 goto lower_omp_ordered_ret
;
13834 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
13835 if (POINTER_TYPE_P (itype
))
13837 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
13838 TYPE_PRECISION (itype
),
13839 TYPE_SIGN (itype
));
13841 /* Ignore invalid offsets that are not multiples of the step. */
13842 if (!wi::multiple_of_p
13843 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
13846 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
13847 "ignoring sink clause with offset that is not "
13848 "a multiple of the loop step");
13850 goto next_ordered_clause
;
13853 /* Calculate the first dimension. The first dimension of
13854 the folded dependency vector is the GCD of the first
13855 elements, while ignoring any first elements whose offset
13859 /* Ignore dependence vectors whose first dimension is 0. */
13863 goto next_ordered_clause
;
13867 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
13869 error_at (OMP_CLAUSE_LOCATION (c
),
13870 "first offset must be in opposite direction "
13871 "of loop iterations");
13872 goto lower_omp_ordered_ret
;
13876 neg_offset_p
= forward
;
13877 /* Initialize the first time around. */
13878 if (folded_dep
== NULL_TREE
)
13881 folded_deps
[0] = offset
;
13884 folded_deps
[0] = wi::gcd (folded_deps
[0],
13888 /* Calculate minimum for the remaining dimensions. */
13891 folded_deps
[len
+ i
- 1] = offset
;
13892 if (folded_dep
== c
)
13893 folded_deps
[i
] = offset
;
13894 else if (maybe_lexically_later
13895 && !wi::eq_p (folded_deps
[i
], offset
))
13897 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
13901 for (j
= 1; j
<= i
; j
++)
13902 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
13905 maybe_lexically_later
= false;
13909 gcc_assert (i
== len
);
13913 next_ordered_clause
:
13915 *list_p
= OMP_CLAUSE_CHAIN (c
);
13917 list_p
= &OMP_CLAUSE_CHAIN (c
);
13923 folded_deps
[0] = -folded_deps
[0];
13925 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
13926 if (POINTER_TYPE_P (itype
))
13929 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
13930 = wide_int_to_tree (itype
, folded_deps
[0]);
13931 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
13932 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
13935 lower_omp_ordered_ret
:
13937 /* Ordered without clauses is #pragma omp threads, while we want
13938 a nop instead if we remove all clauses. */
13939 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
13940 gsi_replace (gsi_p
, gimple_build_nop (), true);
13944 /* Expand code for an OpenMP ordered directive. */
13947 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13950 gimple
*stmt
= gsi_stmt (*gsi_p
);
13951 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
13955 = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
), OMP_CLAUSE_SIMD
);
13957 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13958 OMP_CLAUSE_DEPEND
))
13960 /* FIXME: This is needs to be moved to the expansion to verify various
13961 conditions only testable on cfg with dominators computed, and also
13962 all the depend clauses to be merged still might need to be available
13963 for the runtime checks. */
13965 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
13969 push_gimplify_context ();
13971 block
= make_node (BLOCK
);
13972 bind
= gimple_build_bind (NULL
, NULL
, block
);
13973 gsi_replace (gsi_p
, bind
, true);
13974 gimple_bind_add_stmt (bind
, stmt
);
13978 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 0);
13979 cfun
->has_simduid_loops
= true;
13982 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
13984 gimple_bind_add_stmt (bind
, x
);
13986 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13987 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13988 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13989 gimple_omp_set_body (stmt
, NULL
);
13992 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 0);
13994 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
13996 gimple_bind_add_stmt (bind
, x
);
13998 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14000 pop_gimplify_context (bind
);
14002 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14003 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14007 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14008 substitution of a couple of function calls. But in the NAMED case,
14009 requires that languages coordinate a symbol name. It is therefore
14010 best put here in common code. */
14012 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14015 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14018 tree name
, lock
, unlock
;
14019 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14021 location_t loc
= gimple_location (stmt
);
14024 name
= gimple_omp_critical_name (stmt
);
14029 if (!critical_name_mutexes
)
14030 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14032 tree
*n
= critical_name_mutexes
->get (name
);
14037 decl
= create_tmp_var_raw (ptr_type_node
);
14039 new_str
= ACONCAT ((".gomp_critical_user_",
14040 IDENTIFIER_POINTER (name
), NULL
));
14041 DECL_NAME (decl
) = get_identifier (new_str
);
14042 TREE_PUBLIC (decl
) = 1;
14043 TREE_STATIC (decl
) = 1;
14044 DECL_COMMON (decl
) = 1;
14045 DECL_ARTIFICIAL (decl
) = 1;
14046 DECL_IGNORED_P (decl
) = 1;
14048 varpool_node::finalize_decl (decl
);
14050 critical_name_mutexes
->put (name
, decl
);
14055 /* If '#pragma omp critical' is inside offloaded region or
14056 inside function marked as offloadable, the symbol must be
14057 marked as offloadable too. */
14059 if (cgraph_node::get (current_function_decl
)->offloadable
)
14060 varpool_node::get_create (decl
)->offloadable
= 1;
14062 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14063 if (is_gimple_omp_offloaded (octx
->stmt
))
14065 varpool_node::get_create (decl
)->offloadable
= 1;
14069 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14070 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14072 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14073 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14074 build_fold_addr_expr_loc (loc
, decl
));
14078 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14079 lock
= build_call_expr_loc (loc
, lock
, 0);
14081 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14082 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14085 push_gimplify_context ();
14087 block
= make_node (BLOCK
);
14088 bind
= gimple_build_bind (NULL
, NULL
, block
);
14089 gsi_replace (gsi_p
, bind
, true);
14090 gimple_bind_add_stmt (bind
, stmt
);
14092 tbody
= gimple_bind_body (bind
);
14093 gimplify_and_add (lock
, &tbody
);
14094 gimple_bind_set_body (bind
, tbody
);
14096 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14097 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14098 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14099 gimple_omp_set_body (stmt
, NULL
);
14101 tbody
= gimple_bind_body (bind
);
14102 gimplify_and_add (unlock
, &tbody
);
14103 gimple_bind_set_body (bind
, tbody
);
14105 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14107 pop_gimplify_context (bind
);
14108 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14109 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14113 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14114 for a lastprivate clause. Given a loop control predicate of (V
14115 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14116 is appended to *DLIST, iterator initialization is appended to
14120 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14121 gimple_seq
*dlist
, struct omp_context
*ctx
)
14123 tree clauses
, cond
, vinit
;
14124 enum tree_code cond_code
;
14127 cond_code
= fd
->loop
.cond_code
;
14128 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14130 /* When possible, use a strict equality expression. This can let VRP
14131 type optimizations deduce the value and remove a copy. */
14132 if (tree_fits_shwi_p (fd
->loop
.step
))
14134 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14135 if (step
== 1 || step
== -1)
14136 cond_code
= EQ_EXPR
;
14139 tree n2
= fd
->loop
.n2
;
14140 if (fd
->collapse
> 1
14141 && TREE_CODE (n2
) != INTEGER_CST
14142 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14144 struct omp_context
*taskreg_ctx
= NULL
;
14145 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14147 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14148 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14149 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14151 if (gimple_omp_for_combined_into_p (gfor
))
14153 gcc_assert (ctx
->outer
->outer
14154 && is_parallel_ctx (ctx
->outer
->outer
));
14155 taskreg_ctx
= ctx
->outer
->outer
;
14159 struct omp_for_data outer_fd
;
14160 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14161 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14164 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14165 taskreg_ctx
= ctx
->outer
->outer
;
14167 else if (is_taskreg_ctx (ctx
->outer
))
14168 taskreg_ctx
= ctx
->outer
;
14173 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14174 OMP_CLAUSE__LOOPTEMP_
);
14175 gcc_assert (innerc
);
14176 for (i
= 0; i
< fd
->collapse
; i
++)
14178 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14179 OMP_CLAUSE__LOOPTEMP_
);
14180 gcc_assert (innerc
);
14182 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14183 OMP_CLAUSE__LOOPTEMP_
);
14185 n2
= fold_convert (TREE_TYPE (n2
),
14186 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14190 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14192 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14194 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14195 if (!gimple_seq_empty_p (stmts
))
14197 gimple_seq_add_seq (&stmts
, *dlist
);
14200 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
14201 vinit
= fd
->loop
.n1
;
14202 if (cond_code
== EQ_EXPR
14203 && tree_fits_shwi_p (fd
->loop
.n2
)
14204 && ! integer_zerop (fd
->loop
.n2
))
14205 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
14207 vinit
= unshare_expr (vinit
);
14209 /* Initialize the iterator variable, so that threads that don't execute
14210 any iterations don't execute the lastprivate clauses by accident. */
14211 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
14216 /* Lower code for an OMP loop directive. */
14219 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14221 tree
*rhs_p
, block
;
14222 struct omp_for_data fd
, *fdp
= NULL
;
14223 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
14225 gimple_seq omp_for_body
, body
, dlist
;
14226 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
14229 push_gimplify_context ();
14231 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
14233 block
= make_node (BLOCK
);
14234 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14235 /* Replace at gsi right away, so that 'stmt' is no member
14236 of a sequence anymore as we're going to add to a different
14238 gsi_replace (gsi_p
, new_stmt
, true);
14240 /* Move declaration of temporaries in the loop body before we make
14242 omp_for_body
= gimple_omp_body (stmt
);
14243 if (!gimple_seq_empty_p (omp_for_body
)
14244 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
14247 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
14248 tree vars
= gimple_bind_vars (inner_bind
);
14249 gimple_bind_append_vars (new_stmt
, vars
);
14250 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
14251 keep them on the inner_bind and it's block. */
14252 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
14253 if (gimple_bind_block (inner_bind
))
14254 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
14257 if (gimple_omp_for_combined_into_p (stmt
))
14259 extract_omp_for_data (stmt
, &fd
, NULL
);
14262 /* We need two temporaries with fd.loop.v type (istart/iend)
14263 and then (fd.collapse - 1) temporaries with the same
14264 type for count2 ... countN-1 vars if not constant. */
14266 tree type
= fd
.iter_type
;
14267 if (fd
.collapse
> 1
14268 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
14269 count
+= fd
.collapse
- 1;
14271 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
14272 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
14273 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
14274 tree clauses
= *pc
;
14277 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
14278 OMP_CLAUSE__LOOPTEMP_
);
14279 for (i
= 0; i
< count
; i
++)
14284 gcc_assert (outerc
);
14285 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
14286 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
14287 OMP_CLAUSE__LOOPTEMP_
);
14291 temp
= create_tmp_var (type
);
14292 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
14294 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
14295 OMP_CLAUSE_DECL (*pc
) = temp
;
14296 pc
= &OMP_CLAUSE_CHAIN (*pc
);
14301 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
14304 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
14306 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
14308 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14310 /* Lower the header expressions. At this point, we can assume that
14311 the header is of the form:
14313 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
14315 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
14316 using the .omp_data_s mapping, if needed. */
14317 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
14319 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
14320 if (!is_gimple_min_invariant (*rhs_p
))
14321 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14323 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
14324 if (!is_gimple_min_invariant (*rhs_p
))
14325 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14327 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
14328 if (!is_gimple_min_invariant (*rhs_p
))
14329 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14332 /* Once lowered, extract the bounds and clauses. */
14333 extract_omp_for_data (stmt
, &fd
, NULL
);
14335 if (is_gimple_omp_oacc (ctx
->stmt
)
14336 && !ctx_in_oacc_kernels_region (ctx
))
14337 lower_oacc_head_tail (gimple_location (stmt
),
14338 gimple_omp_for_clauses (stmt
),
14339 &oacc_head
, &oacc_tail
, ctx
);
14341 /* Add OpenACC partitioning and reduction markers just before the loop */
14343 gimple_seq_add_seq (&body
, oacc_head
);
14345 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
14347 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
14348 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14349 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
14350 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
14352 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
14353 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
14354 OMP_CLAUSE_LINEAR_STEP (c
)
14355 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
14359 gimple_seq_add_stmt (&body
, stmt
);
14360 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
14362 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
14365 /* After the loop, add exit clauses. */
14366 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
14368 if (ctx
->cancellable
)
14369 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
14371 gimple_seq_add_seq (&body
, dlist
);
14373 body
= maybe_catch_exception (body
);
14375 /* Region exit marker goes at the end of the loop body. */
14376 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
14377 maybe_add_implicit_barrier_cancel (ctx
, &body
);
14379 /* Add OpenACC joining and reduction markers just after the loop. */
14381 gimple_seq_add_seq (&body
, oacc_tail
);
14383 pop_gimplify_context (new_stmt
);
14385 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14386 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
14387 if (BLOCK_VARS (block
))
14388 TREE_USED (block
) = 1;
14390 gimple_bind_set_body (new_stmt
, body
);
14391 gimple_omp_set_body (stmt
, NULL
);
14392 gimple_omp_for_set_pre_body (stmt
, NULL
);
14395 /* Callback for walk_stmts. Check if the current statement only contains
14396 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
14399 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
14400 bool *handled_ops_p
,
14401 struct walk_stmt_info
*wi
)
14403 int *info
= (int *) wi
->info
;
14404 gimple
*stmt
= gsi_stmt (*gsi_p
);
14406 *handled_ops_p
= true;
14407 switch (gimple_code (stmt
))
14411 case GIMPLE_OMP_FOR
:
14412 case GIMPLE_OMP_SECTIONS
:
14413 *info
= *info
== 0 ? 1 : -1;
14422 struct omp_taskcopy_context
14424 /* This field must be at the beginning, as we do "inheritance": Some
14425 callback functions for tree-inline.c (e.g., omp_copy_decl)
14426 receive a copy_body_data pointer that is up-casted to an
14427 omp_context pointer. */
14433 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
14435 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
14437 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
14438 return create_tmp_var (TREE_TYPE (var
));
14444 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
14446 tree name
, new_fields
= NULL
, type
, f
;
14448 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
14449 name
= DECL_NAME (TYPE_NAME (orig_type
));
14450 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
14451 TYPE_DECL
, name
, type
);
14452 TYPE_NAME (type
) = name
;
14454 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
14456 tree new_f
= copy_node (f
);
14457 DECL_CONTEXT (new_f
) = type
;
14458 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
14459 TREE_CHAIN (new_f
) = new_fields
;
14460 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14461 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14462 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
14464 new_fields
= new_f
;
14465 tcctx
->cb
.decl_map
->put (f
, new_f
);
14467 TYPE_FIELDS (type
) = nreverse (new_fields
);
14468 layout_type (type
);
14472 /* Create task copyfn. */
14475 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
14477 struct function
*child_cfun
;
14478 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
14479 tree record_type
, srecord_type
, bind
, list
;
14480 bool record_needs_remap
= false, srecord_needs_remap
= false;
14482 struct omp_taskcopy_context tcctx
;
14483 location_t loc
= gimple_location (task_stmt
);
14485 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
14486 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
14487 gcc_assert (child_cfun
->cfg
== NULL
);
14488 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
14490 /* Reset DECL_CONTEXT on function arguments. */
14491 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
14492 DECL_CONTEXT (t
) = child_fn
;
14494 /* Populate the function. */
14495 push_gimplify_context ();
14496 push_cfun (child_cfun
);
14498 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
14499 TREE_SIDE_EFFECTS (bind
) = 1;
14501 DECL_SAVED_TREE (child_fn
) = bind
;
14502 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
14504 /* Remap src and dst argument types if needed. */
14505 record_type
= ctx
->record_type
;
14506 srecord_type
= ctx
->srecord_type
;
14507 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
14508 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14510 record_needs_remap
= true;
14513 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
14514 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14516 srecord_needs_remap
= true;
14520 if (record_needs_remap
|| srecord_needs_remap
)
14522 memset (&tcctx
, '\0', sizeof (tcctx
));
14523 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
14524 tcctx
.cb
.dst_fn
= child_fn
;
14525 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
14526 gcc_checking_assert (tcctx
.cb
.src_node
);
14527 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
14528 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
14529 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
14530 tcctx
.cb
.eh_lp_nr
= 0;
14531 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
14532 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
14535 if (record_needs_remap
)
14536 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
14537 if (srecord_needs_remap
)
14538 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
14541 tcctx
.cb
.decl_map
= NULL
;
14543 arg
= DECL_ARGUMENTS (child_fn
);
14544 TREE_TYPE (arg
) = build_pointer_type (record_type
);
14545 sarg
= DECL_CHAIN (arg
);
14546 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
14548 /* First pass: initialize temporaries used in record_type and srecord_type
14549 sizes and field offsets. */
14550 if (tcctx
.cb
.decl_map
)
14551 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14552 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14556 decl
= OMP_CLAUSE_DECL (c
);
14557 p
= tcctx
.cb
.decl_map
->get (decl
);
14560 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14561 sf
= (tree
) n
->value
;
14562 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14563 src
= build_simple_mem_ref_loc (loc
, sarg
);
14564 src
= omp_build_component_ref (src
, sf
);
14565 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
14566 append_to_statement_list (t
, &list
);
14569 /* Second pass: copy shared var pointers and copy construct non-VLA
14570 firstprivate vars. */
14571 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14572 switch (OMP_CLAUSE_CODE (c
))
14574 splay_tree_key key
;
14575 case OMP_CLAUSE_SHARED
:
14576 decl
= OMP_CLAUSE_DECL (c
);
14577 key
= (splay_tree_key
) decl
;
14578 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
14579 key
= (splay_tree_key
) &DECL_UID (decl
);
14580 n
= splay_tree_lookup (ctx
->field_map
, key
);
14583 f
= (tree
) n
->value
;
14584 if (tcctx
.cb
.decl_map
)
14585 f
= *tcctx
.cb
.decl_map
->get (f
);
14586 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
14587 sf
= (tree
) n
->value
;
14588 if (tcctx
.cb
.decl_map
)
14589 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14590 src
= build_simple_mem_ref_loc (loc
, sarg
);
14591 src
= omp_build_component_ref (src
, sf
);
14592 dst
= build_simple_mem_ref_loc (loc
, arg
);
14593 dst
= omp_build_component_ref (dst
, f
);
14594 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14595 append_to_statement_list (t
, &list
);
14597 case OMP_CLAUSE_FIRSTPRIVATE
:
14598 decl
= OMP_CLAUSE_DECL (c
);
14599 if (is_variable_sized (decl
))
14601 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14604 f
= (tree
) n
->value
;
14605 if (tcctx
.cb
.decl_map
)
14606 f
= *tcctx
.cb
.decl_map
->get (f
);
14607 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14610 sf
= (tree
) n
->value
;
14611 if (tcctx
.cb
.decl_map
)
14612 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14613 src
= build_simple_mem_ref_loc (loc
, sarg
);
14614 src
= omp_build_component_ref (src
, sf
);
14615 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
14616 src
= build_simple_mem_ref_loc (loc
, src
);
14620 dst
= build_simple_mem_ref_loc (loc
, arg
);
14621 dst
= omp_build_component_ref (dst
, f
);
14622 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14623 append_to_statement_list (t
, &list
);
14625 case OMP_CLAUSE_PRIVATE
:
14626 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
14628 decl
= OMP_CLAUSE_DECL (c
);
14629 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14630 f
= (tree
) n
->value
;
14631 if (tcctx
.cb
.decl_map
)
14632 f
= *tcctx
.cb
.decl_map
->get (f
);
14633 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14636 sf
= (tree
) n
->value
;
14637 if (tcctx
.cb
.decl_map
)
14638 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14639 src
= build_simple_mem_ref_loc (loc
, sarg
);
14640 src
= omp_build_component_ref (src
, sf
);
14641 if (use_pointer_for_field (decl
, NULL
))
14642 src
= build_simple_mem_ref_loc (loc
, src
);
14646 dst
= build_simple_mem_ref_loc (loc
, arg
);
14647 dst
= omp_build_component_ref (dst
, f
);
14648 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14649 append_to_statement_list (t
, &list
);
14655 /* Last pass: handle VLA firstprivates. */
14656 if (tcctx
.cb
.decl_map
)
14657 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14658 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14662 decl
= OMP_CLAUSE_DECL (c
);
14663 if (!is_variable_sized (decl
))
14665 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14668 f
= (tree
) n
->value
;
14669 f
= *tcctx
.cb
.decl_map
->get (f
);
14670 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
14671 ind
= DECL_VALUE_EXPR (decl
);
14672 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
14673 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
14674 n
= splay_tree_lookup (ctx
->sfield_map
,
14675 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14676 sf
= (tree
) n
->value
;
14677 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14678 src
= build_simple_mem_ref_loc (loc
, sarg
);
14679 src
= omp_build_component_ref (src
, sf
);
14680 src
= build_simple_mem_ref_loc (loc
, src
);
14681 dst
= build_simple_mem_ref_loc (loc
, arg
);
14682 dst
= omp_build_component_ref (dst
, f
);
14683 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14684 append_to_statement_list (t
, &list
);
14685 n
= splay_tree_lookup (ctx
->field_map
,
14686 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14687 df
= (tree
) n
->value
;
14688 df
= *tcctx
.cb
.decl_map
->get (df
);
14689 ptr
= build_simple_mem_ref_loc (loc
, arg
);
14690 ptr
= omp_build_component_ref (ptr
, df
);
14691 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
14692 build_fold_addr_expr_loc (loc
, dst
));
14693 append_to_statement_list (t
, &list
);
14696 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
14697 append_to_statement_list (t
, &list
);
14699 if (tcctx
.cb
.decl_map
)
14700 delete tcctx
.cb
.decl_map
;
14701 pop_gimplify_context (NULL
);
14702 BIND_EXPR_BODY (bind
) = list
;
14707 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
14711 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
14713 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
14714 gcc_assert (clauses
);
14715 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14716 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
14717 switch (OMP_CLAUSE_DEPEND_KIND (c
))
14719 case OMP_CLAUSE_DEPEND_IN
:
14722 case OMP_CLAUSE_DEPEND_OUT
:
14723 case OMP_CLAUSE_DEPEND_INOUT
:
14726 case OMP_CLAUSE_DEPEND_SOURCE
:
14727 case OMP_CLAUSE_DEPEND_SINK
:
14730 gcc_unreachable ();
14732 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
14733 tree array
= create_tmp_var (type
);
14734 TREE_ADDRESSABLE (array
) = 1;
14735 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
14737 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
14738 gimple_seq_add_stmt (iseq
, g
);
14739 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
14741 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
14742 gimple_seq_add_stmt (iseq
, g
);
14743 for (i
= 0; i
< 2; i
++)
14745 if ((i
? n_in
: n_out
) == 0)
14747 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14748 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14749 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
14751 tree t
= OMP_CLAUSE_DECL (c
);
14752 t
= fold_convert (ptr_type_node
, t
);
14753 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
14754 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
14755 NULL_TREE
, NULL_TREE
);
14756 g
= gimple_build_assign (r
, t
);
14757 gimple_seq_add_stmt (iseq
, g
);
14760 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
14761 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
14762 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
14764 tree clobber
= build_constructor (type
, NULL
);
14765 TREE_THIS_VOLATILE (clobber
) = 1;
14766 g
= gimple_build_assign (array
, clobber
);
14767 gimple_seq_add_stmt (oseq
, g
);
14770 /* Lower the OpenMP parallel or task directive in the current statement
14771 in GSI_P. CTX holds context information for the directive. */
14774 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14778 gimple
*stmt
= gsi_stmt (*gsi_p
);
14779 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
14780 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
14781 location_t loc
= gimple_location (stmt
);
14783 clauses
= gimple_omp_taskreg_clauses (stmt
);
14785 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
14786 par_body
= gimple_bind_body (par_bind
);
14787 child_fn
= ctx
->cb
.dst_fn
;
14788 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
14789 && !gimple_omp_parallel_combined_p (stmt
))
14791 struct walk_stmt_info wi
;
14794 memset (&wi
, 0, sizeof (wi
));
14796 wi
.val_only
= true;
14797 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
14799 gimple_omp_parallel_set_combined_p (stmt
, true);
14801 gimple_seq dep_ilist
= NULL
;
14802 gimple_seq dep_olist
= NULL
;
14803 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
14804 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14806 push_gimplify_context ();
14807 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14808 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
14809 &dep_ilist
, &dep_olist
);
14812 if (ctx
->srecord_type
)
14813 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
14815 push_gimplify_context ();
14820 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
14821 lower_omp (&par_body
, ctx
);
14822 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
14823 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
14825 /* Declare all the variables created by mapping and the variables
14826 declared in the scope of the parallel body. */
14827 record_vars_into (ctx
->block_vars
, child_fn
);
14828 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
14830 if (ctx
->record_type
)
14833 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
14834 : ctx
->record_type
, ".omp_data_o");
14835 DECL_NAMELESS (ctx
->sender_decl
) = 1;
14836 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
14837 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
14842 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
14843 lower_send_shared_vars (&ilist
, &olist
, ctx
);
14845 if (ctx
->record_type
)
14847 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
14848 TREE_THIS_VOLATILE (clobber
) = 1;
14849 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
14853 /* Once all the expansions are done, sequence all the different
14854 fragments inside gimple_omp_body. */
14858 if (ctx
->record_type
)
14860 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14861 /* fixup_child_record_type might have changed receiver_decl's type. */
14862 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
14863 gimple_seq_add_stmt (&new_body
,
14864 gimple_build_assign (ctx
->receiver_decl
, t
));
14867 gimple_seq_add_seq (&new_body
, par_ilist
);
14868 gimple_seq_add_seq (&new_body
, par_body
);
14869 gimple_seq_add_seq (&new_body
, par_rlist
);
14870 if (ctx
->cancellable
)
14871 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14872 gimple_seq_add_seq (&new_body
, par_olist
);
14873 new_body
= maybe_catch_exception (new_body
);
14874 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
14875 gimple_seq_add_stmt (&new_body
,
14876 gimple_build_omp_continue (integer_zero_node
,
14877 integer_zero_node
));
14878 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
14879 gimple_omp_set_body (stmt
, new_body
);
14881 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
14882 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
14883 gimple_bind_add_seq (bind
, ilist
);
14884 gimple_bind_add_stmt (bind
, stmt
);
14885 gimple_bind_add_seq (bind
, olist
);
14887 pop_gimplify_context (NULL
);
14891 gimple_bind_add_seq (dep_bind
, dep_ilist
);
14892 gimple_bind_add_stmt (dep_bind
, bind
);
14893 gimple_bind_add_seq (dep_bind
, dep_olist
);
14894 pop_gimplify_context (dep_bind
);
14898 /* Lower the GIMPLE_OMP_TARGET in the current statement
14899 in GSI_P. CTX holds context information for the directive. */
14902 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14905 tree child_fn
, t
, c
;
14906 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
14907 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
14908 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
14909 location_t loc
= gimple_location (stmt
);
14910 bool offloaded
, data_region
;
14911 unsigned int map_cnt
= 0;
14912 bool has_depend
= false;
14914 offloaded
= is_gimple_omp_offloaded (stmt
);
14915 switch (gimple_omp_target_kind (stmt
))
14917 case GF_OMP_TARGET_KIND_REGION
:
14918 case GF_OMP_TARGET_KIND_UPDATE
:
14919 case GF_OMP_TARGET_KIND_ENTER_DATA
:
14920 case GF_OMP_TARGET_KIND_EXIT_DATA
:
14921 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
14922 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
14923 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
14924 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
14925 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
14926 data_region
= false;
14928 case GF_OMP_TARGET_KIND_DATA
:
14929 case GF_OMP_TARGET_KIND_OACC_DATA
:
14930 data_region
= true;
14933 gcc_unreachable ();
14936 clauses
= gimple_omp_target_clauses (stmt
);
14938 gimple_seq dep_ilist
= NULL
;
14939 gimple_seq dep_olist
= NULL
;
14940 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14942 push_gimplify_context ();
14943 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14944 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
14945 &dep_ilist
, &dep_olist
);
14953 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
14954 tgt_body
= gimple_bind_body (tgt_bind
);
14956 else if (data_region
)
14957 tgt_body
= gimple_omp_body (stmt
);
14958 child_fn
= ctx
->cb
.dst_fn
;
14960 push_gimplify_context ();
14963 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14964 switch (OMP_CLAUSE_CODE (c
))
14970 case OMP_CLAUSE_MAP
:
14972 /* First check what we're prepared to handle in the following. */
14973 switch (OMP_CLAUSE_MAP_KIND (c
))
14975 case GOMP_MAP_ALLOC
:
14977 case GOMP_MAP_FROM
:
14978 case GOMP_MAP_TOFROM
:
14979 case GOMP_MAP_POINTER
:
14980 case GOMP_MAP_TO_PSET
:
14981 case GOMP_MAP_FORCE_DEALLOC
:
14982 case GOMP_MAP_RELEASE
:
14983 case GOMP_MAP_ALWAYS_TO
:
14984 case GOMP_MAP_ALWAYS_FROM
:
14985 case GOMP_MAP_ALWAYS_TOFROM
:
14986 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
14987 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
14988 case GOMP_MAP_STRUCT
:
14989 case GOMP_MAP_ALWAYS_POINTER
:
14991 case GOMP_MAP_FORCE_ALLOC
:
14992 case GOMP_MAP_FORCE_TO
:
14993 case GOMP_MAP_FORCE_FROM
:
14994 case GOMP_MAP_FORCE_TOFROM
:
14995 case GOMP_MAP_FORCE_PRESENT
:
14996 case GOMP_MAP_FORCE_DEVICEPTR
:
14997 case GOMP_MAP_DEVICE_RESIDENT
:
14998 case GOMP_MAP_LINK
:
14999 gcc_assert (is_gimple_omp_oacc (stmt
));
15002 gcc_unreachable ();
15006 case OMP_CLAUSE_TO
:
15007 case OMP_CLAUSE_FROM
:
15009 var
= OMP_CLAUSE_DECL (c
);
15012 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15013 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15014 && (OMP_CLAUSE_MAP_KIND (c
)
15015 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15020 if (DECL_SIZE (var
)
15021 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15023 tree var2
= DECL_VALUE_EXPR (var
);
15024 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15025 var2
= TREE_OPERAND (var2
, 0);
15026 gcc_assert (DECL_P (var2
));
15031 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15032 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15033 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15035 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15037 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15038 && varpool_node::get_create (var
)->offloadable
)
15041 tree type
= build_pointer_type (TREE_TYPE (var
));
15042 tree new_var
= lookup_decl (var
, ctx
);
15043 x
= create_tmp_var_raw (type
, get_name (new_var
));
15044 gimple_add_tmp_var (x
);
15045 x
= build_simple_mem_ref (x
);
15046 SET_DECL_VALUE_EXPR (new_var
, x
);
15047 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15052 if (!maybe_lookup_field (var
, ctx
))
15057 x
= build_receiver_ref (var
, true, ctx
);
15058 tree new_var
= lookup_decl (var
, ctx
);
15060 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15061 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15062 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15063 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15064 x
= build_simple_mem_ref (x
);
15065 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15067 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15068 if (is_reference (new_var
))
15070 /* Create a local object to hold the instance
15072 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15073 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15074 tree inst
= create_tmp_var (type
, id
);
15075 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15076 x
= build_fold_addr_expr (inst
);
15078 gimplify_assign (new_var
, x
, &fplist
);
15080 else if (DECL_P (new_var
))
15082 SET_DECL_VALUE_EXPR (new_var
, x
);
15083 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15086 gcc_unreachable ();
15091 case OMP_CLAUSE_FIRSTPRIVATE
:
15092 if (is_oacc_parallel (ctx
))
15093 goto oacc_firstprivate
;
15095 var
= OMP_CLAUSE_DECL (c
);
15096 if (!is_reference (var
)
15097 && !is_gimple_reg_type (TREE_TYPE (var
)))
15099 tree new_var
= lookup_decl (var
, ctx
);
15100 if (is_variable_sized (var
))
15102 tree pvar
= DECL_VALUE_EXPR (var
);
15103 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15104 pvar
= TREE_OPERAND (pvar
, 0);
15105 gcc_assert (DECL_P (pvar
));
15106 tree new_pvar
= lookup_decl (pvar
, ctx
);
15107 x
= build_fold_indirect_ref (new_pvar
);
15108 TREE_THIS_NOTRAP (x
) = 1;
15111 x
= build_receiver_ref (var
, true, ctx
);
15112 SET_DECL_VALUE_EXPR (new_var
, x
);
15113 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15117 case OMP_CLAUSE_PRIVATE
:
15118 if (is_gimple_omp_oacc (ctx
->stmt
))
15120 var
= OMP_CLAUSE_DECL (c
);
15121 if (is_variable_sized (var
))
15123 tree new_var
= lookup_decl (var
, ctx
);
15124 tree pvar
= DECL_VALUE_EXPR (var
);
15125 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15126 pvar
= TREE_OPERAND (pvar
, 0);
15127 gcc_assert (DECL_P (pvar
));
15128 tree new_pvar
= lookup_decl (pvar
, ctx
);
15129 x
= build_fold_indirect_ref (new_pvar
);
15130 TREE_THIS_NOTRAP (x
) = 1;
15131 SET_DECL_VALUE_EXPR (new_var
, x
);
15132 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15136 case OMP_CLAUSE_USE_DEVICE_PTR
:
15137 case OMP_CLAUSE_IS_DEVICE_PTR
:
15138 var
= OMP_CLAUSE_DECL (c
);
15140 if (is_variable_sized (var
))
15142 tree new_var
= lookup_decl (var
, ctx
);
15143 tree pvar
= DECL_VALUE_EXPR (var
);
15144 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15145 pvar
= TREE_OPERAND (pvar
, 0);
15146 gcc_assert (DECL_P (pvar
));
15147 tree new_pvar
= lookup_decl (pvar
, ctx
);
15148 x
= build_fold_indirect_ref (new_pvar
);
15149 TREE_THIS_NOTRAP (x
) = 1;
15150 SET_DECL_VALUE_EXPR (new_var
, x
);
15151 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15153 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15155 tree new_var
= lookup_decl (var
, ctx
);
15156 tree type
= build_pointer_type (TREE_TYPE (var
));
15157 x
= create_tmp_var_raw (type
, get_name (new_var
));
15158 gimple_add_tmp_var (x
);
15159 x
= build_simple_mem_ref (x
);
15160 SET_DECL_VALUE_EXPR (new_var
, x
);
15161 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15168 target_nesting_level
++;
15169 lower_omp (&tgt_body
, ctx
);
15170 target_nesting_level
--;
15172 else if (data_region
)
15173 lower_omp (&tgt_body
, ctx
);
15177 /* Declare all the variables created by mapping and the variables
15178 declared in the scope of the target body. */
15179 record_vars_into (ctx
->block_vars
, child_fn
);
15180 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
15185 if (ctx
->record_type
)
15188 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
15189 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15190 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15191 t
= make_tree_vec (3);
15192 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
15193 TREE_VEC_ELT (t
, 1)
15194 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
15195 ".omp_data_sizes");
15196 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
15197 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
15198 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
15199 tree tkind_type
= short_unsigned_type_node
;
15200 int talign_shift
= 8;
15201 TREE_VEC_ELT (t
, 2)
15202 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
15203 ".omp_data_kinds");
15204 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
15205 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
15206 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
15207 gimple_omp_target_set_data_arg (stmt
, t
);
15209 vec
<constructor_elt
, va_gc
> *vsize
;
15210 vec
<constructor_elt
, va_gc
> *vkind
;
15211 vec_alloc (vsize
, map_cnt
);
15212 vec_alloc (vkind
, map_cnt
);
15213 unsigned int map_idx
= 0;
15215 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15216 switch (OMP_CLAUSE_CODE (c
))
15218 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
15219 unsigned int talign
;
15224 case OMP_CLAUSE_MAP
:
15225 case OMP_CLAUSE_TO
:
15226 case OMP_CLAUSE_FROM
:
15227 oacc_firstprivate_map
:
15229 ovar
= OMP_CLAUSE_DECL (c
);
15230 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15231 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15232 || (OMP_CLAUSE_MAP_KIND (c
)
15233 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15235 if (!DECL_P (ovar
))
15237 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15238 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
15240 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
15241 == get_base_address (ovar
));
15242 nc
= OMP_CLAUSE_CHAIN (c
);
15243 ovar
= OMP_CLAUSE_DECL (nc
);
15247 tree x
= build_sender_ref (ovar
, ctx
);
15249 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
15250 gimplify_assign (x
, v
, &ilist
);
15256 if (DECL_SIZE (ovar
)
15257 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
15259 tree ovar2
= DECL_VALUE_EXPR (ovar
);
15260 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
15261 ovar2
= TREE_OPERAND (ovar2
, 0);
15262 gcc_assert (DECL_P (ovar2
));
15265 if (!maybe_lookup_field (ovar
, ctx
))
15269 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
15270 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
15271 talign
= DECL_ALIGN_UNIT (ovar
);
15274 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15275 x
= build_sender_ref (ovar
, ctx
);
15277 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15278 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15279 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15280 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
15282 gcc_assert (offloaded
);
15284 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
15285 mark_addressable (avar
);
15286 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
15287 talign
= DECL_ALIGN_UNIT (avar
);
15288 avar
= build_fold_addr_expr (avar
);
15289 gimplify_assign (x
, avar
, &ilist
);
15291 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15293 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15294 if (!is_reference (var
))
15295 var
= build_fold_addr_expr (var
);
15297 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15298 gimplify_assign (x
, var
, &ilist
);
15300 else if (is_gimple_reg (var
))
15302 gcc_assert (offloaded
);
15303 tree avar
= create_tmp_var (TREE_TYPE (var
));
15304 mark_addressable (avar
);
15305 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
15306 if (GOMP_MAP_COPY_TO_P (map_kind
)
15307 || map_kind
== GOMP_MAP_POINTER
15308 || map_kind
== GOMP_MAP_TO_PSET
15309 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15310 gimplify_assign (avar
, var
, &ilist
);
15311 avar
= build_fold_addr_expr (avar
);
15312 gimplify_assign (x
, avar
, &ilist
);
15313 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
15314 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15315 && !TYPE_READONLY (TREE_TYPE (var
)))
15317 x
= unshare_expr (x
);
15318 x
= build_simple_mem_ref (x
);
15319 gimplify_assign (var
, x
, &olist
);
15324 var
= build_fold_addr_expr (var
);
15325 gimplify_assign (x
, var
, &ilist
);
15329 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15331 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
15332 s
= TREE_TYPE (ovar
);
15333 if (TREE_CODE (s
) == REFERENCE_TYPE
)
15335 s
= TYPE_SIZE_UNIT (s
);
15338 s
= OMP_CLAUSE_SIZE (c
);
15339 if (s
== NULL_TREE
)
15340 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15341 s
= fold_convert (size_type_node
, s
);
15342 purpose
= size_int (map_idx
++);
15343 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15344 if (TREE_CODE (s
) != INTEGER_CST
)
15345 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15347 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
15348 switch (OMP_CLAUSE_CODE (c
))
15350 case OMP_CLAUSE_MAP
:
15351 tkind
= OMP_CLAUSE_MAP_KIND (c
);
15352 tkind_zero
= tkind
;
15353 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
15356 case GOMP_MAP_ALLOC
:
15358 case GOMP_MAP_FROM
:
15359 case GOMP_MAP_TOFROM
:
15360 case GOMP_MAP_ALWAYS_TO
:
15361 case GOMP_MAP_ALWAYS_FROM
:
15362 case GOMP_MAP_ALWAYS_TOFROM
:
15363 case GOMP_MAP_RELEASE
:
15364 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
15366 case GOMP_MAP_DELETE
:
15367 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
15371 if (tkind_zero
!= tkind
)
15373 if (integer_zerop (s
))
15374 tkind
= tkind_zero
;
15375 else if (integer_nonzerop (s
))
15376 tkind_zero
= tkind
;
15379 case OMP_CLAUSE_FIRSTPRIVATE
:
15380 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
15381 tkind
= GOMP_MAP_TO
;
15382 tkind_zero
= tkind
;
15384 case OMP_CLAUSE_TO
:
15385 tkind
= GOMP_MAP_TO
;
15386 tkind_zero
= tkind
;
15388 case OMP_CLAUSE_FROM
:
15389 tkind
= GOMP_MAP_FROM
;
15390 tkind_zero
= tkind
;
15393 gcc_unreachable ();
15395 gcc_checking_assert (tkind
15396 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15397 gcc_checking_assert (tkind_zero
15398 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15399 talign
= ceil_log2 (talign
);
15400 tkind
|= talign
<< talign_shift
;
15401 tkind_zero
|= talign
<< talign_shift
;
15402 gcc_checking_assert (tkind
15403 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15404 gcc_checking_assert (tkind_zero
15405 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15406 if (tkind
== tkind_zero
)
15407 x
= build_int_cstu (tkind_type
, tkind
);
15410 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
15411 x
= build3 (COND_EXPR
, tkind_type
,
15412 fold_build2 (EQ_EXPR
, boolean_type_node
,
15413 unshare_expr (s
), size_zero_node
),
15414 build_int_cstu (tkind_type
, tkind_zero
),
15415 build_int_cstu (tkind_type
, tkind
));
15417 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
15422 case OMP_CLAUSE_FIRSTPRIVATE
:
15423 if (is_oacc_parallel (ctx
))
15424 goto oacc_firstprivate_map
;
15425 ovar
= OMP_CLAUSE_DECL (c
);
15426 if (is_reference (ovar
))
15427 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15429 talign
= DECL_ALIGN_UNIT (ovar
);
15430 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15431 x
= build_sender_ref (ovar
, ctx
);
15432 tkind
= GOMP_MAP_FIRSTPRIVATE
;
15433 type
= TREE_TYPE (ovar
);
15434 if (is_reference (ovar
))
15435 type
= TREE_TYPE (type
);
15436 bool use_firstprivate_int
, force_addr
;
15437 use_firstprivate_int
= false;
15438 force_addr
= false;
15439 if ((INTEGRAL_TYPE_P (type
)
15440 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15441 || TREE_CODE (type
) == POINTER_TYPE
)
15442 use_firstprivate_int
= true;
15445 if (is_reference (var
))
15446 use_firstprivate_int
= false;
15447 else if (is_gimple_reg (var
))
15449 if (DECL_HAS_VALUE_EXPR_P (var
))
15451 tree v
= get_base_address (var
);
15452 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15454 use_firstprivate_int
= false;
15458 switch (TREE_CODE (v
))
15462 use_firstprivate_int
= false;
15471 use_firstprivate_int
= false;
15473 if (use_firstprivate_int
)
15475 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15477 if (is_reference (var
))
15478 t
= build_simple_mem_ref (var
);
15479 if (TREE_CODE (type
) != POINTER_TYPE
)
15480 t
= fold_convert (pointer_sized_int_node
, t
);
15481 t
= fold_convert (TREE_TYPE (x
), t
);
15482 gimplify_assign (x
, t
, &ilist
);
15484 else if (is_reference (var
))
15485 gimplify_assign (x
, var
, &ilist
);
15486 else if (!force_addr
&& is_gimple_reg (var
))
15488 tree avar
= create_tmp_var (TREE_TYPE (var
));
15489 mark_addressable (avar
);
15490 gimplify_assign (avar
, var
, &ilist
);
15491 avar
= build_fold_addr_expr (avar
);
15492 gimplify_assign (x
, avar
, &ilist
);
15496 var
= build_fold_addr_expr (var
);
15497 gimplify_assign (x
, var
, &ilist
);
15499 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
15501 else if (is_reference (var
))
15502 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15504 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15505 s
= fold_convert (size_type_node
, s
);
15506 purpose
= size_int (map_idx
++);
15507 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15508 if (TREE_CODE (s
) != INTEGER_CST
)
15509 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15511 gcc_checking_assert (tkind
15512 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15513 talign
= ceil_log2 (talign
);
15514 tkind
|= talign
<< talign_shift
;
15515 gcc_checking_assert (tkind
15516 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15517 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15518 build_int_cstu (tkind_type
, tkind
));
15521 case OMP_CLAUSE_USE_DEVICE_PTR
:
15522 case OMP_CLAUSE_IS_DEVICE_PTR
:
15523 ovar
= OMP_CLAUSE_DECL (c
);
15524 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15525 x
= build_sender_ref (ovar
, ctx
);
15526 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15527 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
15529 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15530 type
= TREE_TYPE (ovar
);
15531 if (TREE_CODE (type
) == ARRAY_TYPE
)
15532 var
= build_fold_addr_expr (var
);
15535 if (is_reference (ovar
))
15537 type
= TREE_TYPE (type
);
15538 if (TREE_CODE (type
) != ARRAY_TYPE
)
15539 var
= build_simple_mem_ref (var
);
15540 var
= fold_convert (TREE_TYPE (x
), var
);
15543 gimplify_assign (x
, var
, &ilist
);
15545 purpose
= size_int (map_idx
++);
15546 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15547 gcc_checking_assert (tkind
15548 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15549 gcc_checking_assert (tkind
15550 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15551 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15552 build_int_cstu (tkind_type
, tkind
));
15556 gcc_assert (map_idx
== map_cnt
);
15558 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
15559 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
15560 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
15561 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
15562 for (int i
= 1; i
<= 2; i
++)
15563 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
15565 gimple_seq initlist
= NULL
;
15566 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
15567 TREE_VEC_ELT (t
, i
)),
15568 &initlist
, true, NULL_TREE
);
15569 gimple_seq_add_seq (&ilist
, initlist
);
15571 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
15573 TREE_THIS_VOLATILE (clobber
) = 1;
15574 gimple_seq_add_stmt (&olist
,
15575 gimple_build_assign (TREE_VEC_ELT (t
, i
),
15579 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
15580 TREE_THIS_VOLATILE (clobber
) = 1;
15581 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15585 /* Once all the expansions are done, sequence all the different
15586 fragments inside gimple_omp_body. */
15591 && ctx
->record_type
)
15593 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15594 /* fixup_child_record_type might have changed receiver_decl's type. */
15595 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15596 gimple_seq_add_stmt (&new_body
,
15597 gimple_build_assign (ctx
->receiver_decl
, t
));
15599 gimple_seq_add_seq (&new_body
, fplist
);
15601 if (offloaded
|| data_region
)
15603 tree prev
= NULL_TREE
;
15604 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15605 switch (OMP_CLAUSE_CODE (c
))
15610 case OMP_CLAUSE_FIRSTPRIVATE
:
15611 if (is_gimple_omp_oacc (ctx
->stmt
))
15613 var
= OMP_CLAUSE_DECL (c
);
15614 if (is_reference (var
)
15615 || is_gimple_reg_type (TREE_TYPE (var
)))
15617 tree new_var
= lookup_decl (var
, ctx
);
15619 type
= TREE_TYPE (var
);
15620 if (is_reference (var
))
15621 type
= TREE_TYPE (type
);
15622 bool use_firstprivate_int
;
15623 use_firstprivate_int
= false;
15624 if ((INTEGRAL_TYPE_P (type
)
15625 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15626 || TREE_CODE (type
) == POINTER_TYPE
)
15627 use_firstprivate_int
= true;
15630 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
15631 if (is_reference (v
))
15632 use_firstprivate_int
= false;
15633 else if (is_gimple_reg (v
))
15635 if (DECL_HAS_VALUE_EXPR_P (v
))
15637 v
= get_base_address (v
);
15638 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15639 use_firstprivate_int
= false;
15641 switch (TREE_CODE (v
))
15645 use_firstprivate_int
= false;
15653 use_firstprivate_int
= false;
15655 if (use_firstprivate_int
)
15657 x
= build_receiver_ref (var
, false, ctx
);
15658 if (TREE_CODE (type
) != POINTER_TYPE
)
15659 x
= fold_convert (pointer_sized_int_node
, x
);
15660 x
= fold_convert (type
, x
);
15661 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15663 if (is_reference (var
))
15665 tree v
= create_tmp_var_raw (type
, get_name (var
));
15666 gimple_add_tmp_var (v
);
15667 TREE_ADDRESSABLE (v
) = 1;
15668 gimple_seq_add_stmt (&new_body
,
15669 gimple_build_assign (v
, x
));
15670 x
= build_fold_addr_expr (v
);
15672 gimple_seq_add_stmt (&new_body
,
15673 gimple_build_assign (new_var
, x
));
15677 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
15678 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15680 gimple_seq_add_stmt (&new_body
,
15681 gimple_build_assign (new_var
, x
));
15684 else if (is_variable_sized (var
))
15686 tree pvar
= DECL_VALUE_EXPR (var
);
15687 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15688 pvar
= TREE_OPERAND (pvar
, 0);
15689 gcc_assert (DECL_P (pvar
));
15690 tree new_var
= lookup_decl (pvar
, ctx
);
15691 x
= build_receiver_ref (var
, false, ctx
);
15692 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15693 gimple_seq_add_stmt (&new_body
,
15694 gimple_build_assign (new_var
, x
));
15697 case OMP_CLAUSE_PRIVATE
:
15698 if (is_gimple_omp_oacc (ctx
->stmt
))
15700 var
= OMP_CLAUSE_DECL (c
);
15701 if (is_reference (var
))
15703 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15704 tree new_var
= lookup_decl (var
, ctx
);
15705 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
15706 if (TREE_CONSTANT (x
))
15708 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
15710 gimple_add_tmp_var (x
);
15711 TREE_ADDRESSABLE (x
) = 1;
15712 x
= build_fold_addr_expr_loc (clause_loc
, x
);
15717 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15718 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
15719 tree al
= size_int (TYPE_ALIGN (rtype
));
15720 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15723 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15724 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15725 gimple_seq_add_stmt (&new_body
,
15726 gimple_build_assign (new_var
, x
));
15729 case OMP_CLAUSE_USE_DEVICE_PTR
:
15730 case OMP_CLAUSE_IS_DEVICE_PTR
:
15731 var
= OMP_CLAUSE_DECL (c
);
15732 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15733 x
= build_sender_ref (var
, ctx
);
15735 x
= build_receiver_ref (var
, false, ctx
);
15736 if (is_variable_sized (var
))
15738 tree pvar
= DECL_VALUE_EXPR (var
);
15739 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15740 pvar
= TREE_OPERAND (pvar
, 0);
15741 gcc_assert (DECL_P (pvar
));
15742 tree new_var
= lookup_decl (pvar
, ctx
);
15743 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15744 gimple_seq_add_stmt (&new_body
,
15745 gimple_build_assign (new_var
, x
));
15747 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15749 tree new_var
= lookup_decl (var
, ctx
);
15750 new_var
= DECL_VALUE_EXPR (new_var
);
15751 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
15752 new_var
= TREE_OPERAND (new_var
, 0);
15753 gcc_assert (DECL_P (new_var
));
15754 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15755 gimple_seq_add_stmt (&new_body
,
15756 gimple_build_assign (new_var
, x
));
15760 tree type
= TREE_TYPE (var
);
15761 tree new_var
= lookup_decl (var
, ctx
);
15762 if (is_reference (var
))
15764 type
= TREE_TYPE (type
);
15765 if (TREE_CODE (type
) != ARRAY_TYPE
)
15767 tree v
= create_tmp_var_raw (type
, get_name (var
));
15768 gimple_add_tmp_var (v
);
15769 TREE_ADDRESSABLE (v
) = 1;
15770 x
= fold_convert (type
, x
);
15771 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15773 gimple_seq_add_stmt (&new_body
,
15774 gimple_build_assign (v
, x
));
15775 x
= build_fold_addr_expr (v
);
15778 x
= fold_convert (TREE_TYPE (new_var
), x
);
15779 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15780 gimple_seq_add_stmt (&new_body
,
15781 gimple_build_assign (new_var
, x
));
15785 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
15786 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
15787 are already handled. */
15788 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15789 switch (OMP_CLAUSE_CODE (c
))
15794 case OMP_CLAUSE_MAP
:
15795 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15796 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
15798 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15799 HOST_WIDE_INT offset
= 0;
15801 var
= OMP_CLAUSE_DECL (c
);
15803 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
15804 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
15806 && varpool_node::get_create (var
)->offloadable
)
15808 if (TREE_CODE (var
) == INDIRECT_REF
15809 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
15810 var
= TREE_OPERAND (var
, 0);
15811 if (TREE_CODE (var
) == COMPONENT_REF
)
15813 var
= get_addr_base_and_unit_offset (var
, &offset
);
15814 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
15816 else if (DECL_SIZE (var
)
15817 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15819 tree var2
= DECL_VALUE_EXPR (var
);
15820 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15821 var2
= TREE_OPERAND (var2
, 0);
15822 gcc_assert (DECL_P (var2
));
15825 tree new_var
= lookup_decl (var
, ctx
), x
;
15826 tree type
= TREE_TYPE (new_var
);
15828 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
15829 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
15832 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
15834 new_var
= build2 (MEM_REF
, type
,
15835 build_fold_addr_expr (new_var
),
15836 build_int_cst (build_pointer_type (type
),
15839 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
15841 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
15842 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
15843 new_var
= build2 (MEM_REF
, type
,
15844 build_fold_addr_expr (new_var
),
15845 build_int_cst (build_pointer_type (type
),
15849 is_ref
= is_reference (var
);
15850 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
15852 bool ref_to_array
= false;
15855 type
= TREE_TYPE (type
);
15856 if (TREE_CODE (type
) == ARRAY_TYPE
)
15858 type
= build_pointer_type (type
);
15859 ref_to_array
= true;
15862 else if (TREE_CODE (type
) == ARRAY_TYPE
)
15864 tree decl2
= DECL_VALUE_EXPR (new_var
);
15865 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
15866 decl2
= TREE_OPERAND (decl2
, 0);
15867 gcc_assert (DECL_P (decl2
));
15869 type
= TREE_TYPE (new_var
);
15871 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
15872 x
= fold_convert_loc (clause_loc
, type
, x
);
15873 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
15875 tree bias
= OMP_CLAUSE_SIZE (c
);
15877 bias
= lookup_decl (bias
, ctx
);
15878 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
15879 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
15881 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
15882 TREE_TYPE (x
), x
, bias
);
15885 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15886 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15887 if (is_ref
&& !ref_to_array
)
15889 tree t
= create_tmp_var_raw (type
, get_name (var
));
15890 gimple_add_tmp_var (t
);
15891 TREE_ADDRESSABLE (t
) = 1;
15892 gimple_seq_add_stmt (&new_body
,
15893 gimple_build_assign (t
, x
));
15894 x
= build_fold_addr_expr_loc (clause_loc
, t
);
15896 gimple_seq_add_stmt (&new_body
,
15897 gimple_build_assign (new_var
, x
));
15900 else if (OMP_CLAUSE_CHAIN (c
)
15901 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
15903 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15904 == GOMP_MAP_FIRSTPRIVATE_POINTER
15905 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15906 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15909 case OMP_CLAUSE_PRIVATE
:
15910 var
= OMP_CLAUSE_DECL (c
);
15911 if (is_variable_sized (var
))
15913 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15914 tree new_var
= lookup_decl (var
, ctx
);
15915 tree pvar
= DECL_VALUE_EXPR (var
);
15916 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15917 pvar
= TREE_OPERAND (pvar
, 0);
15918 gcc_assert (DECL_P (pvar
));
15919 tree new_pvar
= lookup_decl (pvar
, ctx
);
15920 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15921 tree al
= size_int (DECL_ALIGN (var
));
15922 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
15923 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15924 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
15925 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15926 gimple_seq_add_stmt (&new_body
,
15927 gimple_build_assign (new_pvar
, x
));
15932 gimple_seq fork_seq
= NULL
;
15933 gimple_seq join_seq
= NULL
;
15935 if (is_oacc_parallel (ctx
))
15937 /* If there are reductions on the offloaded region itself, treat
15938 them as a dummy GANG loop. */
15939 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
15941 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
15942 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
15945 gimple_seq_add_seq (&new_body
, fork_seq
);
15946 gimple_seq_add_seq (&new_body
, tgt_body
);
15947 gimple_seq_add_seq (&new_body
, join_seq
);
15950 new_body
= maybe_catch_exception (new_body
);
15952 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15953 gimple_omp_set_body (stmt
, new_body
);
15956 bind
= gimple_build_bind (NULL
, NULL
,
15957 tgt_bind
? gimple_bind_block (tgt_bind
)
15959 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15960 gimple_bind_add_seq (bind
, ilist
);
15961 gimple_bind_add_stmt (bind
, stmt
);
15962 gimple_bind_add_seq (bind
, olist
);
15964 pop_gimplify_context (NULL
);
15968 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15969 gimple_bind_add_stmt (dep_bind
, bind
);
15970 gimple_bind_add_seq (dep_bind
, dep_olist
);
15971 pop_gimplify_context (dep_bind
);
15975 /* Expand code for an OpenMP teams directive. */
15978 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15980 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
15981 push_gimplify_context ();
15983 tree block
= make_node (BLOCK
);
15984 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
15985 gsi_replace (gsi_p
, bind
, true);
15986 gimple_seq bind_body
= NULL
;
15987 gimple_seq dlist
= NULL
;
15988 gimple_seq olist
= NULL
;
15990 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15991 OMP_CLAUSE_NUM_TEAMS
);
15992 if (num_teams
== NULL_TREE
)
15993 num_teams
= build_int_cst (unsigned_type_node
, 0);
15996 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
15997 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
15998 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16000 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16001 OMP_CLAUSE_THREAD_LIMIT
);
16002 if (thread_limit
== NULL_TREE
)
16003 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16006 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16007 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16008 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16012 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16013 &bind_body
, &dlist
, ctx
, NULL
);
16014 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16015 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16016 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16018 location_t loc
= gimple_location (teams_stmt
);
16019 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16020 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16021 gimple_set_location (call
, loc
);
16022 gimple_seq_add_stmt (&bind_body
, call
);
16024 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16025 gimple_omp_set_body (teams_stmt
, NULL
);
16026 gimple_seq_add_seq (&bind_body
, olist
);
16027 gimple_seq_add_seq (&bind_body
, dlist
);
16028 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16029 gimple_bind_set_body (bind
, bind_body
);
16031 pop_gimplify_context (bind
);
16033 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16034 BLOCK_VARS (block
) = ctx
->block_vars
;
16035 if (BLOCK_VARS (block
))
16036 TREE_USED (block
) = 1;
16040 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16041 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16042 of OMP context, but with task_shared_vars set. */
16045 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16050 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16051 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16054 if (task_shared_vars
16056 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16059 /* If a global variable has been privatized, TREE_CONSTANT on
16060 ADDR_EXPR might be wrong. */
16061 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16062 recompute_tree_invariant_for_addr_expr (t
);
16064 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16068 /* Data to be communicated between lower_omp_regimplify_operands and
16069 lower_omp_regimplify_operands_p. */
16071 struct lower_omp_regimplify_operands_data
16077 /* Helper function for lower_omp_regimplify_operands. Find
16078 omp_member_access_dummy_var vars and adjust temporarily their
16079 DECL_VALUE_EXPRs if needed. */
16082 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16085 tree t
= omp_member_access_dummy_var (*tp
);
16088 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16089 lower_omp_regimplify_operands_data
*ldata
16090 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16091 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16094 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16095 ldata
->decls
->safe_push (*tp
);
16096 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16097 SET_DECL_VALUE_EXPR (*tp
, v
);
16100 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16104 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16105 of omp_member_access_dummy_var vars during regimplification. */
16108 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16109 gimple_stmt_iterator
*gsi_p
)
16111 auto_vec
<tree
, 10> decls
;
16114 struct walk_stmt_info wi
;
16115 memset (&wi
, '\0', sizeof (wi
));
16116 struct lower_omp_regimplify_operands_data data
;
16118 data
.decls
= &decls
;
16120 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16122 gimple_regimplify_operands (stmt
, gsi_p
);
16123 while (!decls
.is_empty ())
16125 tree t
= decls
.pop ();
16126 tree v
= decls
.pop ();
16127 SET_DECL_VALUE_EXPR (t
, v
);
16132 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16134 gimple
*stmt
= gsi_stmt (*gsi_p
);
16135 struct walk_stmt_info wi
;
16138 if (gimple_has_location (stmt
))
16139 input_location
= gimple_location (stmt
);
16141 if (task_shared_vars
)
16142 memset (&wi
, '\0', sizeof (wi
));
16144 /* If we have issued syntax errors, avoid doing any heavy lifting.
16145 Just replace the OMP directives with a NOP to avoid
16146 confusing RTL expansion. */
16147 if (seen_error () && is_gimple_omp (stmt
))
16149 gsi_replace (gsi_p
, gimple_build_nop (), true);
16153 switch (gimple_code (stmt
))
16157 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16158 if ((ctx
|| task_shared_vars
)
16159 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16160 lower_omp_regimplify_p
,
16161 ctx
? NULL
: &wi
, NULL
)
16162 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16163 lower_omp_regimplify_p
,
16164 ctx
? NULL
: &wi
, NULL
)))
16165 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16169 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16171 case GIMPLE_EH_FILTER
:
16172 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16175 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16176 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16178 case GIMPLE_TRANSACTION
:
16179 lower_omp (gimple_transaction_body_ptr (
16180 as_a
<gtransaction
*> (stmt
)),
16184 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16186 case GIMPLE_OMP_PARALLEL
:
16187 case GIMPLE_OMP_TASK
:
16188 ctx
= maybe_lookup_ctx (stmt
);
16190 if (ctx
->cancellable
)
16191 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16192 lower_omp_taskreg (gsi_p
, ctx
);
16194 case GIMPLE_OMP_FOR
:
16195 ctx
= maybe_lookup_ctx (stmt
);
16197 if (ctx
->cancellable
)
16198 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16199 lower_omp_for (gsi_p
, ctx
);
16201 case GIMPLE_OMP_SECTIONS
:
16202 ctx
= maybe_lookup_ctx (stmt
);
16204 if (ctx
->cancellable
)
16205 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16206 lower_omp_sections (gsi_p
, ctx
);
16208 case GIMPLE_OMP_SINGLE
:
16209 ctx
= maybe_lookup_ctx (stmt
);
16211 lower_omp_single (gsi_p
, ctx
);
16213 case GIMPLE_OMP_MASTER
:
16214 ctx
= maybe_lookup_ctx (stmt
);
16216 lower_omp_master (gsi_p
, ctx
);
16218 case GIMPLE_OMP_TASKGROUP
:
16219 ctx
= maybe_lookup_ctx (stmt
);
16221 lower_omp_taskgroup (gsi_p
, ctx
);
16223 case GIMPLE_OMP_ORDERED
:
16224 ctx
= maybe_lookup_ctx (stmt
);
16226 lower_omp_ordered (gsi_p
, ctx
);
16228 case GIMPLE_OMP_CRITICAL
:
16229 ctx
= maybe_lookup_ctx (stmt
);
16231 lower_omp_critical (gsi_p
, ctx
);
16233 case GIMPLE_OMP_ATOMIC_LOAD
:
16234 if ((ctx
|| task_shared_vars
)
16235 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
16236 as_a
<gomp_atomic_load
*> (stmt
)),
16237 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
16238 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16240 case GIMPLE_OMP_TARGET
:
16241 ctx
= maybe_lookup_ctx (stmt
);
16243 lower_omp_target (gsi_p
, ctx
);
16245 case GIMPLE_OMP_TEAMS
:
16246 ctx
= maybe_lookup_ctx (stmt
);
16248 lower_omp_teams (gsi_p
, ctx
);
16252 call_stmt
= as_a
<gcall
*> (stmt
);
16253 fndecl
= gimple_call_fndecl (call_stmt
);
16255 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
16256 switch (DECL_FUNCTION_CODE (fndecl
))
16258 case BUILT_IN_GOMP_BARRIER
:
16262 case BUILT_IN_GOMP_CANCEL
:
16263 case BUILT_IN_GOMP_CANCELLATION_POINT
:
16266 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
16267 cctx
= cctx
->outer
;
16268 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
16269 if (!cctx
->cancellable
)
16271 if (DECL_FUNCTION_CODE (fndecl
)
16272 == BUILT_IN_GOMP_CANCELLATION_POINT
)
16274 stmt
= gimple_build_nop ();
16275 gsi_replace (gsi_p
, stmt
, false);
16279 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
16281 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
16282 gimple_call_set_fndecl (call_stmt
, fndecl
);
16283 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
16286 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
16287 gimple_call_set_lhs (call_stmt
, lhs
);
16288 tree fallthru_label
;
16289 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
16291 g
= gimple_build_label (fallthru_label
);
16292 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16293 g
= gimple_build_cond (NE_EXPR
, lhs
,
16294 fold_convert (TREE_TYPE (lhs
),
16295 boolean_false_node
),
16296 cctx
->cancel_label
, fallthru_label
);
16297 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16304 if ((ctx
|| task_shared_vars
)
16305 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
16308 /* Just remove clobbers, this should happen only if we have
16309 "privatized" local addressable variables in SIMD regions,
16310 the clobber isn't needed in that case and gimplifying address
16311 of the ARRAY_REF into a pointer and creating MEM_REF based
16312 clobber would create worse code than we get with the clobber
16314 if (gimple_clobber_p (stmt
))
16316 gsi_replace (gsi_p
, gimple_build_nop (), true);
16319 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16326 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
16328 location_t saved_location
= input_location
;
16329 gimple_stmt_iterator gsi
;
16330 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16331 lower_omp_1 (&gsi
, ctx
);
16332 /* During gimplification, we haven't folded statments inside offloading
16333 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
16334 if (target_nesting_level
|| taskreg_nesting_level
)
16335 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16337 input_location
= saved_location
;
16340 /* Main entry point. */
16342 static unsigned int
16343 execute_lower_omp (void)
16349 /* This pass always runs, to provide PROP_gimple_lomp.
16350 But often, there is nothing to do. */
16351 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
16352 && flag_openmp_simd
== 0)
16355 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
16356 delete_omp_context
);
16358 body
= gimple_body (current_function_decl
);
16359 scan_omp (&body
, NULL
);
16360 gcc_assert (taskreg_nesting_level
== 0);
16361 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
16362 finish_taskreg_scan (ctx
);
16363 taskreg_contexts
.release ();
16365 if (all_contexts
->root
)
16367 if (task_shared_vars
)
16368 push_gimplify_context ();
16369 lower_omp (&body
, NULL
);
16370 if (task_shared_vars
)
16371 pop_gimplify_context (NULL
);
16376 splay_tree_delete (all_contexts
);
16377 all_contexts
= NULL
;
16379 BITMAP_FREE (task_shared_vars
);
16385 const pass_data pass_data_lower_omp
=
16387 GIMPLE_PASS
, /* type */
16388 "omplower", /* name */
16389 OPTGROUP_NONE
, /* optinfo_flags */
16390 TV_NONE
, /* tv_id */
16391 PROP_gimple_any
, /* properties_required */
16392 PROP_gimple_lomp
, /* properties_provided */
16393 0, /* properties_destroyed */
16394 0, /* todo_flags_start */
16395 0, /* todo_flags_finish */
16398 class pass_lower_omp
: public gimple_opt_pass
16401 pass_lower_omp (gcc::context
*ctxt
)
16402 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
16405 /* opt_pass methods: */
16406 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
16408 }; // class pass_lower_omp
16410 } // anon namespace
16413 make_pass_lower_omp (gcc::context
*ctxt
)
16415 return new pass_lower_omp (ctxt
);
16418 /* The following is a utility to diagnose structured block violations.
16419 It is not part of the "omplower" pass, as that's invoked too late. It
16420 should be invoked by the respective front ends after gimplification. */
16422 static splay_tree all_labels
;
16424 /* Check for mismatched contexts and generate an error if needed. Return
16425 true if an error is detected. */
16428 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
16429 gimple
*branch_ctx
, gimple
*label_ctx
)
16431 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
16432 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
16434 if (label_ctx
== branch_ctx
)
16437 const char* kind
= NULL
;
16442 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
16443 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
16445 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
16446 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
16447 kind
= "Cilk Plus";
16451 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
16452 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
16454 gcc_checking_assert (kind
== NULL
);
16460 gcc_checking_assert (flag_openmp
);
16465 Previously we kept track of the label's entire context in diagnose_sb_[12]
16466 so we could traverse it and issue a correct "exit" or "enter" error
16467 message upon a structured block violation.
16469 We built the context by building a list with tree_cons'ing, but there is
16470 no easy counterpart in gimple tuples. It seems like far too much work
16471 for issuing exit/enter error messages. If someone really misses the
16472 distinct error message... patches welcome.
16476 /* Try to avoid confusing the user by producing and error message
16477 with correct "exit" or "enter" verbiage. We prefer "exit"
16478 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
16479 if (branch_ctx
== NULL
)
16485 if (TREE_VALUE (label_ctx
) == branch_ctx
)
16490 label_ctx
= TREE_CHAIN (label_ctx
);
16495 error ("invalid exit from %s structured block", kind
);
16497 error ("invalid entry to %s structured block", kind
);
16500 /* If it's obvious we have an invalid entry, be specific about the error. */
16501 if (branch_ctx
== NULL
)
16502 error ("invalid entry to %s structured block", kind
);
16505 /* Otherwise, be vague and lazy, but efficient. */
16506 error ("invalid branch to/from %s structured block", kind
);
16509 gsi_replace (gsi_p
, gimple_build_nop (), false);
16513 /* Pass 1: Create a minimal tree of structured blocks, and record
16514 where each label is found. */
16517 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16518 struct walk_stmt_info
*wi
)
16520 gimple
*context
= (gimple
*) wi
->info
;
16521 gimple
*inner_context
;
16522 gimple
*stmt
= gsi_stmt (*gsi_p
);
16524 *handled_ops_p
= true;
16526 switch (gimple_code (stmt
))
16530 case GIMPLE_OMP_PARALLEL
:
16531 case GIMPLE_OMP_TASK
:
16532 case GIMPLE_OMP_SECTIONS
:
16533 case GIMPLE_OMP_SINGLE
:
16534 case GIMPLE_OMP_SECTION
:
16535 case GIMPLE_OMP_MASTER
:
16536 case GIMPLE_OMP_ORDERED
:
16537 case GIMPLE_OMP_CRITICAL
:
16538 case GIMPLE_OMP_TARGET
:
16539 case GIMPLE_OMP_TEAMS
:
16540 case GIMPLE_OMP_TASKGROUP
:
16541 /* The minimal context here is just the current OMP construct. */
16542 inner_context
= stmt
;
16543 wi
->info
= inner_context
;
16544 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16545 wi
->info
= context
;
16548 case GIMPLE_OMP_FOR
:
16549 inner_context
= stmt
;
16550 wi
->info
= inner_context
;
16551 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16553 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
16554 diagnose_sb_1
, NULL
, wi
);
16555 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16556 wi
->info
= context
;
16560 splay_tree_insert (all_labels
,
16561 (splay_tree_key
) gimple_label_label (
16562 as_a
<glabel
*> (stmt
)),
16563 (splay_tree_value
) context
);
16573 /* Pass 2: Check each branch and see if its context differs from that of
16574 the destination label's context. */
16577 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16578 struct walk_stmt_info
*wi
)
16580 gimple
*context
= (gimple
*) wi
->info
;
16582 gimple
*stmt
= gsi_stmt (*gsi_p
);
16584 *handled_ops_p
= true;
16586 switch (gimple_code (stmt
))
16590 case GIMPLE_OMP_PARALLEL
:
16591 case GIMPLE_OMP_TASK
:
16592 case GIMPLE_OMP_SECTIONS
:
16593 case GIMPLE_OMP_SINGLE
:
16594 case GIMPLE_OMP_SECTION
:
16595 case GIMPLE_OMP_MASTER
:
16596 case GIMPLE_OMP_ORDERED
:
16597 case GIMPLE_OMP_CRITICAL
:
16598 case GIMPLE_OMP_TARGET
:
16599 case GIMPLE_OMP_TEAMS
:
16600 case GIMPLE_OMP_TASKGROUP
:
16602 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16603 wi
->info
= context
;
16606 case GIMPLE_OMP_FOR
:
16608 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16610 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
16611 diagnose_sb_2
, NULL
, wi
);
16612 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16613 wi
->info
= context
;
16618 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16619 tree lab
= gimple_cond_true_label (cond_stmt
);
16622 n
= splay_tree_lookup (all_labels
,
16623 (splay_tree_key
) lab
);
16624 diagnose_sb_0 (gsi_p
, context
,
16625 n
? (gimple
*) n
->value
: NULL
);
16627 lab
= gimple_cond_false_label (cond_stmt
);
16630 n
= splay_tree_lookup (all_labels
,
16631 (splay_tree_key
) lab
);
16632 diagnose_sb_0 (gsi_p
, context
,
16633 n
? (gimple
*) n
->value
: NULL
);
16640 tree lab
= gimple_goto_dest (stmt
);
16641 if (TREE_CODE (lab
) != LABEL_DECL
)
16644 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16645 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
16649 case GIMPLE_SWITCH
:
16651 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
16653 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
16655 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
16656 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16657 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
16663 case GIMPLE_RETURN
:
16664 diagnose_sb_0 (gsi_p
, context
, NULL
);
16674 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
16677 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
16680 gimple
*last
= last_stmt (bb
);
16681 enum gimple_code code
= gimple_code (last
);
16682 struct omp_region
*cur_region
= *region
;
16683 bool fallthru
= false;
16687 case GIMPLE_OMP_PARALLEL
:
16688 case GIMPLE_OMP_TASK
:
16689 case GIMPLE_OMP_FOR
:
16690 case GIMPLE_OMP_SINGLE
:
16691 case GIMPLE_OMP_TEAMS
:
16692 case GIMPLE_OMP_MASTER
:
16693 case GIMPLE_OMP_TASKGROUP
:
16694 case GIMPLE_OMP_CRITICAL
:
16695 case GIMPLE_OMP_SECTION
:
16696 cur_region
= new_omp_region (bb
, code
, cur_region
);
16700 case GIMPLE_OMP_ORDERED
:
16701 cur_region
= new_omp_region (bb
, code
, cur_region
);
16703 if (find_omp_clause (gimple_omp_ordered_clauses
16704 (as_a
<gomp_ordered
*> (last
)),
16705 OMP_CLAUSE_DEPEND
))
16706 cur_region
= cur_region
->outer
;
16709 case GIMPLE_OMP_TARGET
:
16710 cur_region
= new_omp_region (bb
, code
, cur_region
);
16712 switch (gimple_omp_target_kind (last
))
16714 case GF_OMP_TARGET_KIND_REGION
:
16715 case GF_OMP_TARGET_KIND_DATA
:
16716 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
16717 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
16718 case GF_OMP_TARGET_KIND_OACC_DATA
:
16720 case GF_OMP_TARGET_KIND_UPDATE
:
16721 case GF_OMP_TARGET_KIND_ENTER_DATA
:
16722 case GF_OMP_TARGET_KIND_EXIT_DATA
:
16723 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
16724 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
16725 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
16726 cur_region
= cur_region
->outer
;
16729 gcc_unreachable ();
16733 case GIMPLE_OMP_SECTIONS
:
16734 cur_region
= new_omp_region (bb
, code
, cur_region
);
16738 case GIMPLE_OMP_SECTIONS_SWITCH
:
16742 case GIMPLE_OMP_ATOMIC_LOAD
:
16743 case GIMPLE_OMP_ATOMIC_STORE
:
16747 case GIMPLE_OMP_RETURN
:
16748 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
16749 somewhere other than the next block. This will be
16751 cur_region
->exit
= bb
;
16752 if (cur_region
->type
== GIMPLE_OMP_TASK
)
16753 /* Add an edge corresponding to not scheduling the task
16755 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
16756 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
16757 cur_region
= cur_region
->outer
;
16760 case GIMPLE_OMP_CONTINUE
:
16761 cur_region
->cont
= bb
;
16762 switch (cur_region
->type
)
16764 case GIMPLE_OMP_FOR
:
16765 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
16766 succs edges as abnormal to prevent splitting
16768 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
16769 /* Make the loopback edge. */
16770 make_edge (bb
, single_succ (cur_region
->entry
),
16773 /* Create an edge from GIMPLE_OMP_FOR to exit, which
16774 corresponds to the case that the body of the loop
16775 is not executed at all. */
16776 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
16777 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
16781 case GIMPLE_OMP_SECTIONS
:
16782 /* Wire up the edges into and out of the nested sections. */
16784 basic_block switch_bb
= single_succ (cur_region
->entry
);
16786 struct omp_region
*i
;
16787 for (i
= cur_region
->inner
; i
; i
= i
->next
)
16789 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
16790 make_edge (switch_bb
, i
->entry
, 0);
16791 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
16794 /* Make the loopback edge to the block with
16795 GIMPLE_OMP_SECTIONS_SWITCH. */
16796 make_edge (bb
, switch_bb
, 0);
16798 /* Make the edge from the switch to exit. */
16799 make_edge (switch_bb
, bb
->next_bb
, 0);
16804 case GIMPLE_OMP_TASK
:
16809 gcc_unreachable ();
16814 gcc_unreachable ();
16817 if (*region
!= cur_region
)
16819 *region
= cur_region
;
16821 *region_idx
= cur_region
->entry
->index
;
16829 static unsigned int
16830 diagnose_omp_structured_block_errors (void)
16832 struct walk_stmt_info wi
;
16833 gimple_seq body
= gimple_body (current_function_decl
);
16835 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
16837 memset (&wi
, 0, sizeof (wi
));
16838 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
16840 memset (&wi
, 0, sizeof (wi
));
16841 wi
.want_locations
= true;
16842 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
16844 gimple_set_body (current_function_decl
, body
);
16846 splay_tree_delete (all_labels
);
16854 const pass_data pass_data_diagnose_omp_blocks
=
16856 GIMPLE_PASS
, /* type */
16857 "*diagnose_omp_blocks", /* name */
16858 OPTGROUP_NONE
, /* optinfo_flags */
16859 TV_NONE
, /* tv_id */
16860 PROP_gimple_any
, /* properties_required */
16861 0, /* properties_provided */
16862 0, /* properties_destroyed */
16863 0, /* todo_flags_start */
16864 0, /* todo_flags_finish */
16867 class pass_diagnose_omp_blocks
: public gimple_opt_pass
16870 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16871 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
16874 /* opt_pass methods: */
16875 virtual bool gate (function
*)
16877 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
16879 virtual unsigned int execute (function
*)
16881 return diagnose_omp_structured_block_errors ();
16884 }; // class pass_diagnose_omp_blocks
16886 } // anon namespace
16889 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16891 return new pass_diagnose_omp_blocks (ctxt
);
16894 /* SIMD clone supporting code. */
16896 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
16897 of arguments to reserve space for. */
16899 static struct cgraph_simd_clone
*
16900 simd_clone_struct_alloc (int nargs
)
16902 struct cgraph_simd_clone
*clone_info
;
16903 size_t len
= (sizeof (struct cgraph_simd_clone
)
16904 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
16905 clone_info
= (struct cgraph_simd_clone
*)
16906 ggc_internal_cleared_alloc (len
);
16910 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
16913 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
16914 struct cgraph_simd_clone
*from
)
16916 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
16917 + ((from
->nargs
- from
->inbranch
)
16918 * sizeof (struct cgraph_simd_clone_arg
))));
16921 /* Return vector of parameter types of function FNDECL. This uses
16922 TYPE_ARG_TYPES if available, otherwise falls back to types of
16923 DECL_ARGUMENTS types. */
16926 simd_clone_vector_of_formal_parm_types (tree fndecl
)
16928 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
16929 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
16930 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
16933 FOR_EACH_VEC_ELT (args
, i
, arg
)
16934 args
[i
] = TREE_TYPE (args
[i
]);
16938 /* Given a simd function in NODE, extract the simd specific
16939 information from the OMP clauses passed in CLAUSES, and return
16940 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
16941 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
16942 otherwise set to FALSE. */
16944 static struct cgraph_simd_clone
*
16945 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
16946 bool *inbranch_specified
)
16948 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
16951 *inbranch_specified
= false;
16953 n
= args
.length ();
16954 if (n
> 0 && args
.last () == void_type_node
)
16957 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
16958 be cloned have a distinctive artificial label in addition to "omp
16962 && lookup_attribute ("cilk simd function",
16963 DECL_ATTRIBUTES (node
->decl
)));
16965 /* Allocate one more than needed just in case this is an in-branch
16966 clone which will require a mask argument. */
16967 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
16968 clone_info
->nargs
= n
;
16969 clone_info
->cilk_elemental
= cilk_clone
;
16976 clauses
= TREE_VALUE (clauses
);
16977 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
16980 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
16982 switch (OMP_CLAUSE_CODE (t
))
16984 case OMP_CLAUSE_INBRANCH
:
16985 clone_info
->inbranch
= 1;
16986 *inbranch_specified
= true;
16988 case OMP_CLAUSE_NOTINBRANCH
:
16989 clone_info
->inbranch
= 0;
16990 *inbranch_specified
= true;
16992 case OMP_CLAUSE_SIMDLEN
:
16993 clone_info
->simdlen
16994 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
16996 case OMP_CLAUSE_LINEAR
:
16998 tree decl
= OMP_CLAUSE_DECL (t
);
16999 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
17000 int argno
= TREE_INT_CST_LOW (decl
);
17001 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
17003 enum cgraph_simd_clone_arg_type arg_type
;
17004 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
17005 switch (OMP_CLAUSE_LINEAR_KIND (t
))
17007 case OMP_CLAUSE_LINEAR_REF
:
17009 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
17011 case OMP_CLAUSE_LINEAR_UVAL
:
17013 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
17015 case OMP_CLAUSE_LINEAR_VAL
:
17016 case OMP_CLAUSE_LINEAR_DEFAULT
:
17018 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
17021 gcc_unreachable ();
17024 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
17025 clone_info
->args
[argno
].arg_type
= arg_type
;
17026 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
17027 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
17028 && clone_info
->args
[argno
].linear_step
< n
);
17032 if (POINTER_TYPE_P (args
[argno
]))
17033 step
= fold_convert (ssizetype
, step
);
17034 if (!tree_fits_shwi_p (step
))
17036 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
17037 "ignoring large linear step");
17041 else if (integer_zerop (step
))
17043 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
17044 "ignoring zero linear step");
17050 enum cgraph_simd_clone_arg_type arg_type
;
17051 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
17052 switch (OMP_CLAUSE_LINEAR_KIND (t
))
17054 case OMP_CLAUSE_LINEAR_REF
:
17056 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
17058 case OMP_CLAUSE_LINEAR_UVAL
:
17060 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
17062 case OMP_CLAUSE_LINEAR_VAL
:
17063 case OMP_CLAUSE_LINEAR_DEFAULT
:
17065 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
17068 gcc_unreachable ();
17071 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
17072 clone_info
->args
[argno
].arg_type
= arg_type
;
17073 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
17078 case OMP_CLAUSE_UNIFORM
:
17080 tree decl
= OMP_CLAUSE_DECL (t
);
17081 int argno
= tree_to_uhwi (decl
);
17082 clone_info
->args
[argno
].arg_type
17083 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
17086 case OMP_CLAUSE_ALIGNED
:
17088 tree decl
= OMP_CLAUSE_DECL (t
);
17089 int argno
= tree_to_uhwi (decl
);
17090 clone_info
->args
[argno
].alignment
17091 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
17102 /* Given a SIMD clone in NODE, calculate the characteristic data
17103 type and return the coresponding type. The characteristic data
17104 type is computed as described in the Intel Vector ABI. */
17107 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
17108 struct cgraph_simd_clone
*clone_info
)
17110 tree type
= integer_type_node
;
17111 tree fndecl
= node
->decl
;
17113 /* a) For non-void function, the characteristic data type is the
17115 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
17116 type
= TREE_TYPE (TREE_TYPE (fndecl
));
17118 /* b) If the function has any non-uniform, non-linear parameters,
17119 then the characteristic data type is the type of the first
17123 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
17124 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
17125 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
17133 /* c) If the characteristic data type determined by a) or b) above
17134 is struct, union, or class type which is pass-by-value (except
17135 for the type that maps to the built-in complex data type), the
17136 characteristic data type is int. */
17137 if (RECORD_OR_UNION_TYPE_P (type
)
17138 && !aggregate_value_p (type
, NULL
)
17139 && TREE_CODE (type
) != COMPLEX_TYPE
)
17140 return integer_type_node
;
17142 /* d) If none of the above three classes is applicable, the
17143 characteristic data type is int. */
17147 /* e) For Intel Xeon Phi native and offload compilation, if the
17148 resulting characteristic data type is 8-bit or 16-bit integer
17149 data type, the characteristic data type is int. */
17150 /* Well, we don't handle Xeon Phi yet. */
17154 simd_clone_mangle (struct cgraph_node
*node
,
17155 struct cgraph_simd_clone
*clone_info
)
17157 char vecsize_mangle
= clone_info
->vecsize_mangle
;
17158 char mask
= clone_info
->inbranch
? 'M' : 'N';
17159 unsigned int simdlen
= clone_info
->simdlen
;
17163 gcc_assert (vecsize_mangle
&& simdlen
);
17165 pp_string (&pp
, "_ZGV");
17166 pp_character (&pp
, vecsize_mangle
);
17167 pp_character (&pp
, mask
);
17168 pp_decimal_int (&pp
, simdlen
);
17170 for (n
= 0; n
< clone_info
->nargs
; ++n
)
17172 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
17174 switch (arg
.arg_type
)
17176 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
17177 pp_character (&pp
, 'u');
17179 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
17180 pp_character (&pp
, 'l');
17181 goto mangle_linear
;
17182 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17183 pp_character (&pp
, 'R');
17184 goto mangle_linear
;
17185 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17186 pp_character (&pp
, 'L');
17187 goto mangle_linear
;
17188 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17189 pp_character (&pp
, 'U');
17190 goto mangle_linear
;
17192 gcc_assert (arg
.linear_step
!= 0);
17193 if (arg
.linear_step
> 1)
17194 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17195 else if (arg
.linear_step
< 0)
17197 pp_character (&pp
, 'n');
17198 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
17202 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17203 pp_string (&pp
, "ls");
17204 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17206 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
17207 pp_string (&pp
, "Rs");
17208 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17210 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17211 pp_string (&pp
, "Ls");
17212 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17214 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17215 pp_string (&pp
, "Us");
17216 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17219 pp_character (&pp
, 'v');
17223 pp_character (&pp
, 'a');
17224 pp_decimal_int (&pp
, arg
.alignment
);
17228 pp_underscore (&pp
);
17229 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
17232 pp_string (&pp
, str
);
17233 str
= pp_formatted_text (&pp
);
17235 /* If there already is a SIMD clone with the same mangled name, don't
17236 add another one. This can happen e.g. for
17237 #pragma omp declare simd
17238 #pragma omp declare simd simdlen(8)
17239 int foo (int, int);
17240 if the simdlen is assumed to be 8 for the first one, etc. */
17241 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
17242 clone
= clone
->simdclone
->next_clone
)
17243 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
17247 return get_identifier (str
);
17250 /* Create a simd clone of OLD_NODE and return it. */
17252 static struct cgraph_node
*
17253 simd_clone_create (struct cgraph_node
*old_node
)
17255 struct cgraph_node
*new_node
;
17256 if (old_node
->definition
)
17258 if (!old_node
->has_gimple_body_p ())
17260 old_node
->get_body ();
17261 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
17267 tree old_decl
= old_node
->decl
;
17268 tree new_decl
= copy_node (old_node
->decl
);
17269 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
17270 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
17271 SET_DECL_RTL (new_decl
, NULL
);
17272 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
17273 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
17274 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
17275 if (old_node
->in_other_partition
)
17276 new_node
->in_other_partition
= 1;
17277 symtab
->call_cgraph_insertion_hooks (new_node
);
17279 if (new_node
== NULL
)
17282 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
17284 /* The function cgraph_function_versioning () will force the new
17285 symbol local. Undo this, and inherit external visability from
17287 new_node
->local
.local
= old_node
->local
.local
;
17288 new_node
->externally_visible
= old_node
->externally_visible
;
17293 /* Adjust the return type of the given function to its appropriate
17294 vector counterpart. Returns a simd array to be used throughout the
17295 function as a return value. */
17298 simd_clone_adjust_return_type (struct cgraph_node
*node
)
17300 tree fndecl
= node
->decl
;
17301 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
17302 unsigned int veclen
;
17305 /* Adjust the function return type. */
17306 if (orig_rettype
== void_type_node
)
17308 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
17309 t
= TREE_TYPE (TREE_TYPE (fndecl
));
17310 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
17311 veclen
= node
->simdclone
->vecsize_int
;
17313 veclen
= node
->simdclone
->vecsize_float
;
17314 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
17315 if (veclen
> node
->simdclone
->simdlen
)
17316 veclen
= node
->simdclone
->simdlen
;
17317 if (POINTER_TYPE_P (t
))
17318 t
= pointer_sized_int_node
;
17319 if (veclen
== node
->simdclone
->simdlen
)
17320 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
17323 t
= build_vector_type (t
, veclen
);
17324 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
17326 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
17327 if (!node
->definition
)
17330 t
= DECL_RESULT (fndecl
);
17331 /* Adjust the DECL_RESULT. */
17332 gcc_assert (TREE_TYPE (t
) != void_type_node
);
17333 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
17336 tree atype
= build_array_type_nelts (orig_rettype
,
17337 node
->simdclone
->simdlen
);
17338 if (veclen
!= node
->simdclone
->simdlen
)
17339 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
17341 /* Set up a SIMD array to use as the return value. */
17342 tree retval
= create_tmp_var_raw (atype
, "retval");
17343 gimple_add_tmp_var (retval
);
17347 /* Each vector argument has a corresponding array to be used locally
17348 as part of the eventual loop. Create such temporary array and
17351 PREFIX is the prefix to be used for the temporary.
17353 TYPE is the inner element type.
17355 SIMDLEN is the number of elements. */
17358 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
17360 tree atype
= build_array_type_nelts (type
, simdlen
);
17361 tree avar
= create_tmp_var_raw (atype
, prefix
);
17362 gimple_add_tmp_var (avar
);
17366 /* Modify the function argument types to their corresponding vector
17367 counterparts if appropriate. Also, create one array for each simd
17368 argument to be used locally when using the function arguments as
17371 NODE is the function whose arguments are to be adjusted.
17373 Returns an adjustment vector that will be filled describing how the
17374 argument types will be adjusted. */
17376 static ipa_parm_adjustment_vec
17377 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
17380 ipa_parm_adjustment_vec adjustments
;
17382 if (node
->definition
)
17383 args
= ipa_get_vector_of_formal_parms (node
->decl
);
17385 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
17386 adjustments
.create (args
.length ());
17387 unsigned i
, j
, veclen
;
17388 struct ipa_parm_adjustment adj
;
17389 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
17391 memset (&adj
, 0, sizeof (adj
));
17392 tree parm
= args
[i
];
17393 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
17394 adj
.base_index
= i
;
17397 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
17398 node
->simdclone
->args
[i
].orig_type
= parm_type
;
17400 switch (node
->simdclone
->args
[i
].arg_type
)
17403 /* No adjustment necessary for scalar arguments. */
17404 adj
.op
= IPA_PARM_OP_COPY
;
17406 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17407 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17408 if (node
->definition
)
17409 node
->simdclone
->args
[i
].simd_array
17410 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17411 TREE_TYPE (parm_type
),
17412 node
->simdclone
->simdlen
);
17413 adj
.op
= IPA_PARM_OP_COPY
;
17415 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17416 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17417 case SIMD_CLONE_ARG_TYPE_VECTOR
:
17418 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
17419 veclen
= node
->simdclone
->vecsize_int
;
17421 veclen
= node
->simdclone
->vecsize_float
;
17422 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
17423 if (veclen
> node
->simdclone
->simdlen
)
17424 veclen
= node
->simdclone
->simdlen
;
17425 adj
.arg_prefix
= "simd";
17426 if (POINTER_TYPE_P (parm_type
))
17427 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17429 adj
.type
= build_vector_type (parm_type
, veclen
);
17430 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
17431 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17433 adjustments
.safe_push (adj
);
17436 memset (&adj
, 0, sizeof (adj
));
17437 adj
.op
= IPA_PARM_OP_NEW
;
17438 adj
.arg_prefix
= "simd";
17439 adj
.base_index
= i
;
17440 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
17444 if (node
->definition
)
17445 node
->simdclone
->args
[i
].simd_array
17446 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17447 parm_type
, node
->simdclone
->simdlen
);
17449 adjustments
.safe_push (adj
);
17452 if (node
->simdclone
->inbranch
)
17455 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
17458 memset (&adj
, 0, sizeof (adj
));
17459 adj
.op
= IPA_PARM_OP_NEW
;
17460 adj
.arg_prefix
= "mask";
17462 adj
.base_index
= i
;
17463 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
17464 veclen
= node
->simdclone
->vecsize_int
;
17466 veclen
= node
->simdclone
->vecsize_float
;
17467 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
17468 if (veclen
> node
->simdclone
->simdlen
)
17469 veclen
= node
->simdclone
->simdlen
;
17470 if (POINTER_TYPE_P (base_type
))
17471 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17473 adj
.type
= build_vector_type (base_type
, veclen
);
17474 adjustments
.safe_push (adj
);
17476 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17477 adjustments
.safe_push (adj
);
17479 /* We have previously allocated one extra entry for the mask. Use
17481 struct cgraph_simd_clone
*sc
= node
->simdclone
;
17483 if (node
->definition
)
17485 sc
->args
[i
].orig_arg
17486 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
17487 sc
->args
[i
].simd_array
17488 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
17490 sc
->args
[i
].orig_type
= base_type
;
17491 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
17494 if (node
->definition
)
17495 ipa_modify_formal_parameters (node
->decl
, adjustments
);
17498 tree new_arg_types
= NULL_TREE
, new_reversed
;
17499 bool last_parm_void
= false;
17500 if (args
.length () > 0 && args
.last () == void_type_node
)
17501 last_parm_void
= true;
17503 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
17504 j
= adjustments
.length ();
17505 for (i
= 0; i
< j
; i
++)
17507 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
17509 if (adj
->op
== IPA_PARM_OP_COPY
)
17510 ptype
= args
[adj
->base_index
];
17513 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
17515 new_reversed
= nreverse (new_arg_types
);
17516 if (last_parm_void
)
17519 TREE_CHAIN (new_arg_types
) = void_list_node
;
17521 new_reversed
= void_list_node
;
17524 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
17525 TYPE_ARG_TYPES (new_type
) = new_reversed
;
17526 TREE_TYPE (node
->decl
) = new_type
;
17528 adjustments
.release ();
17531 return adjustments
;
17534 /* Initialize and copy the function arguments in NODE to their
17535 corresponding local simd arrays. Returns a fresh gimple_seq with
17536 the instruction sequence generated. */
17539 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
17540 ipa_parm_adjustment_vec adjustments
)
17542 gimple_seq seq
= NULL
;
17543 unsigned i
= 0, j
= 0, k
;
17545 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
17547 arg
= DECL_CHAIN (arg
), i
++, j
++)
17549 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
17550 || POINTER_TYPE_P (TREE_TYPE (arg
)))
17553 node
->simdclone
->args
[i
].vector_arg
= arg
;
17555 tree array
= node
->simdclone
->args
[i
].simd_array
;
17556 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
17558 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17559 tree ptr
= build_fold_addr_expr (array
);
17560 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17561 build_int_cst (ptype
, 0));
17562 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17563 gimplify_and_add (t
, &seq
);
17567 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
17568 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17569 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
17571 tree ptr
= build_fold_addr_expr (array
);
17575 arg
= DECL_CHAIN (arg
);
17579 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
17580 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17581 build_int_cst (ptype
, k
* elemsize
));
17582 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17583 gimplify_and_add (t
, &seq
);
17590 /* Callback info for ipa_simd_modify_stmt_ops below. */
17592 struct modify_stmt_info
{
17593 ipa_parm_adjustment_vec adjustments
;
17595 /* True if the parent statement was modified by
17596 ipa_simd_modify_stmt_ops. */
17600 /* Callback for walk_gimple_op.
17602 Adjust operands from a given statement as specified in the
17603 adjustments vector in the callback data. */
17606 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
17608 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17609 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
17610 tree
*orig_tp
= tp
;
17611 if (TREE_CODE (*tp
) == ADDR_EXPR
)
17612 tp
= &TREE_OPERAND (*tp
, 0);
17613 struct ipa_parm_adjustment
*cand
= NULL
;
17614 if (TREE_CODE (*tp
) == PARM_DECL
)
17615 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
17619 *walk_subtrees
= 0;
17622 tree repl
= NULL_TREE
;
17624 repl
= unshare_expr (cand
->new_decl
);
17629 *walk_subtrees
= 0;
17630 bool modified
= info
->modified
;
17631 info
->modified
= false;
17632 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
17633 if (!info
->modified
)
17635 info
->modified
= modified
;
17638 info
->modified
= modified
;
17647 repl
= build_fold_addr_expr (repl
);
17649 if (is_gimple_debug (info
->stmt
))
17651 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
17652 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
17653 DECL_ARTIFICIAL (vexpr
) = 1;
17654 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
17655 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
17660 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
17661 repl
= gimple_assign_lhs (stmt
);
17663 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
17664 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17667 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
17669 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
17675 info
->modified
= true;
17679 /* Traverse the function body and perform all modifications as
17680 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
17681 modified such that the replacement/reduction value will now be an
17682 offset into the corresponding simd_array.
17684 This function will replace all function argument uses with their
17685 corresponding simd array elements, and ajust the return values
17689 ipa_simd_modify_function_body (struct cgraph_node
*node
,
17690 ipa_parm_adjustment_vec adjustments
,
17691 tree retval_array
, tree iter
)
17694 unsigned int i
, j
, l
;
17696 /* Re-use the adjustments array, but this time use it to replace
17697 every function argument use to an offset into the corresponding
17699 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
17701 if (!node
->simdclone
->args
[i
].vector_arg
)
17704 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17705 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
17706 adjustments
[j
].new_decl
17707 = build4 (ARRAY_REF
,
17709 node
->simdclone
->args
[i
].simd_array
,
17711 NULL_TREE
, NULL_TREE
);
17712 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
17713 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
17714 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
17717 l
= adjustments
.length ();
17718 for (i
= 1; i
< num_ssa_names
; i
++)
17720 tree name
= ssa_name (i
);
17722 && SSA_NAME_VAR (name
)
17723 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
17725 for (j
= 0; j
< l
; j
++)
17726 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
17727 && adjustments
[j
].new_decl
)
17730 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
17733 = copy_var_decl (adjustments
[j
].base
,
17734 DECL_NAME (adjustments
[j
].base
),
17735 TREE_TYPE (adjustments
[j
].base
));
17736 adjustments
[j
].new_ssa_base
= base_var
;
17739 base_var
= adjustments
[j
].new_ssa_base
;
17740 if (SSA_NAME_IS_DEFAULT_DEF (name
))
17742 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17743 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
17744 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
17745 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
17746 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17747 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
17748 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
17749 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17752 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17757 struct modify_stmt_info info
;
17758 info
.adjustments
= adjustments
;
17760 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
17762 gimple_stmt_iterator gsi
;
17764 gsi
= gsi_start_bb (bb
);
17765 while (!gsi_end_p (gsi
))
17767 gimple
*stmt
= gsi_stmt (gsi
);
17769 struct walk_stmt_info wi
;
17771 memset (&wi
, 0, sizeof (wi
));
17772 info
.modified
= false;
17774 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
17776 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
17778 tree retval
= gimple_return_retval (return_stmt
);
17781 gsi_remove (&gsi
, true);
17785 /* Replace `return foo' with `retval_array[iter] = foo'. */
17786 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
17787 retval_array
, iter
, NULL
, NULL
);
17788 stmt
= gimple_build_assign (ref
, retval
);
17789 gsi_replace (&gsi
, stmt
, true);
17790 info
.modified
= true;
17795 update_stmt (stmt
);
17796 if (maybe_clean_eh_stmt (stmt
))
17797 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
17804 /* Helper function of simd_clone_adjust, return linear step addend
17805 of Ith argument. */
17808 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
17809 tree addtype
, basic_block entry_bb
)
17811 tree ptype
= NULL_TREE
;
17812 switch (node
->simdclone
->args
[i
].arg_type
)
17814 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
17815 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17816 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17817 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17818 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
17819 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17820 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
17821 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17823 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17824 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17825 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
17828 gcc_unreachable ();
17831 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
17832 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
17833 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
17834 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17837 if (is_gimple_reg (arg
))
17838 ret
= get_or_create_ssa_default_def (cfun
, arg
);
17841 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
17842 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17843 ret
= gimple_assign_lhs (g
);
17845 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
17847 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
17848 build_simple_mem_ref (ret
));
17849 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17850 ret
= gimple_assign_lhs (g
);
17852 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
17854 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
17855 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17856 ret
= gimple_assign_lhs (g
);
17858 if (POINTER_TYPE_P (ptype
))
17860 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
17861 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
17863 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
17864 ret
, fold_convert (addtype
, size
));
17865 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17866 ret
= gimple_assign_lhs (g
);
17872 /* Adjust the argument types in NODE to their appropriate vector
17876 simd_clone_adjust (struct cgraph_node
*node
)
17878 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
17880 targetm
.simd_clone
.adjust (node
);
17882 tree retval
= simd_clone_adjust_return_type (node
);
17883 ipa_parm_adjustment_vec adjustments
17884 = simd_clone_adjust_argument_types (node
);
17886 push_gimplify_context ();
17888 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
17890 /* Adjust all uses of vector arguments accordingly. Adjust all
17891 return values accordingly. */
17892 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
17893 tree iter1
= make_ssa_name (iter
);
17894 tree iter2
= make_ssa_name (iter
);
17895 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
17897 /* Initialize the iteration variable. */
17898 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17899 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
17900 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17901 /* Insert the SIMD array and iv initialization at function
17903 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
17905 pop_gimplify_context (NULL
);
17907 /* Create a new BB right before the original exit BB, to hold the
17908 iteration increment and the condition/branch. */
17909 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
17910 basic_block incr_bb
= create_empty_bb (orig_exit
);
17911 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
17912 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
17913 flag. Set it now to be a FALLTHRU_EDGE. */
17914 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
17915 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
17916 for (unsigned i
= 0;
17917 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
17919 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
17920 redirect_edge_succ (e
, incr_bb
);
17922 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
17923 e
->probability
= REG_BR_PROB_BASE
;
17924 gsi
= gsi_last_bb (incr_bb
);
17925 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
17926 build_int_cst (unsigned_type_node
, 1));
17927 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17929 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
17930 struct loop
*loop
= alloc_loop ();
17931 cfun
->has_force_vectorize_loops
= true;
17932 loop
->safelen
= node
->simdclone
->simdlen
;
17933 loop
->force_vectorize
= true;
17934 loop
->header
= body_bb
;
17936 /* Branch around the body if the mask applies. */
17937 if (node
->simdclone
->inbranch
)
17939 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
17941 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
17942 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
17943 tree aref
= build4 (ARRAY_REF
,
17944 TREE_TYPE (TREE_TYPE (mask_array
)),
17947 g
= gimple_build_assign (mask
, aref
);
17948 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17949 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
17950 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
17952 aref
= build1 (VIEW_CONVERT_EXPR
,
17953 build_nonstandard_integer_type (bitsize
, 0), mask
);
17954 mask
= make_ssa_name (TREE_TYPE (aref
));
17955 g
= gimple_build_assign (mask
, aref
);
17956 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17959 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
17961 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17962 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
17963 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
17966 /* Generate the condition. */
17967 g
= gimple_build_cond (LT_EXPR
,
17969 build_int_cst (unsigned_type_node
,
17970 node
->simdclone
->simdlen
),
17972 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17973 e
= split_block (incr_bb
, gsi_stmt (gsi
));
17974 basic_block latch_bb
= e
->dest
;
17975 basic_block new_exit_bb
;
17976 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
17977 loop
->latch
= latch_bb
;
17979 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
17981 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
17982 /* The successor of incr_bb is already pointing to latch_bb; just
17984 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
17985 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
17987 gphi
*phi
= create_phi_node (iter1
, body_bb
);
17988 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
17989 edge latch_edge
= single_succ_edge (latch_bb
);
17990 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
17992 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17994 /* Generate the new return. */
17995 gsi
= gsi_last_bb (new_exit_bb
);
17997 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
17998 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
17999 retval
= TREE_OPERAND (retval
, 0);
18002 retval
= build1 (VIEW_CONVERT_EXPR
,
18003 TREE_TYPE (TREE_TYPE (node
->decl
)),
18005 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
18006 false, GSI_CONTINUE_LINKING
);
18008 g
= gimple_build_return (retval
);
18009 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
18011 /* Handle aligned clauses by replacing default defs of the aligned
18012 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
18013 lhs. Handle linear by adding PHIs. */
18014 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
18015 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
18016 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
18017 || !is_gimple_reg_type
18018 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
18020 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18021 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
18022 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
18025 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
18026 gimple_add_tmp_var (iter1
);
18028 gsi
= gsi_after_labels (entry_bb
);
18029 g
= gimple_build_assign (iter1
, orig_arg
);
18030 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18031 gsi
= gsi_after_labels (body_bb
);
18032 g
= gimple_build_assign (orig_arg
, iter1
);
18033 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18035 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
18036 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
18037 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
18039 && TREE_ADDRESSABLE
18040 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
18042 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18043 tree def
= ssa_default_def (cfun
, orig_arg
);
18044 if (def
&& !has_zero_uses (def
))
18046 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
18047 gimple_add_tmp_var (iter1
);
18048 gsi
= gsi_after_labels (entry_bb
);
18049 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
18050 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18051 gsi
= gsi_after_labels (body_bb
);
18052 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
18053 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18056 else if (node
->simdclone
->args
[i
].alignment
18057 && node
->simdclone
->args
[i
].arg_type
18058 == SIMD_CLONE_ARG_TYPE_UNIFORM
18059 && (node
->simdclone
->args
[i
].alignment
18060 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
18061 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
18064 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
18065 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18066 tree def
= ssa_default_def (cfun
, orig_arg
);
18067 if (def
&& !has_zero_uses (def
))
18069 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
18070 gimple_seq seq
= NULL
;
18071 bool need_cvt
= false;
18073 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
18075 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
18078 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
18079 gimple_call_set_lhs (g
, t
);
18080 gimple_seq_add_stmt_without_update (&seq
, g
);
18083 t
= make_ssa_name (orig_arg
);
18084 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
18085 gimple_seq_add_stmt_without_update (&seq
, g
);
18087 gsi_insert_seq_on_edge_immediate
18088 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
18090 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
18091 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
18093 node
->create_edge (cgraph_node::get_create (fn
),
18094 call
, entry_bb
->count
, freq
);
18096 imm_use_iterator iter
;
18097 use_operand_p use_p
;
18099 tree repl
= gimple_get_lhs (g
);
18100 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18101 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
18104 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18105 SET_USE (use_p
, repl
);
18108 else if ((node
->simdclone
->args
[i
].arg_type
18109 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
18110 || (node
->simdclone
->args
[i
].arg_type
18111 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
18112 || (node
->simdclone
->args
[i
].arg_type
18113 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
18114 || (node
->simdclone
->args
[i
].arg_type
18115 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
18117 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18118 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18119 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
18120 tree def
= NULL_TREE
;
18121 if (TREE_ADDRESSABLE (orig_arg
))
18123 def
= make_ssa_name (TREE_TYPE (orig_arg
));
18124 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
18125 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
18126 gsi
= gsi_after_labels (entry_bb
);
18127 g
= gimple_build_assign (def
, orig_arg
);
18128 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18132 def
= ssa_default_def (cfun
, orig_arg
);
18133 if (!def
|| has_zero_uses (def
))
18137 iter1
= make_ssa_name (orig_arg
);
18138 iter2
= make_ssa_name (orig_arg
);
18143 phi
= create_phi_node (iter1
, body_bb
);
18144 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
18145 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18146 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18147 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18148 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18149 ? TREE_TYPE (orig_arg
) : sizetype
;
18150 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
18152 gsi
= gsi_last_bb (incr_bb
);
18153 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
18154 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18156 imm_use_iterator iter
;
18157 use_operand_p use_p
;
18159 if (TREE_ADDRESSABLE (orig_arg
))
18161 gsi
= gsi_after_labels (body_bb
);
18162 g
= gimple_build_assign (orig_arg
, iter1
);
18163 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18166 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18167 if (use_stmt
== phi
)
18170 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18171 SET_USE (use_p
, iter1
);
18174 else if (node
->simdclone
->args
[i
].arg_type
18175 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
18176 || (node
->simdclone
->args
[i
].arg_type
18177 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
18179 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18180 tree def
= ssa_default_def (cfun
, orig_arg
);
18181 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
18182 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
18183 if (def
&& !has_zero_uses (def
))
18185 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
18186 iter1
= make_ssa_name (orig_arg
);
18187 iter2
= make_ssa_name (orig_arg
);
18188 tree iter3
= make_ssa_name (rtype
);
18189 tree iter4
= make_ssa_name (rtype
);
18190 tree iter5
= make_ssa_name (rtype
);
18191 gsi
= gsi_after_labels (entry_bb
);
18193 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
18194 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
18196 tree array
= node
->simdclone
->args
[i
].simd_array
;
18197 TREE_ADDRESSABLE (array
) = 1;
18198 tree ptr
= build_fold_addr_expr (array
);
18199 phi
= create_phi_node (iter1
, body_bb
);
18200 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
18201 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18202 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
18203 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
18204 gsi
= gsi_last_bb (incr_bb
);
18205 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18207 phi
= create_phi_node (iter4
, body_bb
);
18208 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
18209 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
18210 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18211 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18212 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18213 ? TREE_TYPE (iter3
) : sizetype
;
18214 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
18216 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
18217 gsi
= gsi_last_bb (incr_bb
);
18218 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18220 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
18221 gsi
= gsi_after_labels (body_bb
);
18222 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18224 imm_use_iterator iter
;
18225 use_operand_p use_p
;
18227 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18228 if (use_stmt
== load
)
18231 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18232 SET_USE (use_p
, iter1
);
18234 if (!TYPE_READONLY (rtype
))
18236 tree v
= make_ssa_name (rtype
);
18237 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
18238 size_zero_node
, NULL_TREE
,
18240 gsi
= gsi_after_labels (new_exit_bb
);
18241 g
= gimple_build_assign (v
, aref
);
18242 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18243 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
18244 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18249 calculate_dominance_info (CDI_DOMINATORS
);
18250 add_loop (loop
, loop
->header
->loop_father
);
18251 update_ssa (TODO_update_ssa
);
18256 /* If the function in NODE is tagged as an elemental SIMD function,
18257 create the appropriate SIMD clones. */
18260 expand_simd_clones (struct cgraph_node
*node
)
18262 tree attr
= lookup_attribute ("omp declare simd",
18263 DECL_ATTRIBUTES (node
->decl
));
18264 if (attr
== NULL_TREE
18265 || node
->global
.inlined_to
18266 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
18270 #pragma omp declare simd
18272 in C, there we don't know the argument types at all. */
18273 if (!node
->definition
18274 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
18279 /* Start with parsing the "omp declare simd" attribute(s). */
18280 bool inbranch_clause_specified
;
18281 struct cgraph_simd_clone
*clone_info
18282 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
18283 &inbranch_clause_specified
);
18284 if (clone_info
== NULL
)
18287 int orig_simdlen
= clone_info
->simdlen
;
18288 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
18289 /* The target can return 0 (no simd clones should be created),
18290 1 (just one ISA of simd clones should be created) or higher
18291 count of ISA variants. In that case, clone_info is initialized
18292 for the first ISA variant. */
18294 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
18299 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
18300 also create one inbranch and one !inbranch clone of it. */
18301 for (int i
= 0; i
< count
* 2; i
++)
18303 struct cgraph_simd_clone
*clone
= clone_info
;
18304 if (inbranch_clause_specified
&& (i
& 1) != 0)
18309 clone
= simd_clone_struct_alloc (clone_info
->nargs
18311 simd_clone_struct_copy (clone
, clone_info
);
18312 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
18313 and simd_clone_adjust_argument_types did to the first
18315 clone
->nargs
-= clone_info
->inbranch
;
18316 clone
->simdlen
= orig_simdlen
;
18317 /* And call the target hook again to get the right ISA. */
18318 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
18322 clone
->inbranch
= 1;
18325 /* simd_clone_mangle might fail if such a clone has been created
18327 tree id
= simd_clone_mangle (node
, clone
);
18328 if (id
== NULL_TREE
)
18331 /* Only when we are sure we want to create the clone actually
18332 clone the function (or definitions) or create another
18333 extern FUNCTION_DECL (for prototypes without definitions). */
18334 struct cgraph_node
*n
= simd_clone_create (node
);
18338 n
->simdclone
= clone
;
18339 clone
->origin
= node
;
18340 clone
->next_clone
= NULL
;
18341 if (node
->simd_clones
== NULL
)
18343 clone
->prev_clone
= n
;
18344 node
->simd_clones
= n
;
18348 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
18349 clone
->prev_clone
->simdclone
->next_clone
= n
;
18350 node
->simd_clones
->simdclone
->prev_clone
= n
;
18352 symtab
->change_decl_assembler_name (n
->decl
, id
);
18353 /* And finally adjust the return type, parameters and for
18354 definitions also function body. */
18355 if (node
->definition
)
18356 simd_clone_adjust (n
);
18359 simd_clone_adjust_return_type (n
);
18360 simd_clone_adjust_argument_types (n
);
18364 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
18367 /* Entry point for IPA simd clone creation pass. */
18369 static unsigned int
18370 ipa_omp_simd_clone (void)
18372 struct cgraph_node
*node
;
18373 FOR_EACH_FUNCTION (node
)
18374 expand_simd_clones (node
);
18380 const pass_data pass_data_omp_simd_clone
=
18382 SIMPLE_IPA_PASS
, /* type */
18383 "simdclone", /* name */
18384 OPTGROUP_NONE
, /* optinfo_flags */
18385 TV_NONE
, /* tv_id */
18386 ( PROP_ssa
| PROP_cfg
), /* properties_required */
18387 0, /* properties_provided */
18388 0, /* properties_destroyed */
18389 0, /* todo_flags_start */
18390 0, /* todo_flags_finish */
18393 class pass_omp_simd_clone
: public simple_ipa_opt_pass
18396 pass_omp_simd_clone(gcc::context
*ctxt
)
18397 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
18400 /* opt_pass methods: */
18401 virtual bool gate (function
*);
18402 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
18406 pass_omp_simd_clone::gate (function
*)
18408 return ((flag_openmp
|| flag_openmp_simd
18410 || (in_lto_p
&& !flag_wpa
))
18411 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
18414 } // anon namespace
18416 simple_ipa_opt_pass
*
18417 make_pass_omp_simd_clone (gcc::context
*ctxt
)
18419 return new pass_omp_simd_clone (ctxt
);
18422 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18423 adds their addresses and sizes to constructor-vector V_CTOR. */
18425 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18426 vec
<constructor_elt
, va_gc
> *v_ctor
)
18428 unsigned len
= vec_safe_length (v_decls
);
18429 for (unsigned i
= 0; i
< len
; i
++)
18431 tree it
= (*v_decls
)[i
];
18432 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
18434 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
18436 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
18437 fold_convert (const_ptr_type_node
,
18438 DECL_SIZE_UNIT (it
)));
18442 /* Create new symbols containing (address, size) pairs for global variables,
18443 marked with "omp declare target" attribute, as well as addresses for the
18444 functions, which are outlined offloading regions. */
18446 omp_finish_file (void)
18448 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18449 unsigned num_vars
= vec_safe_length (offload_vars
);
18451 if (num_funcs
== 0 && num_vars
== 0)
18454 if (targetm_common
.have_named_sections
)
18456 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18457 vec_alloc (v_f
, num_funcs
);
18458 vec_alloc (v_v
, num_vars
* 2);
18460 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18461 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18463 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18465 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18467 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18468 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18469 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18470 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18471 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18472 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18473 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18474 get_identifier (".offload_func_table"),
18476 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18477 get_identifier (".offload_var_table"),
18479 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18480 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18481 otherwise a joint table in a binary will contain padding between
18482 tables from multiple object files. */
18483 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18484 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
18485 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
18486 DECL_INITIAL (funcs_decl
) = ctor_f
;
18487 DECL_INITIAL (vars_decl
) = ctor_v
;
18488 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18489 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18491 varpool_node::finalize_decl (vars_decl
);
18492 varpool_node::finalize_decl (funcs_decl
);
18496 for (unsigned i
= 0; i
< num_funcs
; i
++)
18498 tree it
= (*offload_funcs
)[i
];
18499 targetm
.record_offload_symbol (it
);
18501 for (unsigned i
= 0; i
< num_vars
; i
++)
18503 tree it
= (*offload_vars
)[i
];
18504 targetm
.record_offload_symbol (it
);
18509 /* Find the number of threads (POS = false), or thread number (POS =
18510 true) for an OpenACC region partitioned as MASK. Setup code
18511 required for the calculation is added to SEQ. */
18514 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18516 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18519 /* Start at gang level, and examine relevant dimension indices. */
18520 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18521 if (GOMP_DIM_MASK (ix
) & mask
)
18523 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18527 /* We had an outer index, so scale that by the size of
18529 tree n
= create_tmp_var (integer_type_node
);
18531 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18533 gimple_call_set_lhs (call
, n
);
18534 gimple_seq_add_stmt (seq
, call
);
18535 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18539 /* Determine index in this dimension. */
18540 tree id
= create_tmp_var (integer_type_node
);
18541 gimple
*call
= gimple_build_call_internal
18542 (IFN_GOACC_DIM_POS
, 1, arg
);
18544 gimple_call_set_lhs (call
, id
);
18545 gimple_seq_add_stmt (seq
, call
);
18547 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18553 if (res
== NULL_TREE
)
18554 res
= integer_zero_node
;
18559 /* Transform IFN_GOACC_LOOP calls to actual code. See
18560 expand_oacc_for for where these are generated. At the vector
18561 level, we stride loops, such that each member of a warp will
18562 operate on adjacent iterations. At the worker and gang level,
18563 each gang/warp executes a set of contiguous iterations. Chunking
18564 can override this such that each iteration engine executes a
18565 contiguous chunk, and then moves on to stride to the next chunk. */
18568 oacc_xform_loop (gcall
*call
)
18570 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18571 enum ifn_goacc_loop_kind code
18572 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18573 tree dir
= gimple_call_arg (call
, 1);
18574 tree range
= gimple_call_arg (call
, 2);
18575 tree step
= gimple_call_arg (call
, 3);
18576 tree chunk_size
= NULL_TREE
;
18577 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18578 tree lhs
= gimple_call_lhs (call
);
18579 tree type
= TREE_TYPE (lhs
);
18580 tree diff_type
= TREE_TYPE (range
);
18581 tree r
= NULL_TREE
;
18582 gimple_seq seq
= NULL
;
18583 bool chunking
= false, striding
= true;
18584 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18585 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18587 #ifdef ACCEL_COMPILER
18588 chunk_size
= gimple_call_arg (call
, 4);
18589 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18590 || integer_zerop (chunk_size
)) /* Default (also static). */
18592 /* If we're at the gang level, we want each to execute a
18593 contiguous run of iterations. Otherwise we want each element
18595 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18600 /* Chunk of size 1 is striding. */
18601 striding
= integer_onep (chunk_size
);
18602 chunking
= !striding
;
18606 /* striding=true, chunking=true
18608 striding=true, chunking=false
18610 striding=false,chunking=true
18611 -> chunks=ceil (range/(chunksize*threads*step))
18612 striding=false,chunking=false
18613 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18614 push_gimplify_context (true);
18618 default: gcc_unreachable ();
18620 case IFN_GOACC_LOOP_CHUNKS
:
18622 r
= build_int_cst (type
, 1);
18626 = (range - dir) / (chunks * step * num_threads) + dir */
18627 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18628 per
= fold_convert (type
, per
);
18629 chunk_size
= fold_convert (type
, chunk_size
);
18630 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18631 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18632 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18633 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18634 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18638 case IFN_GOACC_LOOP_STEP
:
18640 /* If striding, step by the entire compute volume, otherwise
18641 step by the inner volume. */
18642 unsigned volume
= striding
? mask
: inner_mask
;
18644 r
= oacc_thread_numbers (false, volume
, &seq
);
18645 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18649 case IFN_GOACC_LOOP_OFFSET
:
18652 r
= oacc_thread_numbers (true, mask
, &seq
);
18653 r
= fold_convert (diff_type
, r
);
18657 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18658 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18659 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18660 inner_size
, outer_size
);
18662 volume
= fold_convert (diff_type
, volume
);
18664 chunk_size
= fold_convert (diff_type
, chunk_size
);
18667 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18669 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18670 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18671 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18674 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18675 fold_convert (diff_type
, inner_size
));
18676 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18677 r
= fold_convert (diff_type
, r
);
18678 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18680 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18681 inner
= fold_convert (diff_type
, inner
);
18682 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18686 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18688 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18689 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18691 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18694 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18695 if (type
!= diff_type
)
18696 r
= fold_convert (type
, r
);
18699 case IFN_GOACC_LOOP_BOUND
:
18704 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18705 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18706 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18707 inner_size
, outer_size
);
18709 volume
= fold_convert (diff_type
, volume
);
18711 chunk_size
= fold_convert (diff_type
, chunk_size
);
18714 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18716 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18717 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18718 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18721 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18722 fold_convert (diff_type
, inner_size
));
18724 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18726 tree offset
= gimple_call_arg (call
, 6);
18727 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18728 fold_convert (diff_type
, offset
));
18729 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18730 diff_type
, r
, range
);
18732 if (diff_type
!= type
)
18733 r
= fold_convert (type
, r
);
18737 gimplify_assign (lhs
, r
, &seq
);
18739 pop_gimplify_context (NULL
);
18741 gsi_replace_with_seq (&gsi
, seq
, true);
18744 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18745 raw attribute. DIMS is an array of dimensions, which is returned.
18746 Returns the function level dimensionality -- the level at which an
18747 offload routine wishes to partition a loop. */
18750 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
18752 tree purpose
[GOMP_DIM_MAX
];
18754 tree pos
= TREE_VALUE (attrs
);
18757 /* Make sure the attribute creator attached the dimension
18761 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18763 purpose
[ix
] = TREE_PURPOSE (pos
);
18767 if (integer_zerop (purpose
[ix
]))
18769 else if (fn_level
< 0)
18773 tree val
= TREE_VALUE (pos
);
18774 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18775 pos
= TREE_CHAIN (pos
);
18778 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
18780 /* Default anything left to 1. */
18781 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18790 /* Replace the attribute with new values. */
18792 for (ix
= GOMP_DIM_MAX
; ix
--;)
18793 pos
= tree_cons (purpose
[ix
],
18794 build_int_cst (integer_type_node
, dims
[ix
]),
18796 replace_oacc_fn_attrib (fn
, pos
);
18802 /* Create an empty OpenACC loop structure at LOC. */
18805 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18807 oacc_loop
*loop
= XCNEW (oacc_loop
);
18809 loop
->parent
= parent
;
18810 loop
->child
= loop
->sibling
= NULL
;
18814 loop
->sibling
= parent
->child
;
18815 parent
->child
= loop
;
18819 loop
->marker
= NULL
;
18820 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18821 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18822 loop
->routine
= NULL_TREE
;
18824 loop
->mask
= loop
->flags
= 0;
18825 loop
->chunk_size
= 0;
18826 loop
->head_end
= NULL
;
18831 /* Create an outermost, dummy OpenACC loop for offloaded function
18835 new_oacc_loop_outer (tree decl
)
18837 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18840 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18841 Link into PARENT loop. Return the new loop. */
18844 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18846 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18848 loop
->marker
= marker
;
18850 /* TODO: This is where device_type flattening would occur for the loop
18853 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18855 tree chunk_size
= integer_zero_node
;
18856 if (loop
->flags
& OLF_GANG_STATIC
)
18857 chunk_size
= gimple_call_arg (marker
, 4);
18858 loop
->chunk_size
= chunk_size
;
18863 /* Create a dummy loop encompassing a call to a openACC routine.
18864 Extract the routine's partitioning requirements. */
18867 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18869 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18870 int dims
[GOMP_DIM_MAX
];
18871 int level
= oacc_validate_dims (decl
, attrs
, dims
);
18873 gcc_assert (level
>= 0);
18875 loop
->marker
= call
;
18876 loop
->routine
= decl
;
18877 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18878 ^ (GOMP_DIM_MASK (level
) - 1));
18881 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18882 Return the parent loop. */
18885 finish_oacc_loop (oacc_loop
*loop
)
18887 return loop
->parent
;
18890 /* Free all OpenACC loop structures within LOOP (inclusive). */
18893 free_oacc_loop (oacc_loop
*loop
)
18896 free_oacc_loop (loop
->sibling
);
18898 free_oacc_loop (loop
->child
);
18903 /* Dump out the OpenACC loop head or tail beginning at FROM. */
18906 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
18907 const char *title
, int level
)
18909 enum ifn_unique_kind kind
18910 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
18912 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
18913 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
18915 gimple
*stmt
= gsi_stmt (gsi
);
18917 if (is_gimple_call (stmt
)
18918 && gimple_call_internal_p (stmt
)
18919 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
18921 enum ifn_unique_kind k
18922 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
18923 (gimple_call_arg (stmt
, 0)));
18925 if (k
== kind
&& stmt
!= from
)
18928 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
18931 while (gsi_end_p (gsi
))
18932 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
18936 /* Dump OpenACC loops LOOP, its siblings and its children. */
18939 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
18943 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
18944 loop
->flags
, loop
->mask
,
18945 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
18948 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
18951 fprintf (file
, "%*sRoutine %s:%u:%s\n",
18952 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
18953 DECL_SOURCE_LINE (loop
->routine
),
18954 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
18956 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18957 if (loop
->heads
[ix
])
18958 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
18959 for (ix
= GOMP_DIM_MAX
; ix
--;)
18960 if (loop
->tails
[ix
])
18961 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
18964 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
18966 dump_oacc_loop (file
, loop
->sibling
, depth
);
18969 void debug_oacc_loop (oacc_loop
*);
18971 /* Dump loops to stderr. */
18973 DEBUG_FUNCTION
void
18974 debug_oacc_loop (oacc_loop
*loop
)
18976 dump_oacc_loop (stderr
, loop
, 0);
18979 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
18980 structures as we go. By construction these loops are properly
18984 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
18989 if (bb
->flags
& BB_VISITED
)
18993 bb
->flags
|= BB_VISITED
;
18995 /* Scan for loop markers. */
18996 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
18999 gimple
*stmt
= gsi_stmt (gsi
);
19001 if (!is_gimple_call (stmt
))
19004 gcall
*call
= as_a
<gcall
*> (stmt
);
19006 /* If this is a routine, make a dummy loop for it. */
19007 if (tree decl
= gimple_call_fndecl (call
))
19008 if (tree attrs
= get_oacc_fn_attrib (decl
))
19010 gcc_assert (!marker
);
19011 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
19014 if (!gimple_call_internal_p (call
))
19017 if (gimple_call_internal_fn (call
) != IFN_UNIQUE
)
19020 enum ifn_unique_kind kind
19021 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19022 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
19023 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19025 if (gimple_call_num_args (call
) == 2)
19027 gcc_assert (marker
&& !remaining
);
19029 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19030 loop
= finish_oacc_loop (loop
);
19032 loop
->head_end
= call
;
19036 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
19040 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19041 loop
= new_oacc_loop (loop
, call
);
19044 gcc_assert (count
== remaining
);
19048 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19049 loop
->heads
[marker
] = call
;
19051 loop
->tails
[remaining
] = call
;
19057 if (remaining
|| marker
)
19059 bb
= single_succ (bb
);
19060 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
19064 /* Walk successor blocks. */
19068 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
19069 oacc_loop_discover_walk (loop
, e
->dest
);
19072 /* LOOP is the first sibling. Reverse the order in place and return
19073 the new first sibling. Recurse to child loops. */
19076 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
19078 oacc_loop
*last
= NULL
;
19082 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
19084 oacc_loop
*next
= loop
->sibling
;
19085 loop
->sibling
= last
;
19094 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19095 the current function. */
19098 oacc_loop_discovery ()
19102 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19103 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19105 /* The siblings were constructed in reverse order, reverse them so
19106 that diagnostics come out in an unsurprising order. */
19107 top
= oacc_loop_sibling_nreverse (top
);
19109 /* Reset the visited flags. */
19110 FOR_ALL_BB_FN (bb
, cfun
)
19111 bb
->flags
&= ~BB_VISITED
;
19116 /* Transform the abstract internal function markers starting at FROM
19117 to be for partitioning level LEVEL. Stop when we meet another HEAD
19121 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19123 enum ifn_unique_kind kind
19124 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19125 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19127 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19129 gimple
*stmt
= gsi_stmt (gsi
);
19131 if (is_gimple_call (stmt
)
19132 && gimple_call_internal_p (stmt
)
19133 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19135 enum ifn_unique_kind k
19136 = ((enum ifn_unique_kind
)
19137 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19139 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19140 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19141 else if (k
== kind
&& stmt
!= from
)
19144 else if (is_gimple_call (stmt
)
19145 && gimple_call_internal_p (stmt
)
19146 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
19147 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
19150 while (gsi_end_p (gsi
))
19151 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19155 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19156 determined partitioning mask and chunking argument. */
19159 oacc_loop_xform_loop (gcall
*end_marker
, tree mask_arg
, tree chunk_arg
)
19161 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19165 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19167 gimple
*stmt
= gsi_stmt (gsi
);
19169 if (!is_gimple_call (stmt
))
19172 gcall
*call
= as_a
<gcall
*> (stmt
);
19174 if (!gimple_call_internal_p (call
))
19177 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19180 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19181 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19182 if (TREE_INT_CST_LOW (gimple_call_arg (call
, 0))
19183 == IFN_GOACC_LOOP_BOUND
)
19187 /* If we didn't see LOOP_BOUND, it should be in the single
19188 successor block. */
19189 basic_block bb
= single_succ (gsi_bb (gsi
));
19190 gsi
= gsi_start_bb (bb
);
19194 /* Process the discovered OpenACC loops, setting the correct
19195 partitioning level etc. */
19198 oacc_loop_process (oacc_loop
*loop
)
19201 oacc_loop_process (loop
->child
);
19203 if (loop
->mask
&& !loop
->routine
)
19206 unsigned mask
= loop
->mask
;
19207 unsigned dim
= GOMP_DIM_GANG
;
19208 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19209 tree chunk_arg
= loop
->chunk_size
;
19211 oacc_loop_xform_loop (loop
->head_end
, mask_arg
, chunk_arg
);
19213 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
19217 while (!(GOMP_DIM_MASK (dim
) & mask
))
19220 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19221 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19223 mask
^= GOMP_DIM_MASK (dim
);
19228 oacc_loop_process (loop
->sibling
);
19231 /* Walk the OpenACC loop heirarchy checking and assigning the
19232 programmer-specified partitionings. OUTER_MASK is the partitioning
19233 this loop is contained within. Return true if we contain an
19234 auto-partitionable loop. */
19237 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19239 unsigned this_mask
= loop
->mask
;
19240 bool has_auto
= false;
19243 #ifdef ACCEL_COMPILER
19244 /* When device_type is supported, we want the device compiler to be
19245 noisy, if the loop parameters are device_type-specific. */
19249 if (!loop
->routine
)
19251 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19252 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19254 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19255 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19257 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19260 error_at (loop
->loc
,
19262 ? "%<seq%> overrides other OpenACC loop specifiers"
19263 : "%<auto%> conflicts with other OpenACC loop specifiers");
19265 loop
->flags
&= ~OLF_AUTO
;
19269 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19273 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19277 if (this_mask
& outer_mask
)
19279 const oacc_loop
*outer
;
19280 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19281 if (outer
->mask
& this_mask
)
19288 error_at (loop
->loc
,
19289 "%s uses same OpenACC parallelism as containing loop",
19290 loop
->routine
? "routine call" : "inner loop");
19291 inform (outer
->loc
, "containing loop here");
19294 error_at (loop
->loc
,
19295 "%s uses OpenACC parallelism disallowed by containing routine",
19296 loop
->routine
? "routine call" : "loop");
19299 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19300 "routine %qD declared here", loop
->routine
);
19302 this_mask
&= ~outer_mask
;
19306 unsigned outermost
= this_mask
& -this_mask
;
19308 if (outermost
&& outermost
<= outer_mask
)
19312 error_at (loop
->loc
,
19313 "incorrectly nested OpenACC loop parallelism");
19315 const oacc_loop
*outer
;
19316 for (outer
= loop
->parent
;
19317 outer
->flags
&& outer
->flags
< outermost
;
19318 outer
= outer
->parent
)
19320 inform (outer
->loc
, "containing loop here");
19323 this_mask
&= ~outermost
;
19327 loop
->mask
= this_mask
;
19330 && oacc_loop_fixed_partitions (loop
->child
, outer_mask
| this_mask
))
19334 && oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
))
19340 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
19341 OUTER_MASK is the partitioning this loop is contained within.
19342 Return the cumulative partitioning used by this loop, siblings and
19346 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19348 unsigned inner_mask
= 0;
19351 #ifdef ACCEL_COMPILER
19352 /* When device_type is supported, we want the device compiler to be
19353 noisy, if the loop parameters are device_type-specific. */
19358 inner_mask
|= oacc_loop_auto_partitions (loop
->child
,
19359 outer_mask
| loop
->mask
);
19361 if ((loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
))
19363 unsigned this_mask
= 0;
19365 /* Determine the outermost partitioning used within this loop. */
19366 this_mask
= inner_mask
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
19367 this_mask
= (this_mask
& -this_mask
);
19369 /* Pick the partitioning just inside that one. */
19372 /* And avoid picking one use by an outer loop. */
19373 this_mask
&= ~outer_mask
;
19375 if (!this_mask
&& noisy
)
19376 warning_at (loop
->loc
, 0,
19377 "insufficient partitioning available to parallelize loop");
19379 loop
->mask
= this_mask
;
19381 inner_mask
|= loop
->mask
;
19384 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
19389 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19393 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
19395 if (oacc_loop_fixed_partitions (loop
, outer_mask
))
19396 oacc_loop_auto_partitions (loop
, outer_mask
);
19399 /* Default fork/join early expander. Delete the function calls if
19400 there is no RTL expander. */
19403 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19404 const int *ARG_UNUSED (dims
), bool is_fork
)
19407 return targetm
.have_oacc_fork ();
19409 return targetm
.have_oacc_join ();
19412 /* Default goacc.reduction early expander.
19414 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19415 If RES_PTR is not integer-zerop:
19416 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19417 TEARDOWN - emit '*RES_PTR = VAR'
19419 emit 'LHS = VAR' */
19422 default_goacc_reduction (gcall
*call
)
19424 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19425 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19426 tree lhs
= gimple_call_lhs (call
);
19427 tree var
= gimple_call_arg (call
, 2);
19428 gimple_seq seq
= NULL
;
19430 if (code
== IFN_GOACC_REDUCTION_SETUP
19431 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19433 /* Setup and Teardown need to copy from/to the receiver object,
19434 if there is one. */
19435 tree ref_to_res
= gimple_call_arg (call
, 1);
19437 if (!integer_zerop (ref_to_res
))
19439 tree dst
= build_simple_mem_ref (ref_to_res
);
19442 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19448 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19452 /* Copy VAR to LHS, if there is an LHS. */
19454 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19456 gsi_replace_with_seq (&gsi
, seq
, true);
19459 /* Main entry point for oacc transformations which run on the device
19460 compiler after LTO, so we know what the target device is at this
19461 point (including the host fallback). */
19463 static unsigned int
19464 execute_oacc_device_lower ()
19466 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19467 int dims
[GOMP_DIM_MAX
];
19470 /* Not an offloaded function. */
19473 int fn_level
= oacc_validate_dims (current_function_decl
, attrs
, dims
);
19475 /* Discover, partition and process the loops. */
19476 oacc_loop
*loops
= oacc_loop_discovery ();
19477 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
19478 oacc_loop_partition (loops
, outer_mask
);
19479 oacc_loop_process (loops
);
19482 fprintf (dump_file
, "OpenACC loops\n");
19483 dump_oacc_loop (dump_file
, loops
, 0);
19484 fprintf (dump_file
, "\n");
19487 /* Offloaded targets may introduce new basic blocks, which require
19488 dominance information to update SSA. */
19489 calculate_dominance_info (CDI_DOMINATORS
);
19491 /* Now lower internal loop functions to target-specific code
19494 FOR_ALL_BB_FN (bb
, cfun
)
19495 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19497 gimple
*stmt
= gsi_stmt (gsi
);
19498 if (!is_gimple_call (stmt
))
19504 gcall
*call
= as_a
<gcall
*> (stmt
);
19505 if (!gimple_call_internal_p (call
))
19511 /* Rewind to allow rescan. */
19513 bool rescan
= false, remove
= false;
19514 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19520 case IFN_GOACC_LOOP
:
19521 oacc_xform_loop (call
);
19525 case IFN_GOACC_REDUCTION
:
19526 /* Mark the function for SSA renaming. */
19527 mark_virtual_operands_for_renaming (cfun
);
19529 /* If the level is -1, this ended up being an unused
19530 axis. Handle as a default. */
19531 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19532 default_goacc_reduction (call
);
19534 targetm
.goacc
.reduction (call
);
19540 enum ifn_unique_kind kind
19541 = ((enum ifn_unique_kind
)
19542 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19547 gcc_unreachable ();
19549 case IFN_UNIQUE_OACC_FORK
:
19550 case IFN_UNIQUE_OACC_JOIN
:
19551 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19553 else if (!targetm
.goacc
.fork_join
19554 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19558 case IFN_UNIQUE_OACC_HEAD_MARK
:
19559 case IFN_UNIQUE_OACC_TAIL_MARK
:
19567 if (gsi_end_p (gsi
))
19568 /* We rewound past the beginning of the BB. */
19569 gsi
= gsi_start_bb (bb
);
19571 /* Undo the rewind. */
19576 if (gimple_vdef (call
))
19577 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19578 if (gimple_call_lhs (call
))
19580 /* Propagate the data dependency var. */
19581 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19582 gimple_call_arg (call
, 1));
19583 gsi_replace (&gsi
, ass
, false);
19586 gsi_remove (&gsi
, true);
19589 /* If not rescanning, advance over the call. */
19593 free_oacc_loop (loops
);
19598 /* Default launch dimension validator. Force everything to 1. A
19599 backend that wants to provide larger dimensions must override this
19603 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19604 int ARG_UNUSED (fn_level
))
19606 bool changed
= false;
19608 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19620 /* Default dimension bound is unknown on accelerator and 1 on host. */
19623 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19625 #ifdef ACCEL_COMPILER
19634 const pass_data pass_data_oacc_device_lower
=
19636 GIMPLE_PASS
, /* type */
19637 "oaccdevlow", /* name */
19638 OPTGROUP_NONE
, /* optinfo_flags */
19639 TV_NONE
, /* tv_id */
19640 PROP_cfg
, /* properties_required */
19641 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19642 0, /* properties_destroyed */
19643 0, /* todo_flags_start */
19644 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19647 class pass_oacc_device_lower
: public gimple_opt_pass
19650 pass_oacc_device_lower (gcc::context
*ctxt
)
19651 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19654 /* opt_pass methods: */
19655 virtual unsigned int execute (function
*)
19657 bool gate
= flag_openacc
!= 0;
19662 return execute_oacc_device_lower ();
19665 }; // class pass_oacc_device_lower
19667 } // anon namespace
19670 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19672 return new pass_oacc_device_lower (ctxt
);
19675 #include "gt-omp-low.h"