1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region
*outer
;
107 /* First child region. */
108 struct omp_region
*inner
;
110 /* Next peer region. */
111 struct omp_region
*next
;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec
<tree
, va_gc
> *ws_args
;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type
;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind
;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered
*ord_stmt
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind
;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v
, n1
, n2
, step
;
204 enum tree_code cond_code
;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop
;
217 bool have_nowait
, have_ordered
, simd_schedule
;
218 unsigned char sched_modifiers
;
219 enum omp_clause_schedule_kind sched_kind
;
220 struct omp_for_data_loop
*loops
;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop
*parent
; /* Containing loop. */
230 oacc_loop
*child
; /* First inner loop. */
232 oacc_loop
*sibling
; /* Next loop within same parent. */
234 location_t loc
; /* Location of the loop start. */
236 gcall
*marker
; /* Initial head marker. */
238 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
239 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
241 tree routine
; /* Pseudo-loop enclosing a routine. */
243 unsigned mask
; /* Partitioning mask. */
244 unsigned flags
; /* Partitioning flags. */
245 tree chunk_size
; /* Chunk size. */
246 gcall
*head_end
; /* Final marker of head sequence. */
249 /* Flags for an OpenACC loop. */
251 enum oacc_loop_flags
{
252 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
253 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
254 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
255 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
257 /* Explicitly specified loop axes. */
259 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
260 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
261 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
263 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
267 static splay_tree all_contexts
;
268 static int taskreg_nesting_level
;
269 static int target_nesting_level
;
270 static struct omp_region
*root_omp_region
;
271 static bitmap task_shared_vars
;
272 static vec
<omp_context
*> taskreg_contexts
;
273 static bool omp_any_child_fn_dumped
;
275 static void scan_omp (gimple_seq
*, omp_context
*);
276 static tree
scan_omp_1_op (tree
*, int *, void *);
277 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
279 #define WALK_SUBSTMTS \
283 case GIMPLE_EH_FILTER: \
284 case GIMPLE_TRANSACTION: \
285 /* The sub-statements for these should be walked. */ \
286 *handled_ops_p = false; \
289 /* Return true if CTX corresponds to an oacc parallel region. */
292 is_oacc_parallel (omp_context
*ctx
)
294 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
295 return ((outer_type
== GIMPLE_OMP_TARGET
)
296 && (gimple_omp_target_kind (ctx
->stmt
)
297 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
300 /* Return true if CTX corresponds to an oacc kernels region. */
303 is_oacc_kernels (omp_context
*ctx
)
305 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
306 return ((outer_type
== GIMPLE_OMP_TARGET
)
307 && (gimple_omp_target_kind (ctx
->stmt
)
308 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
311 /* If DECL is the artificial dummy VAR_DECL created for non-static
312 data member privatization, return the underlying "this" parameter,
313 otherwise return NULL. */
316 omp_member_access_dummy_var (tree decl
)
319 || !DECL_ARTIFICIAL (decl
)
320 || !DECL_IGNORED_P (decl
)
321 || !DECL_HAS_VALUE_EXPR_P (decl
)
322 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
325 tree v
= DECL_VALUE_EXPR (decl
);
326 if (TREE_CODE (v
) != COMPONENT_REF
)
330 switch (TREE_CODE (v
))
336 case POINTER_PLUS_EXPR
:
337 v
= TREE_OPERAND (v
, 0);
340 if (DECL_CONTEXT (v
) == current_function_decl
341 && DECL_ARTIFICIAL (v
)
342 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
350 /* Helper for unshare_and_remap, called through walk_tree. */
353 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
355 tree
*pair
= (tree
*) data
;
358 *tp
= unshare_expr (pair
[1]);
361 else if (IS_TYPE_OR_DECL_P (*tp
))
366 /* Return unshare_expr (X) with all occurrences of FROM
370 unshare_and_remap (tree x
, tree from
, tree to
)
372 tree pair
[2] = { from
, to
};
373 x
= unshare_expr (x
);
374 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
378 /* Holds offload tables with decls. */
379 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
381 /* Convenience function for calling scan_omp_1_op on tree operands. */
384 scan_omp_op (tree
*tp
, omp_context
*ctx
)
386 struct walk_stmt_info wi
;
388 memset (&wi
, 0, sizeof (wi
));
390 wi
.want_locations
= true;
392 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
395 static void lower_omp (gimple_seq
*, omp_context
*);
396 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
397 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
399 /* Find an OMP clause of type KIND within CLAUSES. */
402 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
404 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
405 if (OMP_CLAUSE_CODE (clauses
) == kind
)
411 /* Return true if CTX is for an omp parallel. */
414 is_parallel_ctx (omp_context
*ctx
)
416 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
420 /* Return true if CTX is for an omp task. */
423 is_task_ctx (omp_context
*ctx
)
425 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
429 /* Return true if CTX is for an omp taskloop. */
432 is_taskloop_ctx (omp_context
*ctx
)
434 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
435 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
439 /* Return true if CTX is for an omp parallel or omp task. */
442 is_taskreg_ctx (omp_context
*ctx
)
444 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
448 /* Return true if REGION is a combined parallel+workshare region. */
451 is_combined_parallel (struct omp_region
*region
)
453 return region
->is_combined_parallel
;
456 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
460 adjust_for_condition (location_t loc
, enum tree_code
*cond_code
, tree
*n2
)
469 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
470 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, 1);
472 *n2
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (*n2
), *n2
,
473 build_int_cst (TREE_TYPE (*n2
), 1));
474 *cond_code
= LT_EXPR
;
477 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
478 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, -1);
480 *n2
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (*n2
), *n2
,
481 build_int_cst (TREE_TYPE (*n2
), 1));
482 *cond_code
= GT_EXPR
;
489 /* Return the looping step from INCR, extracted from the step of a gimple omp
493 get_omp_for_step_from_incr (location_t loc
, tree incr
)
496 switch (TREE_CODE (incr
))
499 step
= TREE_OPERAND (incr
, 1);
501 case POINTER_PLUS_EXPR
:
502 step
= fold_convert (ssizetype
, TREE_OPERAND (incr
, 1));
505 step
= TREE_OPERAND (incr
, 1);
506 step
= fold_build1_loc (loc
, NEGATE_EXPR
, TREE_TYPE (step
), step
);
514 /* Extract the header elements of parallel loop FOR_STMT and store
518 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
519 struct omp_for_data_loop
*loops
)
521 tree t
, var
, *collapse_iter
, *collapse_count
;
522 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
523 struct omp_for_data_loop
*loop
;
525 struct omp_for_data_loop dummy_loop
;
526 location_t loc
= gimple_location (for_stmt
);
527 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
528 bool distribute
= gimple_omp_for_kind (for_stmt
)
529 == GF_OMP_FOR_KIND_DISTRIBUTE
;
530 bool taskloop
= gimple_omp_for_kind (for_stmt
)
531 == GF_OMP_FOR_KIND_TASKLOOP
;
534 fd
->for_stmt
= for_stmt
;
536 if (gimple_omp_for_collapse (for_stmt
) > 1)
539 fd
->loops
= &fd
->loop
;
541 fd
->have_nowait
= distribute
|| simd
;
542 fd
->have_ordered
= false;
545 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
546 fd
->sched_modifiers
= 0;
547 fd
->chunk_size
= NULL_TREE
;
548 fd
->simd_schedule
= false;
549 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
550 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
551 collapse_iter
= NULL
;
552 collapse_count
= NULL
;
554 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
555 switch (OMP_CLAUSE_CODE (t
))
557 case OMP_CLAUSE_NOWAIT
:
558 fd
->have_nowait
= true;
560 case OMP_CLAUSE_ORDERED
:
561 fd
->have_ordered
= true;
562 if (OMP_CLAUSE_ORDERED_EXPR (t
))
563 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
565 case OMP_CLAUSE_SCHEDULE
:
566 gcc_assert (!distribute
&& !taskloop
);
568 = (enum omp_clause_schedule_kind
)
569 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
570 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
571 & ~OMP_CLAUSE_SCHEDULE_MASK
);
572 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
573 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
575 case OMP_CLAUSE_DIST_SCHEDULE
:
576 gcc_assert (distribute
);
577 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
579 case OMP_CLAUSE_COLLAPSE
:
580 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
581 if (fd
->collapse
> 1)
583 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
584 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
590 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
595 collapse_iter
= &iterv
;
596 collapse_count
= &countv
;
599 /* FIXME: for now map schedule(auto) to schedule(static).
600 There should be analysis to determine whether all iterations
601 are approximately the same amount of work (then schedule(static)
602 is best) or if it varies (then schedule(dynamic,N) is better). */
603 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
605 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
606 gcc_assert (fd
->chunk_size
== NULL
);
608 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
610 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
611 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
612 gcc_assert (fd
->chunk_size
== NULL
);
613 else if (fd
->chunk_size
== NULL
)
615 /* We only need to compute a default chunk size for ordered
616 static loops and dynamic loops. */
617 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
619 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
620 ? integer_zero_node
: integer_one_node
;
623 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
624 for (i
= 0; i
< cnt
; i
++)
626 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
628 else if (loops
!= NULL
)
633 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
634 gcc_assert (SSA_VAR_P (loop
->v
));
635 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
636 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
637 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
638 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
640 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
641 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
642 gcc_assert (loop
->cond_code
!= NE_EXPR
643 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKSIMD
644 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
);
645 adjust_for_condition (loc
, &loop
->cond_code
, &loop
->n2
);
647 t
= gimple_omp_for_incr (for_stmt
, i
);
648 gcc_assert (TREE_OPERAND (t
, 0) == var
);
649 loop
->step
= get_omp_for_step_from_incr (loc
, t
);
652 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
653 && !fd
->have_ordered
))
655 if (fd
->collapse
== 1)
656 iter_type
= TREE_TYPE (loop
->v
);
658 || TYPE_PRECISION (iter_type
)
659 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
661 = build_nonstandard_integer_type
662 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
664 else if (iter_type
!= long_long_unsigned_type_node
)
666 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
667 iter_type
= long_long_unsigned_type_node
;
668 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
669 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
670 >= TYPE_PRECISION (iter_type
))
674 if (loop
->cond_code
== LT_EXPR
)
675 n
= fold_build2_loc (loc
,
676 PLUS_EXPR
, TREE_TYPE (loop
->v
),
677 loop
->n2
, loop
->step
);
680 if (TREE_CODE (n
) != INTEGER_CST
681 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
682 iter_type
= long_long_unsigned_type_node
;
684 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
685 > TYPE_PRECISION (iter_type
))
689 if (loop
->cond_code
== LT_EXPR
)
692 n2
= fold_build2_loc (loc
,
693 PLUS_EXPR
, TREE_TYPE (loop
->v
),
694 loop
->n2
, loop
->step
);
698 n1
= fold_build2_loc (loc
,
699 MINUS_EXPR
, TREE_TYPE (loop
->v
),
700 loop
->n2
, loop
->step
);
703 if (TREE_CODE (n1
) != INTEGER_CST
704 || TREE_CODE (n2
) != INTEGER_CST
705 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
706 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
707 iter_type
= long_long_unsigned_type_node
;
711 if (i
>= fd
->collapse
)
714 if (collapse_count
&& *collapse_count
== NULL
)
716 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
717 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
718 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
719 if (t
&& integer_zerop (t
))
720 count
= build_zero_cst (long_long_unsigned_type_node
);
721 else if ((i
== 0 || count
!= NULL_TREE
)
722 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
723 && TREE_CONSTANT (loop
->n1
)
724 && TREE_CONSTANT (loop
->n2
)
725 && TREE_CODE (loop
->step
) == INTEGER_CST
)
727 tree itype
= TREE_TYPE (loop
->v
);
729 if (POINTER_TYPE_P (itype
))
730 itype
= signed_type_for (itype
);
731 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
732 t
= fold_build2_loc (loc
,
734 fold_convert_loc (loc
, itype
, loop
->step
), t
);
735 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
736 fold_convert_loc (loc
, itype
, loop
->n2
));
737 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
738 fold_convert_loc (loc
, itype
, loop
->n1
));
739 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
740 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
741 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
742 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
743 fold_convert_loc (loc
, itype
,
746 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
747 fold_convert_loc (loc
, itype
, loop
->step
));
748 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
749 if (count
!= NULL_TREE
)
750 count
= fold_build2_loc (loc
,
751 MULT_EXPR
, long_long_unsigned_type_node
,
755 if (TREE_CODE (count
) != INTEGER_CST
)
758 else if (count
&& !integer_zerop (count
))
765 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
766 || fd
->have_ordered
))
768 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
769 iter_type
= long_long_unsigned_type_node
;
771 iter_type
= long_integer_type_node
;
773 else if (collapse_iter
&& *collapse_iter
!= NULL
)
774 iter_type
= TREE_TYPE (*collapse_iter
);
775 fd
->iter_type
= iter_type
;
776 if (collapse_iter
&& *collapse_iter
== NULL
)
777 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
778 if (collapse_count
&& *collapse_count
== NULL
)
781 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
783 *collapse_count
= create_tmp_var (iter_type
, ".count");
786 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
788 fd
->loop
.v
= *collapse_iter
;
789 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
790 fd
->loop
.n2
= *collapse_count
;
791 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
792 fd
->loop
.cond_code
= LT_EXPR
;
799 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
800 is the immediate dominator of PAR_ENTRY_BB, return true if there
801 are no data dependencies that would prevent expanding the parallel
802 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
804 When expanding a combined parallel+workshare region, the call to
805 the child function may need additional arguments in the case of
806 GIMPLE_OMP_FOR regions. In some cases, these arguments are
807 computed out of variables passed in from the parent to the child
808 via 'struct .omp_data_s'. For instance:
810 #pragma omp parallel for schedule (guided, i * 4)
815 # BLOCK 2 (PAR_ENTRY_BB)
817 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
819 # BLOCK 3 (WS_ENTRY_BB)
820 .omp_data_i = &.omp_data_o;
821 D.1667 = .omp_data_i->i;
823 #pragma omp for schedule (guided, D.1598)
825 When we outline the parallel region, the call to the child function
826 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
827 that value is computed *after* the call site. So, in principle we
828 cannot do the transformation.
830 To see whether the code in WS_ENTRY_BB blocks the combined
831 parallel+workshare call, we collect all the variables used in the
832 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
833 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
836 FIXME. If we had the SSA form built at this point, we could merely
837 hoist the code in block 3 into block 2 and be done with it. But at
838 this point we don't have dataflow information and though we could
839 hack something up here, it is really not worth the aggravation. */
842 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
844 struct omp_for_data fd
;
845 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
847 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
850 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
852 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
854 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
856 if (fd
.iter_type
!= long_integer_type_node
)
859 /* FIXME. We give up too easily here. If any of these arguments
860 are not constants, they will likely involve variables that have
861 been mapped into fields of .omp_data_s for sharing with the child
862 function. With appropriate data flow, it would be possible to
864 if (!is_gimple_min_invariant (fd
.loop
.n1
)
865 || !is_gimple_min_invariant (fd
.loop
.n2
)
866 || !is_gimple_min_invariant (fd
.loop
.step
)
867 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
874 static int omp_max_vf (void);
876 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
877 presence (SIMD_SCHEDULE). */
880 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
885 int vf
= omp_max_vf ();
889 tree type
= TREE_TYPE (chunk_size
);
890 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
891 build_int_cst (type
, vf
- 1));
892 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
893 build_int_cst (type
, -vf
));
897 /* Collect additional arguments needed to emit a combined
898 parallel+workshare call. WS_STMT is the workshare directive being
901 static vec
<tree
, va_gc
> *
902 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
905 location_t loc
= gimple_location (ws_stmt
);
906 vec
<tree
, va_gc
> *ws_args
;
908 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
910 struct omp_for_data fd
;
913 extract_omp_for_data (for_stmt
, &fd
, NULL
);
917 if (gimple_omp_for_combined_into_p (for_stmt
))
920 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
921 OMP_CLAUSE__LOOPTEMP_
);
923 n1
= OMP_CLAUSE_DECL (innerc
);
924 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
925 OMP_CLAUSE__LOOPTEMP_
);
927 n2
= OMP_CLAUSE_DECL (innerc
);
930 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
932 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
933 ws_args
->quick_push (t
);
935 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
936 ws_args
->quick_push (t
);
938 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
939 ws_args
->quick_push (t
);
943 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
944 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
945 ws_args
->quick_push (t
);
950 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
952 /* Number of sections is equal to the number of edges from the
953 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
954 the exit of the sections region. */
955 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
956 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
957 vec_alloc (ws_args
, 1);
958 ws_args
->quick_push (t
);
966 /* Discover whether REGION is a combined parallel+workshare region. */
969 determine_parallel_type (struct omp_region
*region
)
971 basic_block par_entry_bb
, par_exit_bb
;
972 basic_block ws_entry_bb
, ws_exit_bb
;
974 if (region
== NULL
|| region
->inner
== NULL
975 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
976 || region
->inner
->cont
== NULL
)
979 /* We only support parallel+for and parallel+sections. */
980 if (region
->type
!= GIMPLE_OMP_PARALLEL
981 || (region
->inner
->type
!= GIMPLE_OMP_FOR
982 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
985 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
986 WS_EXIT_BB -> PAR_EXIT_BB. */
987 par_entry_bb
= region
->entry
;
988 par_exit_bb
= region
->exit
;
989 ws_entry_bb
= region
->inner
->entry
;
990 ws_exit_bb
= region
->inner
->exit
;
992 if (single_succ (par_entry_bb
) == ws_entry_bb
993 && single_succ (ws_exit_bb
) == par_exit_bb
994 && workshare_safe_to_combine_p (ws_entry_bb
)
995 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
996 || (last_and_only_stmt (ws_entry_bb
)
997 && last_and_only_stmt (par_exit_bb
))))
999 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1000 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1002 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1004 /* If this is a combined parallel loop, we need to determine
1005 whether or not to use the combined library calls. There
1006 are two cases where we do not apply the transformation:
1007 static loops and any kind of ordered loop. In the first
1008 case, we already open code the loop so there is no need
1009 to do anything else. In the latter case, the combined
1010 parallel loop call would still need extra synchronization
1011 to implement ordered semantics, so there would not be any
1012 gain in using the combined call. */
1013 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1014 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1016 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1017 == OMP_CLAUSE_SCHEDULE_STATIC
)
1018 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1020 region
->is_combined_parallel
= false;
1021 region
->inner
->is_combined_parallel
= false;
1026 region
->is_combined_parallel
= true;
1027 region
->inner
->is_combined_parallel
= true;
1028 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1033 /* Return true if EXPR is variable sized. */
1036 is_variable_sized (const_tree expr
)
1038 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1041 /* Return true if DECL is a reference type. */
1044 is_reference (tree decl
)
1046 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1049 /* Return the type of a decl. If the decl is reference type,
1050 return its base type. */
1052 get_base_type (tree decl
)
1054 tree type
= TREE_TYPE (decl
);
1055 if (is_reference (decl
))
1056 type
= TREE_TYPE (type
);
1060 /* Lookup variables. The "maybe" form
1061 allows for the variable form to not have been entered, otherwise we
1062 assert that the variable must have been entered. */
1065 lookup_decl (tree var
, omp_context
*ctx
)
1067 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1072 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1074 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1075 return n
? *n
: NULL_TREE
;
1079 lookup_field (tree var
, omp_context
*ctx
)
1082 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1083 return (tree
) n
->value
;
1087 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1090 n
= splay_tree_lookup (ctx
->sfield_map
1091 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1092 return (tree
) n
->value
;
1096 lookup_sfield (tree var
, omp_context
*ctx
)
1098 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1102 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1105 n
= splay_tree_lookup (ctx
->field_map
, key
);
1106 return n
? (tree
) n
->value
: NULL_TREE
;
1110 maybe_lookup_field (tree var
, omp_context
*ctx
)
1112 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1115 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1116 the parallel context if DECL is to be shared. */
1119 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1121 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1124 /* We can only use copy-in/copy-out semantics for shared variables
1125 when we know the value is not accessible from an outer scope. */
1128 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1130 /* ??? Trivially accessible from anywhere. But why would we even
1131 be passing an address in this case? Should we simply assert
1132 this to be false, or should we have a cleanup pass that removes
1133 these from the list of mappings? */
1134 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1137 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1138 without analyzing the expression whether or not its location
1139 is accessible to anyone else. In the case of nested parallel
1140 regions it certainly may be. */
1141 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1144 /* Do not use copy-in/copy-out for variables that have their
1146 if (TREE_ADDRESSABLE (decl
))
1149 /* lower_send_shared_vars only uses copy-in, but not copy-out
1151 if (TREE_READONLY (decl
)
1152 || ((TREE_CODE (decl
) == RESULT_DECL
1153 || TREE_CODE (decl
) == PARM_DECL
)
1154 && DECL_BY_REFERENCE (decl
)))
1157 /* Disallow copy-in/out in nested parallel if
1158 decl is shared in outer parallel, otherwise
1159 each thread could store the shared variable
1160 in its own copy-in location, making the
1161 variable no longer really shared. */
1162 if (shared_ctx
->is_nested
)
1166 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1167 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1174 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1175 c
; c
= OMP_CLAUSE_CHAIN (c
))
1176 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1177 && OMP_CLAUSE_DECL (c
) == decl
)
1181 goto maybe_mark_addressable_and_ret
;
1185 /* For tasks avoid using copy-in/out. As tasks can be
1186 deferred or executed in different thread, when GOMP_task
1187 returns, the task hasn't necessarily terminated. */
1188 if (is_task_ctx (shared_ctx
))
1191 maybe_mark_addressable_and_ret
:
1192 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1193 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1195 /* Taking address of OUTER in lower_send_shared_vars
1196 might need regimplification of everything that uses the
1198 if (!task_shared_vars
)
1199 task_shared_vars
= BITMAP_ALLOC (NULL
);
1200 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1201 TREE_ADDRESSABLE (outer
) = 1;
1210 /* Construct a new automatic decl similar to VAR. */
1213 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1215 tree copy
= copy_var_decl (var
, name
, type
);
1217 DECL_CONTEXT (copy
) = current_function_decl
;
1218 DECL_CHAIN (copy
) = ctx
->block_vars
;
1219 /* If VAR is listed in task_shared_vars, it means it wasn't
1220 originally addressable and is just because task needs to take
1221 it's address. But we don't need to take address of privatizations
1223 if (TREE_ADDRESSABLE (var
)
1225 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1226 TREE_ADDRESSABLE (copy
) = 0;
1227 ctx
->block_vars
= copy
;
1233 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1235 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1238 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1241 omp_build_component_ref (tree obj
, tree field
)
1243 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1244 if (TREE_THIS_VOLATILE (field
))
1245 TREE_THIS_VOLATILE (ret
) |= 1;
1246 if (TREE_READONLY (field
))
1247 TREE_READONLY (ret
) |= 1;
1251 /* Build tree nodes to access the field for VAR on the receiver side. */
1254 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1256 tree x
, field
= lookup_field (var
, ctx
);
1258 /* If the receiver record type was remapped in the child function,
1259 remap the field into the new record type. */
1260 x
= maybe_lookup_field (field
, ctx
);
1264 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1265 TREE_THIS_NOTRAP (x
) = 1;
1266 x
= omp_build_component_ref (x
, field
);
1269 x
= build_simple_mem_ref (x
);
1270 TREE_THIS_NOTRAP (x
) = 1;
1276 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1277 of a parallel, this is a component reference; for workshare constructs
1278 this is some variable. */
1281 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1285 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1287 else if (is_variable_sized (var
))
1289 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1290 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1291 x
= build_simple_mem_ref (x
);
1293 else if (is_taskreg_ctx (ctx
))
1295 bool by_ref
= use_pointer_for_field (var
, NULL
);
1296 x
= build_receiver_ref (var
, by_ref
, ctx
);
1298 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1299 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1301 /* #pragma omp simd isn't a worksharing construct, and can reference even
1302 private vars in its linear etc. clauses. */
1304 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1305 x
= lookup_decl (var
, ctx
->outer
);
1306 else if (ctx
->outer
)
1307 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1311 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1313 gcc_assert (ctx
->outer
);
1315 = splay_tree_lookup (ctx
->outer
->field_map
,
1316 (splay_tree_key
) &DECL_UID (var
));
1319 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1322 x
= lookup_decl (var
, ctx
->outer
);
1326 tree field
= (tree
) n
->value
;
1327 /* If the receiver record type was remapped in the child function,
1328 remap the field into the new record type. */
1329 x
= maybe_lookup_field (field
, ctx
->outer
);
1333 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1334 x
= omp_build_component_ref (x
, field
);
1335 if (use_pointer_for_field (var
, ctx
->outer
))
1336 x
= build_simple_mem_ref (x
);
1339 else if (ctx
->outer
)
1341 omp_context
*outer
= ctx
->outer
;
1342 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
1344 outer
= outer
->outer
;
1346 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
1348 x
= lookup_decl (var
, outer
);
1350 else if (is_reference (var
))
1351 /* This can happen with orphaned constructs. If var is reference, it is
1352 possible it is shared and as such valid. */
1354 else if (omp_member_access_dummy_var (var
))
1361 tree t
= omp_member_access_dummy_var (var
);
1364 x
= DECL_VALUE_EXPR (var
);
1365 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1367 x
= unshare_and_remap (x
, t
, o
);
1369 x
= unshare_expr (x
);
1373 if (is_reference (var
))
1374 x
= build_simple_mem_ref (x
);
1379 /* Build tree nodes to access the field for VAR on the sender side. */
1382 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1384 tree field
= lookup_sfield (key
, ctx
);
1385 return omp_build_component_ref (ctx
->sender_decl
, field
);
1389 build_sender_ref (tree var
, omp_context
*ctx
)
1391 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1394 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1395 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1398 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
,
1399 bool base_pointers_restrict
= false)
1401 tree field
, type
, sfield
= NULL_TREE
;
1402 splay_tree_key key
= (splay_tree_key
) var
;
1404 if ((mask
& 8) != 0)
1406 key
= (splay_tree_key
) &DECL_UID (var
);
1407 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1409 gcc_assert ((mask
& 1) == 0
1410 || !splay_tree_lookup (ctx
->field_map
, key
));
1411 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1412 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1413 gcc_assert ((mask
& 3) == 3
1414 || !is_gimple_omp_oacc (ctx
->stmt
));
1416 type
= TREE_TYPE (var
);
1417 /* Prevent redeclaring the var in the split-off function with a restrict
1418 pointer type. Note that we only clear type itself, restrict qualifiers in
1419 the pointed-to type will be ignored by points-to analysis. */
1420 if (POINTER_TYPE_P (type
)
1421 && TYPE_RESTRICT (type
))
1422 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
1426 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1427 type
= build_pointer_type (build_pointer_type (type
));
1431 type
= build_pointer_type (type
);
1432 if (base_pointers_restrict
)
1433 type
= build_qualified_type (type
, TYPE_QUAL_RESTRICT
);
1435 else if ((mask
& 3) == 1 && is_reference (var
))
1436 type
= TREE_TYPE (type
);
1438 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1439 FIELD_DECL
, DECL_NAME (var
), type
);
1441 /* Remember what variable this field was created for. This does have a
1442 side effect of making dwarf2out ignore this member, so for helpful
1443 debugging we clear it later in delete_omp_context. */
1444 DECL_ABSTRACT_ORIGIN (field
) = var
;
1445 if (type
== TREE_TYPE (var
))
1447 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1448 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1449 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1452 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1454 if ((mask
& 3) == 3)
1456 insert_field_into_struct (ctx
->record_type
, field
);
1457 if (ctx
->srecord_type
)
1459 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1460 FIELD_DECL
, DECL_NAME (var
), type
);
1461 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1462 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1463 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1464 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1465 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1470 if (ctx
->srecord_type
== NULL_TREE
)
1474 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1475 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1476 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1478 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1479 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1480 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1481 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1482 splay_tree_insert (ctx
->sfield_map
,
1483 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1484 (splay_tree_value
) sfield
);
1488 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1489 : ctx
->srecord_type
, field
);
1493 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1494 if ((mask
& 2) && ctx
->sfield_map
)
1495 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1499 install_var_local (tree var
, omp_context
*ctx
)
1501 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1502 insert_decl_map (&ctx
->cb
, var
, new_var
);
1506 /* Adjust the replacement for DECL in CTX for the new context. This means
1507 copying the DECL_VALUE_EXPR, and fixing up the type. */
1510 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1512 tree new_decl
, size
;
1514 new_decl
= lookup_decl (decl
, ctx
);
1516 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1518 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1519 && DECL_HAS_VALUE_EXPR_P (decl
))
1521 tree ve
= DECL_VALUE_EXPR (decl
);
1522 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1523 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1524 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1527 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1529 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1530 if (size
== error_mark_node
)
1531 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1532 DECL_SIZE (new_decl
) = size
;
1534 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1535 if (size
== error_mark_node
)
1536 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1537 DECL_SIZE_UNIT (new_decl
) = size
;
1541 /* The callback for remap_decl. Search all containing contexts for a
1542 mapping of the variable; this avoids having to duplicate the splay
1543 tree ahead of time. We know a mapping doesn't already exist in the
1544 given context. Create new mappings to implement default semantics. */
1547 omp_copy_decl (tree var
, copy_body_data
*cb
)
1549 omp_context
*ctx
= (omp_context
*) cb
;
1552 if (TREE_CODE (var
) == LABEL_DECL
)
1554 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1555 DECL_CONTEXT (new_var
) = current_function_decl
;
1556 insert_decl_map (&ctx
->cb
, var
, new_var
);
1560 while (!is_taskreg_ctx (ctx
))
1565 new_var
= maybe_lookup_decl (var
, ctx
);
1570 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1573 return error_mark_node
;
1577 /* Debugging dumps for parallel regions. */
1578 void dump_omp_region (FILE *, struct omp_region
*, int);
1579 void debug_omp_region (struct omp_region
*);
1580 void debug_all_omp_regions (void);
1582 /* Dump the parallel region tree rooted at REGION. */
1585 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1587 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1588 gimple_code_name
[region
->type
]);
1591 dump_omp_region (file
, region
->inner
, indent
+ 4);
1595 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1596 region
->cont
->index
);
1600 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1601 region
->exit
->index
);
1603 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1606 dump_omp_region (file
, region
->next
, indent
);
1610 debug_omp_region (struct omp_region
*region
)
1612 dump_omp_region (stderr
, region
, 0);
1616 debug_all_omp_regions (void)
1618 dump_omp_region (stderr
, root_omp_region
, 0);
1622 /* Create a new parallel region starting at STMT inside region PARENT. */
1624 static struct omp_region
*
1625 new_omp_region (basic_block bb
, enum gimple_code type
,
1626 struct omp_region
*parent
)
1628 struct omp_region
*region
= XCNEW (struct omp_region
);
1630 region
->outer
= parent
;
1632 region
->type
= type
;
1636 /* This is a nested region. Add it to the list of inner
1637 regions in PARENT. */
1638 region
->next
= parent
->inner
;
1639 parent
->inner
= region
;
1643 /* This is a toplevel region. Add it to the list of toplevel
1644 regions in ROOT_OMP_REGION. */
1645 region
->next
= root_omp_region
;
1646 root_omp_region
= region
;
1652 /* Release the memory associated with the region tree rooted at REGION. */
1655 free_omp_region_1 (struct omp_region
*region
)
1657 struct omp_region
*i
, *n
;
1659 for (i
= region
->inner
; i
; i
= n
)
1662 free_omp_region_1 (i
);
1668 /* Release the memory for the entire omp region tree. */
1671 free_omp_regions (void)
1673 struct omp_region
*r
, *n
;
1674 for (r
= root_omp_region
; r
; r
= n
)
1677 free_omp_region_1 (r
);
1679 root_omp_region
= NULL
;
1683 /* Create a new context, with OUTER_CTX being the surrounding context. */
1685 static omp_context
*
1686 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1688 omp_context
*ctx
= XCNEW (omp_context
);
1690 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1691 (splay_tree_value
) ctx
);
1696 ctx
->outer
= outer_ctx
;
1697 ctx
->cb
= outer_ctx
->cb
;
1698 ctx
->cb
.block
= NULL
;
1699 ctx
->depth
= outer_ctx
->depth
+ 1;
1703 ctx
->cb
.src_fn
= current_function_decl
;
1704 ctx
->cb
.dst_fn
= current_function_decl
;
1705 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1706 gcc_checking_assert (ctx
->cb
.src_node
);
1707 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1708 ctx
->cb
.src_cfun
= cfun
;
1709 ctx
->cb
.copy_decl
= omp_copy_decl
;
1710 ctx
->cb
.eh_lp_nr
= 0;
1711 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1715 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1720 static gimple_seq
maybe_catch_exception (gimple_seq
);
1722 /* Finalize task copyfn. */
1725 finalize_task_copyfn (gomp_task
*task_stmt
)
1727 struct function
*child_cfun
;
1729 gimple_seq seq
= NULL
, new_seq
;
1732 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1733 if (child_fn
== NULL_TREE
)
1736 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1737 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1739 push_cfun (child_cfun
);
1740 bind
= gimplify_body (child_fn
, false);
1741 gimple_seq_add_stmt (&seq
, bind
);
1742 new_seq
= maybe_catch_exception (seq
);
1745 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1747 gimple_seq_add_stmt (&seq
, bind
);
1749 gimple_set_body (child_fn
, seq
);
1752 /* Inform the callgraph about the new function. */
1753 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1754 node
->parallelized_function
= 1;
1755 cgraph_node::add_new_function (child_fn
, false);
1758 /* Destroy a omp_context data structures. Called through the splay tree
1759 value delete callback. */
1762 delete_omp_context (splay_tree_value value
)
1764 omp_context
*ctx
= (omp_context
*) value
;
1766 delete ctx
->cb
.decl_map
;
1769 splay_tree_delete (ctx
->field_map
);
1770 if (ctx
->sfield_map
)
1771 splay_tree_delete (ctx
->sfield_map
);
1773 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1774 it produces corrupt debug information. */
1775 if (ctx
->record_type
)
1778 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1779 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1781 if (ctx
->srecord_type
)
1784 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1785 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1788 if (is_task_ctx (ctx
))
1789 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1794 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1798 fixup_child_record_type (omp_context
*ctx
)
1800 tree f
, type
= ctx
->record_type
;
1802 if (!ctx
->receiver_decl
)
1804 /* ??? It isn't sufficient to just call remap_type here, because
1805 variably_modified_type_p doesn't work the way we expect for
1806 record types. Testing each field for whether it needs remapping
1807 and creating a new record by hand works, however. */
1808 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1809 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1813 tree name
, new_fields
= NULL
;
1815 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1816 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1817 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1818 TYPE_DECL
, name
, type
);
1819 TYPE_NAME (type
) = name
;
1821 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1823 tree new_f
= copy_node (f
);
1824 DECL_CONTEXT (new_f
) = type
;
1825 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1826 DECL_CHAIN (new_f
) = new_fields
;
1827 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1828 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1830 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1834 /* Arrange to be able to look up the receiver field
1835 given the sender field. */
1836 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1837 (splay_tree_value
) new_f
);
1839 TYPE_FIELDS (type
) = nreverse (new_fields
);
1843 /* In a target region we never modify any of the pointers in *.omp_data_i,
1844 so attempt to help the optimizers. */
1845 if (is_gimple_omp_offloaded (ctx
->stmt
))
1846 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1848 TREE_TYPE (ctx
->receiver_decl
)
1849 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1852 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1853 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1857 scan_sharing_clauses (tree clauses
, omp_context
*ctx
,
1858 bool base_pointers_restrict
= false)
1861 bool scan_array_reductions
= false;
1863 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1867 switch (OMP_CLAUSE_CODE (c
))
1869 case OMP_CLAUSE_PRIVATE
:
1870 decl
= OMP_CLAUSE_DECL (c
);
1871 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1873 else if (!is_variable_sized (decl
))
1874 install_var_local (decl
, ctx
);
1877 case OMP_CLAUSE_SHARED
:
1878 decl
= OMP_CLAUSE_DECL (c
);
1879 /* Ignore shared directives in teams construct. */
1880 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1882 /* Global variables don't need to be copied,
1883 the receiver side will use them directly. */
1884 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1885 if (is_global_var (odecl
))
1887 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1890 gcc_assert (is_taskreg_ctx (ctx
));
1891 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1892 || !is_variable_sized (decl
));
1893 /* Global variables don't need to be copied,
1894 the receiver side will use them directly. */
1895 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1897 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1899 use_pointer_for_field (decl
, ctx
);
1902 by_ref
= use_pointer_for_field (decl
, NULL
);
1903 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1904 || TREE_ADDRESSABLE (decl
)
1906 || is_reference (decl
))
1908 by_ref
= use_pointer_for_field (decl
, ctx
);
1909 install_var_field (decl
, by_ref
, 3, ctx
);
1910 install_var_local (decl
, ctx
);
1913 /* We don't need to copy const scalar vars back. */
1914 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1917 case OMP_CLAUSE_REDUCTION
:
1918 decl
= OMP_CLAUSE_DECL (c
);
1919 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1920 && TREE_CODE (decl
) == MEM_REF
)
1922 tree t
= TREE_OPERAND (decl
, 0);
1923 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1924 t
= TREE_OPERAND (t
, 0);
1925 if (TREE_CODE (t
) == INDIRECT_REF
1926 || TREE_CODE (t
) == ADDR_EXPR
)
1927 t
= TREE_OPERAND (t
, 0);
1928 install_var_local (t
, ctx
);
1929 if (is_taskreg_ctx (ctx
)
1930 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1931 && !is_variable_sized (t
))
1933 by_ref
= use_pointer_for_field (t
, ctx
);
1934 install_var_field (t
, by_ref
, 3, ctx
);
1940 case OMP_CLAUSE_LASTPRIVATE
:
1941 /* Let the corresponding firstprivate clause create
1943 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1947 case OMP_CLAUSE_FIRSTPRIVATE
:
1948 case OMP_CLAUSE_LINEAR
:
1949 decl
= OMP_CLAUSE_DECL (c
);
1951 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1952 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1953 && is_gimple_omp_offloaded (ctx
->stmt
))
1955 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1956 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1957 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1958 install_var_field (decl
, true, 3, ctx
);
1960 install_var_field (decl
, false, 3, ctx
);
1962 if (is_variable_sized (decl
))
1964 if (is_task_ctx (ctx
))
1965 install_var_field (decl
, false, 1, ctx
);
1968 else if (is_taskreg_ctx (ctx
))
1971 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1972 by_ref
= use_pointer_for_field (decl
, NULL
);
1974 if (is_task_ctx (ctx
)
1975 && (global
|| by_ref
|| is_reference (decl
)))
1977 install_var_field (decl
, false, 1, ctx
);
1979 install_var_field (decl
, by_ref
, 2, ctx
);
1982 install_var_field (decl
, by_ref
, 3, ctx
);
1984 install_var_local (decl
, ctx
);
1987 case OMP_CLAUSE_USE_DEVICE_PTR
:
1988 decl
= OMP_CLAUSE_DECL (c
);
1989 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1990 install_var_field (decl
, true, 3, ctx
);
1992 install_var_field (decl
, false, 3, ctx
);
1993 if (DECL_SIZE (decl
)
1994 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1996 tree decl2
= DECL_VALUE_EXPR (decl
);
1997 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1998 decl2
= TREE_OPERAND (decl2
, 0);
1999 gcc_assert (DECL_P (decl2
));
2000 install_var_local (decl2
, ctx
);
2002 install_var_local (decl
, ctx
);
2005 case OMP_CLAUSE_IS_DEVICE_PTR
:
2006 decl
= OMP_CLAUSE_DECL (c
);
2009 case OMP_CLAUSE__LOOPTEMP_
:
2010 gcc_assert (is_taskreg_ctx (ctx
));
2011 decl
= OMP_CLAUSE_DECL (c
);
2012 install_var_field (decl
, false, 3, ctx
);
2013 install_var_local (decl
, ctx
);
2016 case OMP_CLAUSE_COPYPRIVATE
:
2017 case OMP_CLAUSE_COPYIN
:
2018 decl
= OMP_CLAUSE_DECL (c
);
2019 by_ref
= use_pointer_for_field (decl
, NULL
);
2020 install_var_field (decl
, by_ref
, 3, ctx
);
2023 case OMP_CLAUSE_DEFAULT
:
2024 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2027 case OMP_CLAUSE_FINAL
:
2029 case OMP_CLAUSE_NUM_THREADS
:
2030 case OMP_CLAUSE_NUM_TEAMS
:
2031 case OMP_CLAUSE_THREAD_LIMIT
:
2032 case OMP_CLAUSE_DEVICE
:
2033 case OMP_CLAUSE_SCHEDULE
:
2034 case OMP_CLAUSE_DIST_SCHEDULE
:
2035 case OMP_CLAUSE_DEPEND
:
2036 case OMP_CLAUSE_PRIORITY
:
2037 case OMP_CLAUSE_GRAINSIZE
:
2038 case OMP_CLAUSE_NUM_TASKS
:
2039 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2040 case OMP_CLAUSE_NUM_GANGS
:
2041 case OMP_CLAUSE_NUM_WORKERS
:
2042 case OMP_CLAUSE_VECTOR_LENGTH
:
2044 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2048 case OMP_CLAUSE_FROM
:
2049 case OMP_CLAUSE_MAP
:
2051 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2052 decl
= OMP_CLAUSE_DECL (c
);
2053 /* Global variables with "omp declare target" attribute
2054 don't need to be copied, the receiver side will use them
2055 directly. However, global variables with "omp declare target link"
2056 attribute need to be copied. */
2057 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2059 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2060 && (OMP_CLAUSE_MAP_KIND (c
)
2061 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2062 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2063 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2064 && varpool_node::get_create (decl
)->offloadable
2065 && !lookup_attribute ("omp declare target link",
2066 DECL_ATTRIBUTES (decl
)))
2068 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2069 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2071 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2072 not offloaded; there is nothing to map for those. */
2073 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2074 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2075 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2078 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2079 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2080 || (OMP_CLAUSE_MAP_KIND (c
)
2081 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2083 if (TREE_CODE (decl
) == COMPONENT_REF
2084 || (TREE_CODE (decl
) == INDIRECT_REF
2085 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2086 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2087 == REFERENCE_TYPE
)))
2089 if (DECL_SIZE (decl
)
2090 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2092 tree decl2
= DECL_VALUE_EXPR (decl
);
2093 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2094 decl2
= TREE_OPERAND (decl2
, 0);
2095 gcc_assert (DECL_P (decl2
));
2096 install_var_local (decl2
, ctx
);
2098 install_var_local (decl
, ctx
);
2103 if (DECL_SIZE (decl
)
2104 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2106 tree decl2
= DECL_VALUE_EXPR (decl
);
2107 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2108 decl2
= TREE_OPERAND (decl2
, 0);
2109 gcc_assert (DECL_P (decl2
));
2110 install_var_field (decl2
, true, 3, ctx
);
2111 install_var_local (decl2
, ctx
);
2112 install_var_local (decl
, ctx
);
2116 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2117 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2118 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2119 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2120 install_var_field (decl
, true, 7, ctx
);
2122 install_var_field (decl
, true, 3, ctx
,
2123 base_pointers_restrict
);
2124 if (is_gimple_omp_offloaded (ctx
->stmt
))
2125 install_var_local (decl
, ctx
);
2130 tree base
= get_base_address (decl
);
2131 tree nc
= OMP_CLAUSE_CHAIN (c
);
2134 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2135 && OMP_CLAUSE_DECL (nc
) == base
2136 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2137 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2139 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2140 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2146 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2147 decl
= OMP_CLAUSE_DECL (c
);
2149 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2150 (splay_tree_key
) decl
));
2152 = build_decl (OMP_CLAUSE_LOCATION (c
),
2153 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2154 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2155 insert_field_into_struct (ctx
->record_type
, field
);
2156 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2157 (splay_tree_value
) field
);
2162 case OMP_CLAUSE__GRIDDIM_
:
2165 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
2166 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
2170 case OMP_CLAUSE_NOWAIT
:
2171 case OMP_CLAUSE_ORDERED
:
2172 case OMP_CLAUSE_COLLAPSE
:
2173 case OMP_CLAUSE_UNTIED
:
2174 case OMP_CLAUSE_MERGEABLE
:
2175 case OMP_CLAUSE_PROC_BIND
:
2176 case OMP_CLAUSE_SAFELEN
:
2177 case OMP_CLAUSE_SIMDLEN
:
2178 case OMP_CLAUSE_THREADS
:
2179 case OMP_CLAUSE_SIMD
:
2180 case OMP_CLAUSE_NOGROUP
:
2181 case OMP_CLAUSE_DEFAULTMAP
:
2182 case OMP_CLAUSE_ASYNC
:
2183 case OMP_CLAUSE_WAIT
:
2184 case OMP_CLAUSE_GANG
:
2185 case OMP_CLAUSE_WORKER
:
2186 case OMP_CLAUSE_VECTOR
:
2187 case OMP_CLAUSE_TILE
:
2188 case OMP_CLAUSE_INDEPENDENT
:
2189 case OMP_CLAUSE_AUTO
:
2190 case OMP_CLAUSE_SEQ
:
2193 case OMP_CLAUSE_ALIGNED
:
2194 decl
= OMP_CLAUSE_DECL (c
);
2195 if (is_global_var (decl
)
2196 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2197 install_var_local (decl
, ctx
);
2200 case OMP_CLAUSE_DEVICE_RESIDENT
:
2201 case OMP_CLAUSE__CACHE_
:
2202 sorry ("Clause not supported yet");
2210 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2212 switch (OMP_CLAUSE_CODE (c
))
2214 case OMP_CLAUSE_LASTPRIVATE
:
2215 /* Let the corresponding firstprivate clause create
2217 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2218 scan_array_reductions
= true;
2219 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2223 case OMP_CLAUSE_FIRSTPRIVATE
:
2224 case OMP_CLAUSE_PRIVATE
:
2225 case OMP_CLAUSE_LINEAR
:
2226 case OMP_CLAUSE_IS_DEVICE_PTR
:
2227 decl
= OMP_CLAUSE_DECL (c
);
2228 if (is_variable_sized (decl
))
2230 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2231 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2232 && is_gimple_omp_offloaded (ctx
->stmt
))
2234 tree decl2
= DECL_VALUE_EXPR (decl
);
2235 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2236 decl2
= TREE_OPERAND (decl2
, 0);
2237 gcc_assert (DECL_P (decl2
));
2238 install_var_local (decl2
, ctx
);
2239 fixup_remapped_decl (decl2
, ctx
, false);
2241 install_var_local (decl
, ctx
);
2243 fixup_remapped_decl (decl
, ctx
,
2244 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2245 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2246 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2247 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2248 scan_array_reductions
= true;
2251 case OMP_CLAUSE_REDUCTION
:
2252 decl
= OMP_CLAUSE_DECL (c
);
2253 if (TREE_CODE (decl
) != MEM_REF
)
2255 if (is_variable_sized (decl
))
2256 install_var_local (decl
, ctx
);
2257 fixup_remapped_decl (decl
, ctx
, false);
2259 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2260 scan_array_reductions
= true;
2263 case OMP_CLAUSE_SHARED
:
2264 /* Ignore shared directives in teams construct. */
2265 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2267 decl
= OMP_CLAUSE_DECL (c
);
2268 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2270 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2272 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2275 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2276 install_var_field (decl
, by_ref
, 11, ctx
);
2279 fixup_remapped_decl (decl
, ctx
, false);
2282 case OMP_CLAUSE_MAP
:
2283 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2285 decl
= OMP_CLAUSE_DECL (c
);
2287 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2288 && (OMP_CLAUSE_MAP_KIND (c
)
2289 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2290 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2291 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2292 && varpool_node::get_create (decl
)->offloadable
)
2296 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2297 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2298 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2299 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2301 tree new_decl
= lookup_decl (decl
, ctx
);
2302 TREE_TYPE (new_decl
)
2303 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2305 else if (DECL_SIZE (decl
)
2306 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2308 tree decl2
= DECL_VALUE_EXPR (decl
);
2309 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2310 decl2
= TREE_OPERAND (decl2
, 0);
2311 gcc_assert (DECL_P (decl2
));
2312 fixup_remapped_decl (decl2
, ctx
, false);
2313 fixup_remapped_decl (decl
, ctx
, true);
2316 fixup_remapped_decl (decl
, ctx
, false);
2320 case OMP_CLAUSE_COPYPRIVATE
:
2321 case OMP_CLAUSE_COPYIN
:
2322 case OMP_CLAUSE_DEFAULT
:
2324 case OMP_CLAUSE_NUM_THREADS
:
2325 case OMP_CLAUSE_NUM_TEAMS
:
2326 case OMP_CLAUSE_THREAD_LIMIT
:
2327 case OMP_CLAUSE_DEVICE
:
2328 case OMP_CLAUSE_SCHEDULE
:
2329 case OMP_CLAUSE_DIST_SCHEDULE
:
2330 case OMP_CLAUSE_NOWAIT
:
2331 case OMP_CLAUSE_ORDERED
:
2332 case OMP_CLAUSE_COLLAPSE
:
2333 case OMP_CLAUSE_UNTIED
:
2334 case OMP_CLAUSE_FINAL
:
2335 case OMP_CLAUSE_MERGEABLE
:
2336 case OMP_CLAUSE_PROC_BIND
:
2337 case OMP_CLAUSE_SAFELEN
:
2338 case OMP_CLAUSE_SIMDLEN
:
2339 case OMP_CLAUSE_ALIGNED
:
2340 case OMP_CLAUSE_DEPEND
:
2341 case OMP_CLAUSE__LOOPTEMP_
:
2343 case OMP_CLAUSE_FROM
:
2344 case OMP_CLAUSE_PRIORITY
:
2345 case OMP_CLAUSE_GRAINSIZE
:
2346 case OMP_CLAUSE_NUM_TASKS
:
2347 case OMP_CLAUSE_THREADS
:
2348 case OMP_CLAUSE_SIMD
:
2349 case OMP_CLAUSE_NOGROUP
:
2350 case OMP_CLAUSE_DEFAULTMAP
:
2351 case OMP_CLAUSE_USE_DEVICE_PTR
:
2352 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2353 case OMP_CLAUSE_ASYNC
:
2354 case OMP_CLAUSE_WAIT
:
2355 case OMP_CLAUSE_NUM_GANGS
:
2356 case OMP_CLAUSE_NUM_WORKERS
:
2357 case OMP_CLAUSE_VECTOR_LENGTH
:
2358 case OMP_CLAUSE_GANG
:
2359 case OMP_CLAUSE_WORKER
:
2360 case OMP_CLAUSE_VECTOR
:
2361 case OMP_CLAUSE_TILE
:
2362 case OMP_CLAUSE_INDEPENDENT
:
2363 case OMP_CLAUSE_AUTO
:
2364 case OMP_CLAUSE_SEQ
:
2365 case OMP_CLAUSE__GRIDDIM_
:
2368 case OMP_CLAUSE_DEVICE_RESIDENT
:
2369 case OMP_CLAUSE__CACHE_
:
2370 sorry ("Clause not supported yet");
2378 gcc_checking_assert (!scan_array_reductions
2379 || !is_gimple_omp_oacc (ctx
->stmt
));
2380 if (scan_array_reductions
)
2381 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2382 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2383 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2385 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2386 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2388 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2389 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2390 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2391 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2392 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2393 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2396 /* Create a new name for omp child function. Returns an identifier. If
2397 IS_CILK_FOR is true then the suffix for the child function is
2401 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2404 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2405 return clone_function_name (current_function_decl
,
2406 task_copy
? "_omp_cpyfn" : "_omp_fn");
2409 /* Returns the type of the induction variable for the child function for
2410 _Cilk_for and the types for _high and _low variables based on TYPE. */
2413 cilk_for_check_loop_diff_type (tree type
)
2415 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2417 if (TYPE_UNSIGNED (type
))
2418 return uint32_type_node
;
2420 return integer_type_node
;
2424 if (TYPE_UNSIGNED (type
))
2425 return uint64_type_node
;
2427 return long_long_integer_type_node
;
2431 /* Build a decl for the omp child function. It'll not contain a body
2432 yet, just the bare decl. */
2435 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2437 tree decl
, type
, name
, t
;
2440 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2441 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2442 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2443 tree cilk_var_type
= NULL_TREE
;
2445 name
= create_omp_child_function_name (task_copy
,
2446 cilk_for_count
!= NULL_TREE
);
2448 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2449 ptr_type_node
, NULL_TREE
);
2450 else if (cilk_for_count
)
2452 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2453 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2454 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2455 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2458 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2460 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2462 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2465 ctx
->cb
.dst_fn
= decl
;
2467 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2469 TREE_STATIC (decl
) = 1;
2470 TREE_USED (decl
) = 1;
2471 DECL_ARTIFICIAL (decl
) = 1;
2472 DECL_IGNORED_P (decl
) = 0;
2473 TREE_PUBLIC (decl
) = 0;
2474 DECL_UNINLINABLE (decl
) = 1;
2475 DECL_EXTERNAL (decl
) = 0;
2476 DECL_CONTEXT (decl
) = NULL_TREE
;
2477 DECL_INITIAL (decl
) = make_node (BLOCK
);
2478 if (cgraph_node::get (current_function_decl
)->offloadable
)
2479 cgraph_node::get_create (decl
)->offloadable
= 1;
2483 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2484 if (is_gimple_omp_offloaded (octx
->stmt
))
2486 cgraph_node::get_create (decl
)->offloadable
= 1;
2487 if (ENABLE_OFFLOADING
)
2488 g
->have_offload
= true;
2494 if (cgraph_node::get_create (decl
)->offloadable
2495 && !lookup_attribute ("omp declare target",
2496 DECL_ATTRIBUTES (current_function_decl
)))
2497 DECL_ATTRIBUTES (decl
)
2498 = tree_cons (get_identifier ("omp target entrypoint"),
2499 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2501 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2502 RESULT_DECL
, NULL_TREE
, void_type_node
);
2503 DECL_ARTIFICIAL (t
) = 1;
2504 DECL_IGNORED_P (t
) = 1;
2505 DECL_CONTEXT (t
) = decl
;
2506 DECL_RESULT (decl
) = t
;
2508 /* _Cilk_for's child function requires two extra parameters called
2509 __low and __high that are set the by Cilk runtime when it calls this
2513 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2514 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2515 DECL_ARTIFICIAL (t
) = 1;
2516 DECL_NAMELESS (t
) = 1;
2517 DECL_ARG_TYPE (t
) = ptr_type_node
;
2518 DECL_CONTEXT (t
) = current_function_decl
;
2520 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2521 DECL_ARGUMENTS (decl
) = t
;
2523 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2524 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2525 DECL_ARTIFICIAL (t
) = 1;
2526 DECL_NAMELESS (t
) = 1;
2527 DECL_ARG_TYPE (t
) = ptr_type_node
;
2528 DECL_CONTEXT (t
) = current_function_decl
;
2530 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2531 DECL_ARGUMENTS (decl
) = t
;
2534 tree data_name
= get_identifier (".omp_data_i");
2535 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2537 DECL_ARTIFICIAL (t
) = 1;
2538 DECL_NAMELESS (t
) = 1;
2539 DECL_ARG_TYPE (t
) = ptr_type_node
;
2540 DECL_CONTEXT (t
) = current_function_decl
;
2542 TREE_READONLY (t
) = 1;
2544 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2545 DECL_ARGUMENTS (decl
) = t
;
2547 ctx
->receiver_decl
= t
;
2550 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2551 PARM_DECL
, get_identifier (".omp_data_o"),
2553 DECL_ARTIFICIAL (t
) = 1;
2554 DECL_NAMELESS (t
) = 1;
2555 DECL_ARG_TYPE (t
) = ptr_type_node
;
2556 DECL_CONTEXT (t
) = current_function_decl
;
2558 TREE_ADDRESSABLE (t
) = 1;
2559 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2560 DECL_ARGUMENTS (decl
) = t
;
2563 /* Allocate memory for the function structure. The call to
2564 allocate_struct_function clobbers CFUN, so we need to restore
2566 push_struct_function (decl
);
2567 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2571 /* Callback for walk_gimple_seq. Check if combined parallel
2572 contains gimple_omp_for_combined_into_p OMP_FOR. */
2575 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2576 bool *handled_ops_p
,
2577 struct walk_stmt_info
*wi
)
2579 gimple
*stmt
= gsi_stmt (*gsi_p
);
2581 *handled_ops_p
= true;
2582 switch (gimple_code (stmt
))
2586 case GIMPLE_OMP_FOR
:
2587 if (gimple_omp_for_combined_into_p (stmt
)
2588 && gimple_omp_for_kind (stmt
)
2589 == *(const enum gf_mask
*) (wi
->info
))
2592 return integer_zero_node
;
2601 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2604 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2605 omp_context
*outer_ctx
)
2607 struct walk_stmt_info wi
;
2609 memset (&wi
, 0, sizeof (wi
));
2611 wi
.info
= (void *) &msk
;
2612 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2613 if (wi
.info
!= (void *) &msk
)
2615 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2616 struct omp_for_data fd
;
2617 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2618 /* We need two temporaries with fd.loop.v type (istart/iend)
2619 and then (fd.collapse - 1) temporaries with the same
2620 type for count2 ... countN-1 vars if not constant. */
2621 size_t count
= 2, i
;
2622 tree type
= fd
.iter_type
;
2624 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2626 count
+= fd
.collapse
- 1;
2627 /* If there are lastprivate clauses on the inner
2628 GIMPLE_OMP_FOR, add one more temporaries for the total number
2629 of iterations (product of count1 ... countN-1). */
2630 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2631 OMP_CLAUSE_LASTPRIVATE
))
2633 else if (msk
== GF_OMP_FOR_KIND_FOR
2634 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2635 OMP_CLAUSE_LASTPRIVATE
))
2638 for (i
= 0; i
< count
; i
++)
2640 tree temp
= create_tmp_var (type
);
2641 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2642 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2643 OMP_CLAUSE_DECL (c
) = temp
;
2644 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2645 gimple_omp_taskreg_set_clauses (stmt
, c
);
2650 /* Scan an OpenMP parallel directive. */
2653 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2657 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2659 /* Ignore parallel directives with empty bodies, unless there
2660 are copyin clauses. */
2662 && empty_body_p (gimple_omp_body (stmt
))
2663 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2664 OMP_CLAUSE_COPYIN
) == NULL
)
2666 gsi_replace (gsi
, gimple_build_nop (), false);
2670 if (gimple_omp_parallel_combined_p (stmt
))
2671 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2673 ctx
= new_omp_context (stmt
, outer_ctx
);
2674 taskreg_contexts
.safe_push (ctx
);
2675 if (taskreg_nesting_level
> 1)
2676 ctx
->is_nested
= true;
2677 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2678 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2679 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2680 name
= create_tmp_var_name (".omp_data_s");
2681 name
= build_decl (gimple_location (stmt
),
2682 TYPE_DECL
, name
, ctx
->record_type
);
2683 DECL_ARTIFICIAL (name
) = 1;
2684 DECL_NAMELESS (name
) = 1;
2685 TYPE_NAME (ctx
->record_type
) = name
;
2686 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2687 if (!gimple_omp_parallel_grid_phony (stmt
))
2689 create_omp_child_function (ctx
, false);
2690 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2693 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2694 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2696 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2697 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2700 /* Scan an OpenMP task directive. */
2703 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2707 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2709 /* Ignore task directives with empty bodies. */
2711 && empty_body_p (gimple_omp_body (stmt
)))
2713 gsi_replace (gsi
, gimple_build_nop (), false);
2717 if (gimple_omp_task_taskloop_p (stmt
))
2718 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2720 ctx
= new_omp_context (stmt
, outer_ctx
);
2721 taskreg_contexts
.safe_push (ctx
);
2722 if (taskreg_nesting_level
> 1)
2723 ctx
->is_nested
= true;
2724 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2725 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2726 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2727 name
= create_tmp_var_name (".omp_data_s");
2728 name
= build_decl (gimple_location (stmt
),
2729 TYPE_DECL
, name
, ctx
->record_type
);
2730 DECL_ARTIFICIAL (name
) = 1;
2731 DECL_NAMELESS (name
) = 1;
2732 TYPE_NAME (ctx
->record_type
) = name
;
2733 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2734 create_omp_child_function (ctx
, false);
2735 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2737 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2739 if (ctx
->srecord_type
)
2741 name
= create_tmp_var_name (".omp_data_a");
2742 name
= build_decl (gimple_location (stmt
),
2743 TYPE_DECL
, name
, ctx
->srecord_type
);
2744 DECL_ARTIFICIAL (name
) = 1;
2745 DECL_NAMELESS (name
) = 1;
2746 TYPE_NAME (ctx
->srecord_type
) = name
;
2747 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2748 create_omp_child_function (ctx
, true);
2751 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2753 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2755 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2756 t
= build_int_cst (long_integer_type_node
, 0);
2757 gimple_omp_task_set_arg_size (stmt
, t
);
2758 t
= build_int_cst (long_integer_type_node
, 1);
2759 gimple_omp_task_set_arg_align (stmt
, t
);
2764 /* If any decls have been made addressable during scan_omp,
2765 adjust their fields if needed, and layout record types
2766 of parallel/task constructs. */
2769 finish_taskreg_scan (omp_context
*ctx
)
2771 if (ctx
->record_type
== NULL_TREE
)
2774 /* If any task_shared_vars were needed, verify all
2775 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2776 statements if use_pointer_for_field hasn't changed
2777 because of that. If it did, update field types now. */
2778 if (task_shared_vars
)
2782 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2783 c
; c
= OMP_CLAUSE_CHAIN (c
))
2784 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2785 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2787 tree decl
= OMP_CLAUSE_DECL (c
);
2789 /* Global variables don't need to be copied,
2790 the receiver side will use them directly. */
2791 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2793 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2794 || !use_pointer_for_field (decl
, ctx
))
2796 tree field
= lookup_field (decl
, ctx
);
2797 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2798 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2800 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2801 TREE_THIS_VOLATILE (field
) = 0;
2802 DECL_USER_ALIGN (field
) = 0;
2803 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2804 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2805 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2806 if (ctx
->srecord_type
)
2808 tree sfield
= lookup_sfield (decl
, ctx
);
2809 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2810 TREE_THIS_VOLATILE (sfield
) = 0;
2811 DECL_USER_ALIGN (sfield
) = 0;
2812 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2813 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2814 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2819 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2821 layout_type (ctx
->record_type
);
2822 fixup_child_record_type (ctx
);
2826 location_t loc
= gimple_location (ctx
->stmt
);
2827 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2828 /* Move VLA fields to the end. */
2829 p
= &TYPE_FIELDS (ctx
->record_type
);
2831 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2832 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2835 *p
= TREE_CHAIN (*p
);
2836 TREE_CHAIN (*q
) = NULL_TREE
;
2837 q
= &TREE_CHAIN (*q
);
2840 p
= &DECL_CHAIN (*p
);
2842 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2844 /* Move fields corresponding to first and second _looptemp_
2845 clause first. There are filled by GOMP_taskloop
2846 and thus need to be in specific positions. */
2847 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2848 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2849 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2850 OMP_CLAUSE__LOOPTEMP_
);
2851 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2852 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2853 p
= &TYPE_FIELDS (ctx
->record_type
);
2855 if (*p
== f1
|| *p
== f2
)
2856 *p
= DECL_CHAIN (*p
);
2858 p
= &DECL_CHAIN (*p
);
2859 DECL_CHAIN (f1
) = f2
;
2860 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2861 TYPE_FIELDS (ctx
->record_type
) = f1
;
2862 if (ctx
->srecord_type
)
2864 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2865 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2866 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2868 if (*p
== f1
|| *p
== f2
)
2869 *p
= DECL_CHAIN (*p
);
2871 p
= &DECL_CHAIN (*p
);
2872 DECL_CHAIN (f1
) = f2
;
2873 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2874 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2877 layout_type (ctx
->record_type
);
2878 fixup_child_record_type (ctx
);
2879 if (ctx
->srecord_type
)
2880 layout_type (ctx
->srecord_type
);
2881 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2882 TYPE_SIZE_UNIT (ctx
->record_type
));
2883 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2884 t
= build_int_cst (long_integer_type_node
,
2885 TYPE_ALIGN_UNIT (ctx
->record_type
));
2886 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2890 /* Find the enclosing offload context. */
2892 static omp_context
*
2893 enclosing_target_ctx (omp_context
*ctx
)
2895 for (; ctx
; ctx
= ctx
->outer
)
2896 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2902 /* Return true if ctx is part of an oacc kernels region. */
2905 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2907 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2909 gimple
*stmt
= ctx
->stmt
;
2910 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2911 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2918 /* Check the parallelism clauses inside a kernels regions.
2919 Until kernels handling moves to use the same loop indirection
2920 scheme as parallel, we need to do this checking early. */
2923 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2925 bool checking
= true;
2926 unsigned outer_mask
= 0;
2927 unsigned this_mask
= 0;
2928 bool has_seq
= false, has_auto
= false;
2931 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2935 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2937 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2940 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2942 switch (OMP_CLAUSE_CODE (c
))
2944 case OMP_CLAUSE_GANG
:
2945 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2947 case OMP_CLAUSE_WORKER
:
2948 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2950 case OMP_CLAUSE_VECTOR
:
2951 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2953 case OMP_CLAUSE_SEQ
:
2956 case OMP_CLAUSE_AUTO
:
2966 if (has_seq
&& (this_mask
|| has_auto
))
2967 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2968 " OpenACC loop specifiers");
2969 else if (has_auto
&& this_mask
)
2970 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2971 " OpenACC loop specifiers");
2973 if (this_mask
& outer_mask
)
2974 error_at (gimple_location (stmt
), "inner loop uses same"
2975 " OpenACC parallelism as containing loop");
2978 return outer_mask
| this_mask
;
2981 /* Scan a GIMPLE_OMP_FOR. */
2984 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2988 tree clauses
= gimple_omp_for_clauses (stmt
);
2990 ctx
= new_omp_context (stmt
, outer_ctx
);
2992 if (is_gimple_omp_oacc (stmt
))
2994 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2996 if (!tgt
|| is_oacc_parallel (tgt
))
2997 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2999 char const *check
= NULL
;
3001 switch (OMP_CLAUSE_CODE (c
))
3003 case OMP_CLAUSE_GANG
:
3007 case OMP_CLAUSE_WORKER
:
3011 case OMP_CLAUSE_VECTOR
:
3019 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3020 error_at (gimple_location (stmt
),
3021 "argument not permitted on %qs clause in"
3022 " OpenACC %<parallel%>", check
);
3025 if (tgt
&& is_oacc_kernels (tgt
))
3027 /* Strip out reductions, as they are not handled yet. */
3028 tree
*prev_ptr
= &clauses
;
3030 while (tree probe
= *prev_ptr
)
3032 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3034 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3035 *prev_ptr
= *next_ptr
;
3037 prev_ptr
= next_ptr
;
3040 gimple_omp_for_set_clauses (stmt
, clauses
);
3041 check_oacc_kernel_gwv (stmt
, ctx
);
3045 scan_sharing_clauses (clauses
, ctx
);
3047 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3048 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3050 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3051 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3052 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3053 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3055 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3058 /* Scan an OpenMP sections directive. */
3061 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3065 ctx
= new_omp_context (stmt
, outer_ctx
);
3066 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3067 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3070 /* Scan an OpenMP single directive. */
3073 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3078 ctx
= new_omp_context (stmt
, outer_ctx
);
3079 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3080 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3081 name
= create_tmp_var_name (".omp_copy_s");
3082 name
= build_decl (gimple_location (stmt
),
3083 TYPE_DECL
, name
, ctx
->record_type
);
3084 TYPE_NAME (ctx
->record_type
) = name
;
3086 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3087 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3089 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3090 ctx
->record_type
= NULL
;
3092 layout_type (ctx
->record_type
);
3095 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3096 used in the corresponding offloaded function are restrict. */
3099 omp_target_base_pointers_restrict_p (tree clauses
)
3101 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3103 if (flag_openacc
== 0)
3106 /* I. Basic example:
3110 unsigned int a[2], b[2];
3112 #pragma acc kernels \
3121 After gimplification, we have:
3123 #pragma omp target oacc_kernels \
3124 map(force_from:a [len: 8]) \
3125 map(force_from:b [len: 8])
3131 Because both mappings have the force prefix, we know that they will be
3132 allocated when calling the corresponding offloaded function, which means we
3133 can mark the base pointers for a and b in the offloaded function as
3137 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3139 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
3142 switch (OMP_CLAUSE_MAP_KIND (c
))
3144 case GOMP_MAP_FORCE_ALLOC
:
3145 case GOMP_MAP_FORCE_TO
:
3146 case GOMP_MAP_FORCE_FROM
:
3147 case GOMP_MAP_FORCE_TOFROM
:
3157 /* Scan a GIMPLE_OMP_TARGET. */
3160 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3164 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3165 tree clauses
= gimple_omp_target_clauses (stmt
);
3167 ctx
= new_omp_context (stmt
, outer_ctx
);
3168 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3169 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3170 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3171 name
= create_tmp_var_name (".omp_data_t");
3172 name
= build_decl (gimple_location (stmt
),
3173 TYPE_DECL
, name
, ctx
->record_type
);
3174 DECL_ARTIFICIAL (name
) = 1;
3175 DECL_NAMELESS (name
) = 1;
3176 TYPE_NAME (ctx
->record_type
) = name
;
3177 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3179 bool base_pointers_restrict
= false;
3182 create_omp_child_function (ctx
, false);
3183 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3185 base_pointers_restrict
= omp_target_base_pointers_restrict_p (clauses
);
3186 if (base_pointers_restrict
3187 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3189 "Base pointers in offloaded function are restrict\n");
3192 scan_sharing_clauses (clauses
, ctx
, base_pointers_restrict
);
3193 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3195 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3196 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3199 TYPE_FIELDS (ctx
->record_type
)
3200 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3203 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3204 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3206 field
= DECL_CHAIN (field
))
3207 gcc_assert (DECL_ALIGN (field
) == align
);
3209 layout_type (ctx
->record_type
);
3211 fixup_child_record_type (ctx
);
3215 /* Scan an OpenMP teams directive. */
3218 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3220 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3221 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3222 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3225 /* Check nesting restrictions. */
3227 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3231 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
3232 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3233 the original copy of its contents. */
3236 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3237 inside an OpenACC CTX. */
3238 if (!(is_gimple_omp (stmt
)
3239 && is_gimple_omp_oacc (stmt
)))
3241 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3242 if (is_gimple_omp (octx
->stmt
)
3243 && is_gimple_omp_oacc (octx
->stmt
)
3244 /* Except for atomic codes that we share with OpenMP. */
3245 && ! (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3246 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3248 error_at (gimple_location (stmt
),
3249 "non-OpenACC construct inside of OpenACC region");
3256 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3257 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3260 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3262 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3263 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3265 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3266 && (ctx
->outer
== NULL
3267 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3268 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3269 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3270 != GF_OMP_FOR_KIND_FOR
)
3271 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3273 error_at (gimple_location (stmt
),
3274 "%<ordered simd threads%> must be closely "
3275 "nested inside of %<for simd%> region");
3281 error_at (gimple_location (stmt
),
3282 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3283 " may not be nested inside %<simd%> region");
3286 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3288 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3289 || (gimple_omp_for_kind (stmt
)
3290 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3291 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3293 error_at (gimple_location (stmt
),
3294 "only %<distribute%> or %<parallel%> regions are "
3295 "allowed to be strictly nested inside %<teams%> "
3301 switch (gimple_code (stmt
))
3303 case GIMPLE_OMP_FOR
:
3304 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3306 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3308 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3310 error_at (gimple_location (stmt
),
3311 "%<distribute%> region must be strictly nested "
3312 "inside %<teams%> construct");
3317 /* We split taskloop into task and nested taskloop in it. */
3318 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3320 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3325 switch (gimple_code (ctx
->stmt
))
3327 case GIMPLE_OMP_FOR
:
3328 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3329 == GF_OMP_FOR_KIND_OACC_LOOP
);
3332 case GIMPLE_OMP_TARGET
:
3333 switch (gimple_omp_target_kind (ctx
->stmt
))
3335 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3336 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3347 else if (get_oacc_fn_attrib (current_function_decl
))
3351 error_at (gimple_location (stmt
),
3352 "OpenACC loop directive must be associated with"
3353 " an OpenACC compute region");
3359 if (is_gimple_call (stmt
)
3360 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3361 == BUILT_IN_GOMP_CANCEL
3362 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3363 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3365 const char *bad
= NULL
;
3366 const char *kind
= NULL
;
3367 const char *construct
3368 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3369 == BUILT_IN_GOMP_CANCEL
)
3370 ? "#pragma omp cancel"
3371 : "#pragma omp cancellation point";
3374 error_at (gimple_location (stmt
), "orphaned %qs construct",
3378 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3379 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3383 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3384 bad
= "#pragma omp parallel";
3385 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3386 == BUILT_IN_GOMP_CANCEL
3387 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3388 ctx
->cancellable
= true;
3392 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3393 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3394 bad
= "#pragma omp for";
3395 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3396 == BUILT_IN_GOMP_CANCEL
3397 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3399 ctx
->cancellable
= true;
3400 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3402 warning_at (gimple_location (stmt
), 0,
3403 "%<#pragma omp cancel for%> inside "
3404 "%<nowait%> for construct");
3405 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3406 OMP_CLAUSE_ORDERED
))
3407 warning_at (gimple_location (stmt
), 0,
3408 "%<#pragma omp cancel for%> inside "
3409 "%<ordered%> for construct");
3414 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3415 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3416 bad
= "#pragma omp sections";
3417 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3418 == BUILT_IN_GOMP_CANCEL
3419 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3421 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3423 ctx
->cancellable
= true;
3424 if (find_omp_clause (gimple_omp_sections_clauses
3427 warning_at (gimple_location (stmt
), 0,
3428 "%<#pragma omp cancel sections%> inside "
3429 "%<nowait%> sections construct");
3433 gcc_assert (ctx
->outer
3434 && gimple_code (ctx
->outer
->stmt
)
3435 == GIMPLE_OMP_SECTIONS
);
3436 ctx
->outer
->cancellable
= true;
3437 if (find_omp_clause (gimple_omp_sections_clauses
3440 warning_at (gimple_location (stmt
), 0,
3441 "%<#pragma omp cancel sections%> inside "
3442 "%<nowait%> sections construct");
3448 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3449 bad
= "#pragma omp task";
3452 for (omp_context
*octx
= ctx
->outer
;
3453 octx
; octx
= octx
->outer
)
3455 switch (gimple_code (octx
->stmt
))
3457 case GIMPLE_OMP_TASKGROUP
:
3459 case GIMPLE_OMP_TARGET
:
3460 if (gimple_omp_target_kind (octx
->stmt
)
3461 != GF_OMP_TARGET_KIND_REGION
)
3464 case GIMPLE_OMP_PARALLEL
:
3465 case GIMPLE_OMP_TEAMS
:
3466 error_at (gimple_location (stmt
),
3467 "%<%s taskgroup%> construct not closely "
3468 "nested inside of %<taskgroup%> region",
3476 ctx
->cancellable
= true;
3481 error_at (gimple_location (stmt
), "invalid arguments");
3486 error_at (gimple_location (stmt
),
3487 "%<%s %s%> construct not closely nested inside of %qs",
3488 construct
, kind
, bad
);
3493 case GIMPLE_OMP_SECTIONS
:
3494 case GIMPLE_OMP_SINGLE
:
3495 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3496 switch (gimple_code (ctx
->stmt
))
3498 case GIMPLE_OMP_FOR
:
3499 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3500 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3503 case GIMPLE_OMP_SECTIONS
:
3504 case GIMPLE_OMP_SINGLE
:
3505 case GIMPLE_OMP_ORDERED
:
3506 case GIMPLE_OMP_MASTER
:
3507 case GIMPLE_OMP_TASK
:
3508 case GIMPLE_OMP_CRITICAL
:
3509 if (is_gimple_call (stmt
))
3511 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3512 != BUILT_IN_GOMP_BARRIER
)
3514 error_at (gimple_location (stmt
),
3515 "barrier region may not be closely nested inside "
3516 "of work-sharing, %<critical%>, %<ordered%>, "
3517 "%<master%>, explicit %<task%> or %<taskloop%> "
3521 error_at (gimple_location (stmt
),
3522 "work-sharing region may not be closely nested inside "
3523 "of work-sharing, %<critical%>, %<ordered%>, "
3524 "%<master%>, explicit %<task%> or %<taskloop%> region");
3526 case GIMPLE_OMP_PARALLEL
:
3527 case GIMPLE_OMP_TEAMS
:
3529 case GIMPLE_OMP_TARGET
:
3530 if (gimple_omp_target_kind (ctx
->stmt
)
3531 == GF_OMP_TARGET_KIND_REGION
)
3538 case GIMPLE_OMP_MASTER
:
3539 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3540 switch (gimple_code (ctx
->stmt
))
3542 case GIMPLE_OMP_FOR
:
3543 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3544 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3547 case GIMPLE_OMP_SECTIONS
:
3548 case GIMPLE_OMP_SINGLE
:
3549 case GIMPLE_OMP_TASK
:
3550 error_at (gimple_location (stmt
),
3551 "%<master%> region may not be closely nested inside "
3552 "of work-sharing, explicit %<task%> or %<taskloop%> "
3555 case GIMPLE_OMP_PARALLEL
:
3556 case GIMPLE_OMP_TEAMS
:
3558 case GIMPLE_OMP_TARGET
:
3559 if (gimple_omp_target_kind (ctx
->stmt
)
3560 == GF_OMP_TARGET_KIND_REGION
)
3567 case GIMPLE_OMP_TASK
:
3568 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3569 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3570 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3571 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3573 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3574 error_at (OMP_CLAUSE_LOCATION (c
),
3575 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3576 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3580 case GIMPLE_OMP_ORDERED
:
3581 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3582 c
; c
= OMP_CLAUSE_CHAIN (c
))
3584 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3586 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3587 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3590 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3591 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3592 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3595 /* Look for containing ordered(N) loop. */
3597 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3599 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3600 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3602 error_at (OMP_CLAUSE_LOCATION (c
),
3603 "%<ordered%> construct with %<depend%> clause "
3604 "must be closely nested inside an %<ordered%> "
3608 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3610 error_at (OMP_CLAUSE_LOCATION (c
),
3611 "%<ordered%> construct with %<depend%> clause "
3612 "must be closely nested inside a loop with "
3613 "%<ordered%> clause with a parameter");
3619 error_at (OMP_CLAUSE_LOCATION (c
),
3620 "invalid depend kind in omp %<ordered%> %<depend%>");
3624 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3625 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3627 /* ordered simd must be closely nested inside of simd region,
3628 and simd region must not encounter constructs other than
3629 ordered simd, therefore ordered simd may be either orphaned,
3630 or ctx->stmt must be simd. The latter case is handled already
3634 error_at (gimple_location (stmt
),
3635 "%<ordered%> %<simd%> must be closely nested inside "
3640 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3641 switch (gimple_code (ctx
->stmt
))
3643 case GIMPLE_OMP_CRITICAL
:
3644 case GIMPLE_OMP_TASK
:
3645 case GIMPLE_OMP_ORDERED
:
3646 ordered_in_taskloop
:
3647 error_at (gimple_location (stmt
),
3648 "%<ordered%> region may not be closely nested inside "
3649 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3650 "%<taskloop%> region");
3652 case GIMPLE_OMP_FOR
:
3653 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3654 goto ordered_in_taskloop
;
3655 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3656 OMP_CLAUSE_ORDERED
) == NULL
)
3658 error_at (gimple_location (stmt
),
3659 "%<ordered%> region must be closely nested inside "
3660 "a loop region with an %<ordered%> clause");
3664 case GIMPLE_OMP_TARGET
:
3665 if (gimple_omp_target_kind (ctx
->stmt
)
3666 != GF_OMP_TARGET_KIND_REGION
)
3669 case GIMPLE_OMP_PARALLEL
:
3670 case GIMPLE_OMP_TEAMS
:
3671 error_at (gimple_location (stmt
),
3672 "%<ordered%> region must be closely nested inside "
3673 "a loop region with an %<ordered%> clause");
3679 case GIMPLE_OMP_CRITICAL
:
3682 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3683 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3684 if (gomp_critical
*other_crit
3685 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3686 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3688 error_at (gimple_location (stmt
),
3689 "%<critical%> region may not be nested inside "
3690 "a %<critical%> region with the same name");
3695 case GIMPLE_OMP_TEAMS
:
3697 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3698 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3700 error_at (gimple_location (stmt
),
3701 "%<teams%> construct not closely nested inside of "
3702 "%<target%> construct");
3706 case GIMPLE_OMP_TARGET
:
3707 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3708 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3709 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3710 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3712 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3713 error_at (OMP_CLAUSE_LOCATION (c
),
3714 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3715 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3718 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3720 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3722 if (is_gimple_omp (stmt
)
3723 && is_gimple_omp_oacc (stmt
)
3724 && is_gimple_omp (ctx
->stmt
))
3726 error_at (gimple_location (stmt
),
3727 "OpenACC construct inside of non-OpenACC region");
3733 const char *stmt_name
, *ctx_stmt_name
;
3734 switch (gimple_omp_target_kind (stmt
))
3736 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3737 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3738 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3739 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3740 stmt_name
= "target enter data"; break;
3741 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3742 stmt_name
= "target exit data"; break;
3743 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3744 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3745 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3746 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3747 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3748 stmt_name
= "enter/exit data"; break;
3749 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3751 default: gcc_unreachable ();
3753 switch (gimple_omp_target_kind (ctx
->stmt
))
3755 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3756 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3757 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3758 ctx_stmt_name
= "parallel"; break;
3759 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3760 ctx_stmt_name
= "kernels"; break;
3761 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3762 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3763 ctx_stmt_name
= "host_data"; break;
3764 default: gcc_unreachable ();
3767 /* OpenACC/OpenMP mismatch? */
3768 if (is_gimple_omp_oacc (stmt
)
3769 != is_gimple_omp_oacc (ctx
->stmt
))
3771 error_at (gimple_location (stmt
),
3772 "%s %qs construct inside of %s %qs region",
3773 (is_gimple_omp_oacc (stmt
)
3774 ? "OpenACC" : "OpenMP"), stmt_name
,
3775 (is_gimple_omp_oacc (ctx
->stmt
)
3776 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3779 if (is_gimple_omp_offloaded (ctx
->stmt
))
3781 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3782 if (is_gimple_omp_oacc (ctx
->stmt
))
3784 error_at (gimple_location (stmt
),
3785 "%qs construct inside of %qs region",
3786 stmt_name
, ctx_stmt_name
);
3791 warning_at (gimple_location (stmt
), 0,
3792 "%qs construct inside of %qs region",
3793 stmt_name
, ctx_stmt_name
);
3805 /* Helper function scan_omp.
3807 Callback for walk_tree or operators in walk_gimple_stmt used to
3808 scan for OMP directives in TP. */
3811 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3813 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3814 omp_context
*ctx
= (omp_context
*) wi
->info
;
3817 switch (TREE_CODE (t
))
3825 tree repl
= remap_decl (t
, &ctx
->cb
);
3826 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3832 if (ctx
&& TYPE_P (t
))
3833 *tp
= remap_type (t
, &ctx
->cb
);
3834 else if (!DECL_P (t
))
3839 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3840 if (tem
!= TREE_TYPE (t
))
3842 if (TREE_CODE (t
) == INTEGER_CST
)
3843 *tp
= wide_int_to_tree (tem
, t
);
3845 TREE_TYPE (t
) = tem
;
3855 /* Return true if FNDECL is a setjmp or a longjmp. */
3858 setjmp_or_longjmp_p (const_tree fndecl
)
3860 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3861 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3862 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3865 tree declname
= DECL_NAME (fndecl
);
3868 const char *name
= IDENTIFIER_POINTER (declname
);
3869 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3873 /* Helper function for scan_omp.
3875 Callback for walk_gimple_stmt used to scan for OMP directives in
3876 the current statement in GSI. */
3879 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3880 struct walk_stmt_info
*wi
)
3882 gimple
*stmt
= gsi_stmt (*gsi
);
3883 omp_context
*ctx
= (omp_context
*) wi
->info
;
3885 if (gimple_has_location (stmt
))
3886 input_location
= gimple_location (stmt
);
3888 /* Check the nesting restrictions. */
3889 bool remove
= false;
3890 if (is_gimple_omp (stmt
))
3891 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3892 else if (is_gimple_call (stmt
))
3894 tree fndecl
= gimple_call_fndecl (stmt
);
3897 if (setjmp_or_longjmp_p (fndecl
)
3899 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3900 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3903 error_at (gimple_location (stmt
),
3904 "setjmp/longjmp inside simd construct");
3906 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3907 switch (DECL_FUNCTION_CODE (fndecl
))
3909 case BUILT_IN_GOMP_BARRIER
:
3910 case BUILT_IN_GOMP_CANCEL
:
3911 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3912 case BUILT_IN_GOMP_TASKYIELD
:
3913 case BUILT_IN_GOMP_TASKWAIT
:
3914 case BUILT_IN_GOMP_TASKGROUP_START
:
3915 case BUILT_IN_GOMP_TASKGROUP_END
:
3916 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3925 stmt
= gimple_build_nop ();
3926 gsi_replace (gsi
, stmt
, false);
3929 *handled_ops_p
= true;
3931 switch (gimple_code (stmt
))
3933 case GIMPLE_OMP_PARALLEL
:
3934 taskreg_nesting_level
++;
3935 scan_omp_parallel (gsi
, ctx
);
3936 taskreg_nesting_level
--;
3939 case GIMPLE_OMP_TASK
:
3940 taskreg_nesting_level
++;
3941 scan_omp_task (gsi
, ctx
);
3942 taskreg_nesting_level
--;
3945 case GIMPLE_OMP_FOR
:
3946 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3949 case GIMPLE_OMP_SECTIONS
:
3950 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3953 case GIMPLE_OMP_SINGLE
:
3954 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3957 case GIMPLE_OMP_SECTION
:
3958 case GIMPLE_OMP_MASTER
:
3959 case GIMPLE_OMP_TASKGROUP
:
3960 case GIMPLE_OMP_ORDERED
:
3961 case GIMPLE_OMP_CRITICAL
:
3962 case GIMPLE_OMP_GRID_BODY
:
3963 ctx
= new_omp_context (stmt
, ctx
);
3964 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3967 case GIMPLE_OMP_TARGET
:
3968 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3971 case GIMPLE_OMP_TEAMS
:
3972 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3979 *handled_ops_p
= false;
3981 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3983 var
= DECL_CHAIN (var
))
3984 insert_decl_map (&ctx
->cb
, var
, var
);
3988 *handled_ops_p
= false;
3996 /* Scan all the statements starting at the current statement. CTX
3997 contains context information about the OMP directives and
3998 clauses found during the scan. */
4001 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
4003 location_t saved_location
;
4004 struct walk_stmt_info wi
;
4006 memset (&wi
, 0, sizeof (wi
));
4008 wi
.want_locations
= true;
4010 saved_location
= input_location
;
4011 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
4012 input_location
= saved_location
;
4015 /* Re-gimplification and code generation routines. */
4017 /* Build a call to GOMP_barrier. */
4020 build_omp_barrier (tree lhs
)
4022 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
4023 : BUILT_IN_GOMP_BARRIER
);
4024 gcall
*g
= gimple_build_call (fndecl
, 0);
4026 gimple_call_set_lhs (g
, lhs
);
4030 /* If a context was created for STMT when it was scanned, return it. */
4032 static omp_context
*
4033 maybe_lookup_ctx (gimple
*stmt
)
4036 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
4037 return n
? (omp_context
*) n
->value
: NULL
;
4041 /* Find the mapping for DECL in CTX or the immediately enclosing
4042 context that has a mapping for DECL.
4044 If CTX is a nested parallel directive, we may have to use the decl
4045 mappings created in CTX's parent context. Suppose that we have the
4046 following parallel nesting (variable UIDs showed for clarity):
4049 #omp parallel shared(iD.1562) -> outer parallel
4050 iD.1562 = iD.1562 + 1;
4052 #omp parallel shared (iD.1562) -> inner parallel
4053 iD.1562 = iD.1562 - 1;
4055 Each parallel structure will create a distinct .omp_data_s structure
4056 for copying iD.1562 in/out of the directive:
4058 outer parallel .omp_data_s.1.i -> iD.1562
4059 inner parallel .omp_data_s.2.i -> iD.1562
4061 A shared variable mapping will produce a copy-out operation before
4062 the parallel directive and a copy-in operation after it. So, in
4063 this case we would have:
4066 .omp_data_o.1.i = iD.1562;
4067 #omp parallel shared(iD.1562) -> outer parallel
4068 .omp_data_i.1 = &.omp_data_o.1
4069 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4071 .omp_data_o.2.i = iD.1562; -> **
4072 #omp parallel shared(iD.1562) -> inner parallel
4073 .omp_data_i.2 = &.omp_data_o.2
4074 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4077 ** This is a problem. The symbol iD.1562 cannot be referenced
4078 inside the body of the outer parallel region. But since we are
4079 emitting this copy operation while expanding the inner parallel
4080 directive, we need to access the CTX structure of the outer
4081 parallel directive to get the correct mapping:
4083 .omp_data_o.2.i = .omp_data_i.1->i
4085 Since there may be other workshare or parallel directives enclosing
4086 the parallel directive, it may be necessary to walk up the context
4087 parent chain. This is not a problem in general because nested
4088 parallelism happens only rarely. */
4091 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4096 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4097 t
= maybe_lookup_decl (decl
, up
);
4099 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
4101 return t
? t
: decl
;
4105 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4106 in outer contexts. */
4109 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4114 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4115 t
= maybe_lookup_decl (decl
, up
);
4117 return t
? t
: decl
;
4121 /* Construct the initialization value for reduction operation OP. */
4124 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4133 case TRUTH_ORIF_EXPR
:
4134 case TRUTH_XOR_EXPR
:
4136 return build_zero_cst (type
);
4139 case TRUTH_AND_EXPR
:
4140 case TRUTH_ANDIF_EXPR
:
4142 return fold_convert_loc (loc
, type
, integer_one_node
);
4145 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4148 if (SCALAR_FLOAT_TYPE_P (type
))
4150 REAL_VALUE_TYPE max
, min
;
4151 if (HONOR_INFINITIES (type
))
4154 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4157 real_maxval (&min
, 1, TYPE_MODE (type
));
4158 return build_real (type
, min
);
4160 else if (POINTER_TYPE_P (type
))
4163 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4164 return wide_int_to_tree (type
, min
);
4168 gcc_assert (INTEGRAL_TYPE_P (type
));
4169 return TYPE_MIN_VALUE (type
);
4173 if (SCALAR_FLOAT_TYPE_P (type
))
4175 REAL_VALUE_TYPE max
;
4176 if (HONOR_INFINITIES (type
))
4179 real_maxval (&max
, 0, TYPE_MODE (type
));
4180 return build_real (type
, max
);
4182 else if (POINTER_TYPE_P (type
))
4185 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4186 return wide_int_to_tree (type
, max
);
4190 gcc_assert (INTEGRAL_TYPE_P (type
));
4191 return TYPE_MAX_VALUE (type
);
4199 /* Construct the initialization value for reduction CLAUSE. */
4202 omp_reduction_init (tree clause
, tree type
)
4204 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4205 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4208 /* Return alignment to be assumed for var in CLAUSE, which should be
4209 OMP_CLAUSE_ALIGNED. */
4212 omp_clause_aligned_alignment (tree clause
)
4214 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4215 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4217 /* Otherwise return implementation defined alignment. */
4218 unsigned int al
= 1;
4219 machine_mode mode
, vmode
;
4220 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4222 vs
= 1 << floor_log2 (vs
);
4223 static enum mode_class classes
[]
4224 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4225 for (int i
= 0; i
< 4; i
+= 2)
4226 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4228 mode
= GET_MODE_WIDER_MODE (mode
))
4230 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4231 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4234 && GET_MODE_SIZE (vmode
) < vs
4235 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4236 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4238 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4239 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4241 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4242 / GET_MODE_SIZE (mode
));
4243 if (TYPE_MODE (type
) != vmode
)
4245 if (TYPE_ALIGN_UNIT (type
) > al
)
4246 al
= TYPE_ALIGN_UNIT (type
);
4248 return build_int_cst (integer_type_node
, al
);
4251 /* Return maximum possible vectorization factor for the target. */
4258 || !flag_tree_loop_optimize
4259 || (!flag_tree_loop_vectorize
4260 && (global_options_set
.x_flag_tree_loop_vectorize
4261 || global_options_set
.x_flag_tree_vectorize
)))
4264 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4267 vs
= 1 << floor_log2 (vs
);
4270 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4271 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4272 return GET_MODE_NUNITS (vqimode
);
4276 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4280 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4281 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4285 max_vf
= omp_max_vf ();
4288 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4289 OMP_CLAUSE_SAFELEN
);
4290 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4292 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4294 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4298 idx
= create_tmp_var (unsigned_type_node
);
4299 lane
= create_tmp_var (unsigned_type_node
);
4305 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4306 tree avar
= create_tmp_var_raw (atype
);
4307 if (TREE_ADDRESSABLE (new_var
))
4308 TREE_ADDRESSABLE (avar
) = 1;
4309 DECL_ATTRIBUTES (avar
)
4310 = tree_cons (get_identifier ("omp simd array"), NULL
,
4311 DECL_ATTRIBUTES (avar
));
4312 gimple_add_tmp_var (avar
);
4313 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4314 NULL_TREE
, NULL_TREE
);
4315 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4316 NULL_TREE
, NULL_TREE
);
4317 if (DECL_P (new_var
))
4319 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4320 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4325 /* Helper function of lower_rec_input_clauses. For a reference
4326 in simd reduction, add an underlying variable it will reference. */
4329 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4331 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4332 if (TREE_CONSTANT (z
))
4334 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4335 get_name (new_vard
));
4336 gimple_add_tmp_var (z
);
4337 TREE_ADDRESSABLE (z
) = 1;
4338 z
= build_fold_addr_expr_loc (loc
, z
);
4339 gimplify_assign (new_vard
, z
, ilist
);
4343 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4344 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4345 private variables. Initialization statements go in ILIST, while calls
4346 to destructors go in DLIST. */
4349 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4350 omp_context
*ctx
, struct omp_for_data
*fd
)
4352 tree c
, dtor
, copyin_seq
, x
, ptr
;
4353 bool copyin_by_ref
= false;
4354 bool lastprivate_firstprivate
= false;
4355 bool reduction_omp_orig_ref
= false;
4357 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4358 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4360 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4361 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4362 gimple_seq llist
[2] = { NULL
, NULL
};
4366 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4367 with data sharing clauses referencing variable sized vars. That
4368 is unnecessarily hard to support and very unlikely to result in
4369 vectorized code anyway. */
4371 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4372 switch (OMP_CLAUSE_CODE (c
))
4374 case OMP_CLAUSE_LINEAR
:
4375 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4378 case OMP_CLAUSE_PRIVATE
:
4379 case OMP_CLAUSE_FIRSTPRIVATE
:
4380 case OMP_CLAUSE_LASTPRIVATE
:
4381 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4384 case OMP_CLAUSE_REDUCTION
:
4385 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4386 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4393 /* Do all the fixed sized types in the first pass, and the variable sized
4394 types in the second pass. This makes sure that the scalar arguments to
4395 the variable sized types are processed before we use them in the
4396 variable sized operations. */
4397 for (pass
= 0; pass
< 2; ++pass
)
4399 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4401 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4404 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4408 case OMP_CLAUSE_PRIVATE
:
4409 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4412 case OMP_CLAUSE_SHARED
:
4413 /* Ignore shared directives in teams construct. */
4414 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4416 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4418 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4419 || is_global_var (OMP_CLAUSE_DECL (c
)));
4422 case OMP_CLAUSE_FIRSTPRIVATE
:
4423 case OMP_CLAUSE_COPYIN
:
4425 case OMP_CLAUSE_LINEAR
:
4426 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4427 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4428 lastprivate_firstprivate
= true;
4430 case OMP_CLAUSE_REDUCTION
:
4431 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4432 reduction_omp_orig_ref
= true;
4434 case OMP_CLAUSE__LOOPTEMP_
:
4435 /* Handle _looptemp_ clauses only on parallel/task. */
4439 case OMP_CLAUSE_LASTPRIVATE
:
4440 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4442 lastprivate_firstprivate
= true;
4443 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4446 /* Even without corresponding firstprivate, if
4447 decl is Fortran allocatable, it needs outer var
4450 && lang_hooks
.decls
.omp_private_outer_ref
4451 (OMP_CLAUSE_DECL (c
)))
4452 lastprivate_firstprivate
= true;
4454 case OMP_CLAUSE_ALIGNED
:
4457 var
= OMP_CLAUSE_DECL (c
);
4458 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4459 && !is_global_var (var
))
4461 new_var
= maybe_lookup_decl (var
, ctx
);
4462 if (new_var
== NULL_TREE
)
4463 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4464 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4465 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4466 omp_clause_aligned_alignment (c
));
4467 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4468 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4469 gimplify_and_add (x
, ilist
);
4471 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4472 && is_global_var (var
))
4474 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4475 new_var
= lookup_decl (var
, ctx
);
4476 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4477 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4478 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4479 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4480 omp_clause_aligned_alignment (c
));
4481 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4482 x
= create_tmp_var (ptype
);
4483 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4484 gimplify_and_add (t
, ilist
);
4485 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4486 SET_DECL_VALUE_EXPR (new_var
, t
);
4487 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4494 new_var
= var
= OMP_CLAUSE_DECL (c
);
4495 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4497 var
= TREE_OPERAND (var
, 0);
4498 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4499 var
= TREE_OPERAND (var
, 0);
4500 if (TREE_CODE (var
) == INDIRECT_REF
4501 || TREE_CODE (var
) == ADDR_EXPR
)
4502 var
= TREE_OPERAND (var
, 0);
4503 if (is_variable_sized (var
))
4505 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4506 var
= DECL_VALUE_EXPR (var
);
4507 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4508 var
= TREE_OPERAND (var
, 0);
4509 gcc_assert (DECL_P (var
));
4513 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4514 new_var
= lookup_decl (var
, ctx
);
4516 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4521 /* C/C++ array section reductions. */
4522 else if (c_kind
== OMP_CLAUSE_REDUCTION
4523 && var
!= OMP_CLAUSE_DECL (c
))
4528 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4529 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4530 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4532 tree b
= TREE_OPERAND (orig_var
, 1);
4533 b
= maybe_lookup_decl (b
, ctx
);
4536 b
= TREE_OPERAND (orig_var
, 1);
4537 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4539 if (integer_zerop (bias
))
4543 bias
= fold_convert_loc (clause_loc
,
4544 TREE_TYPE (b
), bias
);
4545 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4546 TREE_TYPE (b
), b
, bias
);
4548 orig_var
= TREE_OPERAND (orig_var
, 0);
4550 if (TREE_CODE (orig_var
) == INDIRECT_REF
4551 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4552 orig_var
= TREE_OPERAND (orig_var
, 0);
4553 tree d
= OMP_CLAUSE_DECL (c
);
4554 tree type
= TREE_TYPE (d
);
4555 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4556 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4557 const char *name
= get_name (orig_var
);
4558 if (TREE_CONSTANT (v
))
4560 x
= create_tmp_var_raw (type
, name
);
4561 gimple_add_tmp_var (x
);
4562 TREE_ADDRESSABLE (x
) = 1;
4563 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4568 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4569 tree t
= maybe_lookup_decl (v
, ctx
);
4573 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4574 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4575 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4577 build_int_cst (TREE_TYPE (v
), 1));
4578 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4580 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4581 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4582 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4585 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4586 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4587 tree y
= create_tmp_var (ptype
, name
);
4588 gimplify_assign (y
, x
, ilist
);
4592 if (!integer_zerop (bias
))
4594 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4596 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4598 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4599 pointer_sized_int_node
, yb
, bias
);
4600 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4601 yb
= create_tmp_var (ptype
, name
);
4602 gimplify_assign (yb
, x
, ilist
);
4606 d
= TREE_OPERAND (d
, 0);
4607 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4608 d
= TREE_OPERAND (d
, 0);
4609 if (TREE_CODE (d
) == ADDR_EXPR
)
4611 if (orig_var
!= var
)
4613 gcc_assert (is_variable_sized (orig_var
));
4614 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4616 gimplify_assign (new_var
, x
, ilist
);
4617 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4618 tree t
= build_fold_indirect_ref (new_var
);
4619 DECL_IGNORED_P (new_var
) = 0;
4620 TREE_THIS_NOTRAP (t
);
4621 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4622 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4626 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4627 build_int_cst (ptype
, 0));
4628 SET_DECL_VALUE_EXPR (new_var
, x
);
4629 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4634 gcc_assert (orig_var
== var
);
4635 if (TREE_CODE (d
) == INDIRECT_REF
)
4637 x
= create_tmp_var (ptype
, name
);
4638 TREE_ADDRESSABLE (x
) = 1;
4639 gimplify_assign (x
, yb
, ilist
);
4640 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4642 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4643 gimplify_assign (new_var
, x
, ilist
);
4645 tree y1
= create_tmp_var (ptype
, NULL
);
4646 gimplify_assign (y1
, y
, ilist
);
4647 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4648 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4649 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4650 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4652 y2
= create_tmp_var (ptype
, NULL
);
4653 gimplify_assign (y2
, y
, ilist
);
4654 tree ref
= build_outer_var_ref (var
, ctx
);
4655 /* For ref build_outer_var_ref already performs this. */
4656 if (TREE_CODE (d
) == INDIRECT_REF
)
4657 gcc_assert (is_reference (var
));
4658 else if (TREE_CODE (d
) == ADDR_EXPR
)
4659 ref
= build_fold_addr_expr (ref
);
4660 else if (is_reference (var
))
4661 ref
= build_fold_addr_expr (ref
);
4662 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4663 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4664 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4666 y3
= create_tmp_var (ptype
, NULL
);
4667 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4671 y4
= create_tmp_var (ptype
, NULL
);
4672 gimplify_assign (y4
, ref
, dlist
);
4675 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4676 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4677 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4678 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4679 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4682 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4683 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4684 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4685 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4686 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4688 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4690 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4691 tree decl_placeholder
4692 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4693 SET_DECL_VALUE_EXPR (decl_placeholder
,
4694 build_simple_mem_ref (y1
));
4695 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4696 SET_DECL_VALUE_EXPR (placeholder
,
4697 y3
? build_simple_mem_ref (y3
)
4699 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4700 x
= lang_hooks
.decls
.omp_clause_default_ctor
4701 (c
, build_simple_mem_ref (y1
),
4702 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4704 gimplify_and_add (x
, ilist
);
4705 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4707 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4708 lower_omp (&tseq
, ctx
);
4709 gimple_seq_add_seq (ilist
, tseq
);
4711 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4714 SET_DECL_VALUE_EXPR (decl_placeholder
,
4715 build_simple_mem_ref (y2
));
4716 SET_DECL_VALUE_EXPR (placeholder
,
4717 build_simple_mem_ref (y4
));
4718 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4719 lower_omp (&tseq
, ctx
);
4720 gimple_seq_add_seq (dlist
, tseq
);
4721 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4723 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4724 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4725 x
= lang_hooks
.decls
.omp_clause_dtor
4726 (c
, build_simple_mem_ref (y2
));
4729 gimple_seq tseq
= NULL
;
4731 gimplify_stmt (&dtor
, &tseq
);
4732 gimple_seq_add_seq (dlist
, tseq
);
4737 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4738 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4740 /* reduction(-:var) sums up the partial results, so it
4741 acts identically to reduction(+:var). */
4742 if (code
== MINUS_EXPR
)
4745 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4748 x
= build2 (code
, TREE_TYPE (type
),
4749 build_simple_mem_ref (y4
),
4750 build_simple_mem_ref (y2
));
4751 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4755 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4756 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4757 gimple_seq_add_stmt (ilist
, g
);
4760 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4761 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4762 gimple_seq_add_stmt (ilist
, g
);
4764 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4765 build_int_cst (TREE_TYPE (i
), 1));
4766 gimple_seq_add_stmt (ilist
, g
);
4767 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4768 gimple_seq_add_stmt (ilist
, g
);
4769 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4772 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4773 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4774 gimple_seq_add_stmt (dlist
, g
);
4777 g
= gimple_build_assign
4778 (y4
, POINTER_PLUS_EXPR
, y4
,
4779 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4780 gimple_seq_add_stmt (dlist
, g
);
4782 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4783 build_int_cst (TREE_TYPE (i2
), 1));
4784 gimple_seq_add_stmt (dlist
, g
);
4785 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4786 gimple_seq_add_stmt (dlist
, g
);
4787 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4791 else if (is_variable_sized (var
))
4793 /* For variable sized types, we need to allocate the
4794 actual storage here. Call alloca and store the
4795 result in the pointer decl that we created elsewhere. */
4799 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4804 ptr
= DECL_VALUE_EXPR (new_var
);
4805 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4806 ptr
= TREE_OPERAND (ptr
, 0);
4807 gcc_assert (DECL_P (ptr
));
4808 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4810 /* void *tmp = __builtin_alloca */
4811 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4812 stmt
= gimple_build_call (atmp
, 2, x
,
4813 size_int (DECL_ALIGN (var
)));
4814 tmp
= create_tmp_var_raw (ptr_type_node
);
4815 gimple_add_tmp_var (tmp
);
4816 gimple_call_set_lhs (stmt
, tmp
);
4818 gimple_seq_add_stmt (ilist
, stmt
);
4820 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4821 gimplify_assign (ptr
, x
, ilist
);
4824 else if (is_reference (var
) && !is_oacc_parallel (ctx
))
4826 /* For references that are being privatized for Fortran,
4827 allocate new backing storage for the new pointer
4828 variable. This allows us to avoid changing all the
4829 code that expects a pointer to something that expects
4830 a direct variable. */
4834 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4835 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4837 x
= build_receiver_ref (var
, false, ctx
);
4838 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4840 else if (TREE_CONSTANT (x
))
4842 /* For reduction in SIMD loop, defer adding the
4843 initialization of the reference, because if we decide
4844 to use SIMD array for it, the initilization could cause
4846 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4850 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4852 gimple_add_tmp_var (x
);
4853 TREE_ADDRESSABLE (x
) = 1;
4854 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4860 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4861 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4862 tree al
= size_int (TYPE_ALIGN (rtype
));
4863 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4868 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4869 gimplify_assign (new_var
, x
, ilist
);
4872 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4874 else if (c_kind
== OMP_CLAUSE_REDUCTION
4875 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4883 switch (OMP_CLAUSE_CODE (c
))
4885 case OMP_CLAUSE_SHARED
:
4886 /* Ignore shared directives in teams construct. */
4887 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4889 /* Shared global vars are just accessed directly. */
4890 if (is_global_var (new_var
))
4892 /* For taskloop firstprivate/lastprivate, represented
4893 as firstprivate and shared clause on the task, new_var
4894 is the firstprivate var. */
4895 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4897 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4898 needs to be delayed until after fixup_child_record_type so
4899 that we get the correct type during the dereference. */
4900 by_ref
= use_pointer_for_field (var
, ctx
);
4901 x
= build_receiver_ref (var
, by_ref
, ctx
);
4902 SET_DECL_VALUE_EXPR (new_var
, x
);
4903 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4905 /* ??? If VAR is not passed by reference, and the variable
4906 hasn't been initialized yet, then we'll get a warning for
4907 the store into the omp_data_s structure. Ideally, we'd be
4908 able to notice this and not store anything at all, but
4909 we're generating code too early. Suppress the warning. */
4911 TREE_NO_WARNING (var
) = 1;
4914 case OMP_CLAUSE_LASTPRIVATE
:
4915 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4919 case OMP_CLAUSE_PRIVATE
:
4920 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4921 x
= build_outer_var_ref (var
, ctx
);
4922 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4924 if (is_task_ctx (ctx
))
4925 x
= build_receiver_ref (var
, false, ctx
);
4927 x
= build_outer_var_ref (var
, ctx
);
4933 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4934 (c
, unshare_expr (new_var
), x
);
4937 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4938 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4939 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4940 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4941 idx
, lane
, ivar
, lvar
))
4944 x
= lang_hooks
.decls
.omp_clause_default_ctor
4945 (c
, unshare_expr (ivar
), x
);
4947 gimplify_and_add (x
, &llist
[0]);
4950 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4953 gimple_seq tseq
= NULL
;
4956 gimplify_stmt (&dtor
, &tseq
);
4957 gimple_seq_add_seq (&llist
[1], tseq
);
4964 gimplify_and_add (nx
, ilist
);
4968 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4971 gimple_seq tseq
= NULL
;
4974 gimplify_stmt (&dtor
, &tseq
);
4975 gimple_seq_add_seq (dlist
, tseq
);
4979 case OMP_CLAUSE_LINEAR
:
4980 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4981 goto do_firstprivate
;
4982 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4985 x
= build_outer_var_ref (var
, ctx
);
4988 case OMP_CLAUSE_FIRSTPRIVATE
:
4989 if (is_task_ctx (ctx
))
4991 if (is_reference (var
) || is_variable_sized (var
))
4993 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4995 || use_pointer_for_field (var
, NULL
))
4997 x
= build_receiver_ref (var
, false, ctx
);
4998 SET_DECL_VALUE_EXPR (new_var
, x
);
4999 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5004 x
= build_outer_var_ref (var
, ctx
);
5007 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5008 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5010 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5011 tree stept
= TREE_TYPE (t
);
5012 tree ct
= find_omp_clause (clauses
,
5013 OMP_CLAUSE__LOOPTEMP_
);
5015 tree l
= OMP_CLAUSE_DECL (ct
);
5016 tree n1
= fd
->loop
.n1
;
5017 tree step
= fd
->loop
.step
;
5018 tree itype
= TREE_TYPE (l
);
5019 if (POINTER_TYPE_P (itype
))
5020 itype
= signed_type_for (itype
);
5021 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5022 if (TYPE_UNSIGNED (itype
)
5023 && fd
->loop
.cond_code
== GT_EXPR
)
5024 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5025 fold_build1 (NEGATE_EXPR
, itype
, l
),
5026 fold_build1 (NEGATE_EXPR
,
5029 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5030 t
= fold_build2 (MULT_EXPR
, stept
,
5031 fold_convert (stept
, l
), t
);
5033 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5035 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5037 gimplify_and_add (x
, ilist
);
5041 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5042 x
= fold_build2 (POINTER_PLUS_EXPR
,
5043 TREE_TYPE (x
), x
, t
);
5045 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5048 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5049 || TREE_ADDRESSABLE (new_var
))
5050 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5051 idx
, lane
, ivar
, lvar
))
5053 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5055 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5056 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5057 gimplify_and_add (x
, ilist
);
5058 gimple_stmt_iterator gsi
5059 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5061 = gimple_build_assign (unshare_expr (lvar
), iv
);
5062 gsi_insert_before_without_update (&gsi
, g
,
5064 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5065 enum tree_code code
= PLUS_EXPR
;
5066 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5067 code
= POINTER_PLUS_EXPR
;
5068 g
= gimple_build_assign (iv
, code
, iv
, t
);
5069 gsi_insert_before_without_update (&gsi
, g
,
5073 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5074 (c
, unshare_expr (ivar
), x
);
5075 gimplify_and_add (x
, &llist
[0]);
5076 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5079 gimple_seq tseq
= NULL
;
5082 gimplify_stmt (&dtor
, &tseq
);
5083 gimple_seq_add_seq (&llist
[1], tseq
);
5088 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5089 (c
, unshare_expr (new_var
), x
);
5090 gimplify_and_add (x
, ilist
);
5093 case OMP_CLAUSE__LOOPTEMP_
:
5094 gcc_assert (is_taskreg_ctx (ctx
));
5095 x
= build_outer_var_ref (var
, ctx
);
5096 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5097 gimplify_and_add (x
, ilist
);
5100 case OMP_CLAUSE_COPYIN
:
5101 by_ref
= use_pointer_for_field (var
, NULL
);
5102 x
= build_receiver_ref (var
, by_ref
, ctx
);
5103 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5104 append_to_statement_list (x
, ©in_seq
);
5105 copyin_by_ref
|= by_ref
;
5108 case OMP_CLAUSE_REDUCTION
:
5109 /* OpenACC reductions are initialized using the
5110 GOACC_REDUCTION internal function. */
5111 if (is_gimple_omp_oacc (ctx
->stmt
))
5113 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5115 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5117 x
= build_outer_var_ref (var
, ctx
);
5119 if (is_reference (var
)
5120 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5122 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5123 SET_DECL_VALUE_EXPR (placeholder
, x
);
5124 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5125 tree new_vard
= new_var
;
5126 if (is_reference (var
))
5128 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5129 new_vard
= TREE_OPERAND (new_var
, 0);
5130 gcc_assert (DECL_P (new_vard
));
5133 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5134 idx
, lane
, ivar
, lvar
))
5136 if (new_vard
== new_var
)
5138 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5139 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5143 SET_DECL_VALUE_EXPR (new_vard
,
5144 build_fold_addr_expr (ivar
));
5145 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5147 x
= lang_hooks
.decls
.omp_clause_default_ctor
5148 (c
, unshare_expr (ivar
),
5149 build_outer_var_ref (var
, ctx
));
5151 gimplify_and_add (x
, &llist
[0]);
5152 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5154 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5155 lower_omp (&tseq
, ctx
);
5156 gimple_seq_add_seq (&llist
[0], tseq
);
5158 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5159 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5160 lower_omp (&tseq
, ctx
);
5161 gimple_seq_add_seq (&llist
[1], tseq
);
5162 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5163 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5164 if (new_vard
== new_var
)
5165 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5167 SET_DECL_VALUE_EXPR (new_vard
,
5168 build_fold_addr_expr (lvar
));
5169 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5174 gimplify_stmt (&dtor
, &tseq
);
5175 gimple_seq_add_seq (&llist
[1], tseq
);
5179 /* If this is a reference to constant size reduction var
5180 with placeholder, we haven't emitted the initializer
5181 for it because it is undesirable if SIMD arrays are used.
5182 But if they aren't used, we need to emit the deferred
5183 initialization now. */
5184 else if (is_reference (var
) && is_simd
)
5185 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5186 x
= lang_hooks
.decls
.omp_clause_default_ctor
5187 (c
, unshare_expr (new_var
),
5188 build_outer_var_ref (var
, ctx
));
5190 gimplify_and_add (x
, ilist
);
5191 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5193 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5194 lower_omp (&tseq
, ctx
);
5195 gimple_seq_add_seq (ilist
, tseq
);
5197 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5200 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5201 lower_omp (&tseq
, ctx
);
5202 gimple_seq_add_seq (dlist
, tseq
);
5203 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5205 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5210 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5211 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5212 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5214 /* reduction(-:var) sums up the partial results, so it
5215 acts identically to reduction(+:var). */
5216 if (code
== MINUS_EXPR
)
5219 tree new_vard
= new_var
;
5220 if (is_simd
&& is_reference (var
))
5222 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5223 new_vard
= TREE_OPERAND (new_var
, 0);
5224 gcc_assert (DECL_P (new_vard
));
5227 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5228 idx
, lane
, ivar
, lvar
))
5230 tree ref
= build_outer_var_ref (var
, ctx
);
5232 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5234 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5235 ref
= build_outer_var_ref (var
, ctx
);
5236 gimplify_assign (ref
, x
, &llist
[1]);
5238 if (new_vard
!= new_var
)
5240 SET_DECL_VALUE_EXPR (new_vard
,
5241 build_fold_addr_expr (lvar
));
5242 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5247 if (is_reference (var
) && is_simd
)
5248 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5249 gimplify_assign (new_var
, x
, ilist
);
5252 tree ref
= build_outer_var_ref (var
, ctx
);
5254 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5255 ref
= build_outer_var_ref (var
, ctx
);
5256 gimplify_assign (ref
, x
, dlist
);
5270 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5271 /* Don't want uninit warnings on simduid, it is always uninitialized,
5272 but we use it not for the value, but for the DECL_UID only. */
5273 TREE_NO_WARNING (uid
) = 1;
5275 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5276 gimple_call_set_lhs (g
, lane
);
5277 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5278 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5279 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5280 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5281 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5282 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5283 g
= gimple_build_assign (lane
, INTEGER_CST
,
5284 build_int_cst (unsigned_type_node
, 0));
5285 gimple_seq_add_stmt (ilist
, g
);
5286 for (int i
= 0; i
< 2; i
++)
5289 tree vf
= create_tmp_var (unsigned_type_node
);
5290 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5291 gimple_call_set_lhs (g
, vf
);
5292 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5293 gimple_seq_add_stmt (seq
, g
);
5294 tree t
= build_int_cst (unsigned_type_node
, 0);
5295 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5296 gimple_seq_add_stmt (seq
, g
);
5297 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5298 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5299 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5300 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5301 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5302 gimple_seq_add_seq (seq
, llist
[i
]);
5303 t
= build_int_cst (unsigned_type_node
, 1);
5304 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5305 gimple_seq_add_stmt (seq
, g
);
5306 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5307 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5308 gimple_seq_add_stmt (seq
, g
);
5309 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5313 /* The copyin sequence is not to be executed by the main thread, since
5314 that would result in self-copies. Perhaps not visible to scalars,
5315 but it certainly is to C++ operator=. */
5318 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5320 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5321 build_int_cst (TREE_TYPE (x
), 0));
5322 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5323 gimplify_and_add (x
, ilist
);
5326 /* If any copyin variable is passed by reference, we must ensure the
5327 master thread doesn't modify it before it is copied over in all
5328 threads. Similarly for variables in both firstprivate and
5329 lastprivate clauses we need to ensure the lastprivate copying
5330 happens after firstprivate copying in all threads. And similarly
5331 for UDRs if initializer expression refers to omp_orig. */
5332 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5334 /* Don't add any barrier for #pragma omp simd or
5335 #pragma omp distribute. */
5336 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5337 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5338 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5341 /* If max_vf is non-zero, then we can use only a vectorization factor
5342 up to the max_vf we chose. So stick it into the safelen clause. */
5345 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5346 OMP_CLAUSE_SAFELEN
);
5348 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5349 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5352 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5353 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5355 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5356 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5362 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5363 both parallel and workshare constructs. PREDICATE may be NULL if it's
5367 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5370 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5371 bool par_clauses
= false;
5372 tree simduid
= NULL
, lastlane
= NULL
;
5374 /* Early exit if there are no lastprivate or linear clauses. */
5375 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5376 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5377 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5378 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5380 if (clauses
== NULL
)
5382 /* If this was a workshare clause, see if it had been combined
5383 with its parallel. In that case, look for the clauses on the
5384 parallel statement itself. */
5385 if (is_parallel_ctx (ctx
))
5389 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5392 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5393 OMP_CLAUSE_LASTPRIVATE
);
5394 if (clauses
== NULL
)
5402 tree label_true
, arm1
, arm2
;
5404 label
= create_artificial_label (UNKNOWN_LOCATION
);
5405 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5406 arm1
= TREE_OPERAND (predicate
, 0);
5407 arm2
= TREE_OPERAND (predicate
, 1);
5408 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5409 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5410 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5412 gimple_seq_add_stmt (stmt_list
, stmt
);
5413 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5416 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5417 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5419 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5421 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5424 for (c
= clauses
; c
;)
5427 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5429 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5430 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5431 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5433 var
= OMP_CLAUSE_DECL (c
);
5434 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5435 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5436 && is_taskloop_ctx (ctx
))
5438 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5439 new_var
= lookup_decl (var
, ctx
->outer
);
5442 new_var
= lookup_decl (var
, ctx
);
5444 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5446 tree val
= DECL_VALUE_EXPR (new_var
);
5447 if (TREE_CODE (val
) == ARRAY_REF
5448 && VAR_P (TREE_OPERAND (val
, 0))
5449 && lookup_attribute ("omp simd array",
5450 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5453 if (lastlane
== NULL
)
5455 lastlane
= create_tmp_var (unsigned_type_node
);
5457 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5459 TREE_OPERAND (val
, 1));
5460 gimple_call_set_lhs (g
, lastlane
);
5461 gimple_seq_add_stmt (stmt_list
, g
);
5463 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5464 TREE_OPERAND (val
, 0), lastlane
,
5465 NULL_TREE
, NULL_TREE
);
5469 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5470 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5472 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5473 gimple_seq_add_seq (stmt_list
,
5474 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5475 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5477 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5478 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5480 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5481 gimple_seq_add_seq (stmt_list
,
5482 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5483 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5487 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5488 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5490 gcc_checking_assert (is_taskloop_ctx (ctx
));
5491 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5493 if (is_global_var (ovar
))
5497 x
= build_outer_var_ref (var
, ctx
, true);
5498 if (is_reference (var
))
5499 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5500 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5501 gimplify_and_add (x
, stmt_list
);
5503 c
= OMP_CLAUSE_CHAIN (c
);
5504 if (c
== NULL
&& !par_clauses
)
5506 /* If this was a workshare clause, see if it had been combined
5507 with its parallel. In that case, continue looking for the
5508 clauses also on the parallel statement itself. */
5509 if (is_parallel_ctx (ctx
))
5513 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5516 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5517 OMP_CLAUSE_LASTPRIVATE
);
5523 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5526 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5527 (which might be a placeholder). INNER is true if this is an inner
5528 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5529 join markers. Generate the before-loop forking sequence in
5530 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5531 general form of these sequences is
5533 GOACC_REDUCTION_SETUP
5535 GOACC_REDUCTION_INIT
5537 GOACC_REDUCTION_FINI
5539 GOACC_REDUCTION_TEARDOWN. */
5542 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5543 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5544 gimple_seq
*join_seq
, omp_context
*ctx
)
5546 gimple_seq before_fork
= NULL
;
5547 gimple_seq after_fork
= NULL
;
5548 gimple_seq before_join
= NULL
;
5549 gimple_seq after_join
= NULL
;
5550 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5551 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5552 unsigned offset
= 0;
5554 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5555 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5557 tree orig
= OMP_CLAUSE_DECL (c
);
5558 tree var
= maybe_lookup_decl (orig
, ctx
);
5559 tree ref_to_res
= NULL_TREE
;
5560 tree incoming
, outgoing
;
5562 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5563 if (rcode
== MINUS_EXPR
)
5565 else if (rcode
== TRUTH_ANDIF_EXPR
)
5566 rcode
= BIT_AND_EXPR
;
5567 else if (rcode
== TRUTH_ORIF_EXPR
)
5568 rcode
= BIT_IOR_EXPR
;
5569 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5573 gcc_assert (!is_reference (var
));
5575 incoming
= outgoing
= var
;
5579 /* See if an outer construct also reduces this variable. */
5580 omp_context
*outer
= ctx
;
5582 while (omp_context
*probe
= outer
->outer
)
5584 enum gimple_code type
= gimple_code (probe
->stmt
);
5589 case GIMPLE_OMP_FOR
:
5590 cls
= gimple_omp_for_clauses (probe
->stmt
);
5593 case GIMPLE_OMP_TARGET
:
5594 if (gimple_omp_target_kind (probe
->stmt
)
5595 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5598 cls
= gimple_omp_target_clauses (probe
->stmt
);
5606 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5607 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5608 && orig
== OMP_CLAUSE_DECL (cls
))
5609 goto has_outer_reduction
;
5613 /* This is the outermost construct with this reduction,
5614 see if there's a mapping for it. */
5615 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5616 && maybe_lookup_field (orig
, outer
))
5618 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5619 if (is_reference (orig
))
5620 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5623 incoming
= omp_reduction_init_op (loc
, rcode
, TREE_TYPE (var
));
5626 incoming
= outgoing
= orig
;
5628 has_outer_reduction
:;
5632 ref_to_res
= integer_zero_node
;
5634 /* Determine position in reduction buffer, which may be used
5636 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5637 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5638 offset
= (offset
+ align
- 1) & ~(align
- 1);
5639 tree off
= build_int_cst (sizetype
, offset
);
5640 offset
+= GET_MODE_SIZE (mode
);
5644 init_code
= build_int_cst (integer_type_node
,
5645 IFN_GOACC_REDUCTION_INIT
);
5646 fini_code
= build_int_cst (integer_type_node
,
5647 IFN_GOACC_REDUCTION_FINI
);
5648 setup_code
= build_int_cst (integer_type_node
,
5649 IFN_GOACC_REDUCTION_SETUP
);
5650 teardown_code
= build_int_cst (integer_type_node
,
5651 IFN_GOACC_REDUCTION_TEARDOWN
);
5655 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5656 TREE_TYPE (var
), 6, setup_code
,
5657 unshare_expr (ref_to_res
),
5658 incoming
, level
, op
, off
);
5660 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5661 TREE_TYPE (var
), 6, init_code
,
5662 unshare_expr (ref_to_res
),
5663 var
, level
, op
, off
);
5665 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5666 TREE_TYPE (var
), 6, fini_code
,
5667 unshare_expr (ref_to_res
),
5668 var
, level
, op
, off
);
5670 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5671 TREE_TYPE (var
), 6, teardown_code
,
5672 ref_to_res
, var
, level
, op
, off
);
5674 gimplify_assign (var
, setup_call
, &before_fork
);
5675 gimplify_assign (var
, init_call
, &after_fork
);
5676 gimplify_assign (var
, fini_call
, &before_join
);
5677 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5680 /* Now stitch things together. */
5681 gimple_seq_add_seq (fork_seq
, before_fork
);
5683 gimple_seq_add_stmt (fork_seq
, fork
);
5684 gimple_seq_add_seq (fork_seq
, after_fork
);
5686 gimple_seq_add_seq (join_seq
, before_join
);
5688 gimple_seq_add_stmt (join_seq
, join
);
5689 gimple_seq_add_seq (join_seq
, after_join
);
5692 /* Generate code to implement the REDUCTION clauses. */
5695 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5697 gimple_seq sub_seq
= NULL
;
5702 /* OpenACC loop reductions are handled elsewhere. */
5703 if (is_gimple_omp_oacc (ctx
->stmt
))
5706 /* SIMD reductions are handled in lower_rec_input_clauses. */
5707 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5708 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5711 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5712 update in that case, otherwise use a lock. */
5713 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5714 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5716 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5717 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5719 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5729 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5731 tree var
, ref
, new_var
, orig_var
;
5732 enum tree_code code
;
5733 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5735 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5738 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5739 if (TREE_CODE (var
) == MEM_REF
)
5741 var
= TREE_OPERAND (var
, 0);
5742 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5743 var
= TREE_OPERAND (var
, 0);
5744 if (TREE_CODE (var
) == INDIRECT_REF
5745 || TREE_CODE (var
) == ADDR_EXPR
)
5746 var
= TREE_OPERAND (var
, 0);
5748 if (is_variable_sized (var
))
5750 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5751 var
= DECL_VALUE_EXPR (var
);
5752 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5753 var
= TREE_OPERAND (var
, 0);
5754 gcc_assert (DECL_P (var
));
5757 new_var
= lookup_decl (var
, ctx
);
5758 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5759 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5760 ref
= build_outer_var_ref (var
, ctx
);
5761 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5763 /* reduction(-:var) sums up the partial results, so it acts
5764 identically to reduction(+:var). */
5765 if (code
== MINUS_EXPR
)
5770 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5772 addr
= save_expr (addr
);
5773 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5774 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5775 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5776 gimplify_and_add (x
, stmt_seqp
);
5779 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5781 tree d
= OMP_CLAUSE_DECL (c
);
5782 tree type
= TREE_TYPE (d
);
5783 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5784 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5785 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5786 tree bias
= TREE_OPERAND (d
, 1);
5787 d
= TREE_OPERAND (d
, 0);
5788 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5790 tree b
= TREE_OPERAND (d
, 1);
5791 b
= maybe_lookup_decl (b
, ctx
);
5794 b
= TREE_OPERAND (d
, 1);
5795 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5797 if (integer_zerop (bias
))
5801 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5802 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5803 TREE_TYPE (b
), b
, bias
);
5805 d
= TREE_OPERAND (d
, 0);
5807 /* For ref build_outer_var_ref already performs this, so
5808 only new_var needs a dereference. */
5809 if (TREE_CODE (d
) == INDIRECT_REF
)
5811 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5812 gcc_assert (is_reference (var
) && var
== orig_var
);
5814 else if (TREE_CODE (d
) == ADDR_EXPR
)
5816 if (orig_var
== var
)
5818 new_var
= build_fold_addr_expr (new_var
);
5819 ref
= build_fold_addr_expr (ref
);
5824 gcc_assert (orig_var
== var
);
5825 if (is_reference (var
))
5826 ref
= build_fold_addr_expr (ref
);
5830 tree t
= maybe_lookup_decl (v
, ctx
);
5834 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5835 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5837 if (!integer_zerop (bias
))
5839 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5840 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5841 TREE_TYPE (new_var
), new_var
,
5842 unshare_expr (bias
));
5843 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5844 TREE_TYPE (ref
), ref
, bias
);
5846 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5847 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5848 tree m
= create_tmp_var (ptype
, NULL
);
5849 gimplify_assign (m
, new_var
, stmt_seqp
);
5851 m
= create_tmp_var (ptype
, NULL
);
5852 gimplify_assign (m
, ref
, stmt_seqp
);
5854 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5855 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5856 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5857 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5858 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5859 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5860 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5862 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5863 tree decl_placeholder
5864 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5865 SET_DECL_VALUE_EXPR (placeholder
, out
);
5866 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5867 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5868 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5869 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5870 gimple_seq_add_seq (&sub_seq
,
5871 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5872 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5873 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5874 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5878 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5879 out
= unshare_expr (out
);
5880 gimplify_assign (out
, x
, &sub_seq
);
5882 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5883 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5884 gimple_seq_add_stmt (&sub_seq
, g
);
5885 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5886 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5887 gimple_seq_add_stmt (&sub_seq
, g
);
5888 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5889 build_int_cst (TREE_TYPE (i
), 1));
5890 gimple_seq_add_stmt (&sub_seq
, g
);
5891 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5892 gimple_seq_add_stmt (&sub_seq
, g
);
5893 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5895 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5897 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5899 if (is_reference (var
)
5900 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5902 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5903 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5904 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5905 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5906 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5907 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5908 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5912 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5913 ref
= build_outer_var_ref (var
, ctx
);
5914 gimplify_assign (ref
, x
, &sub_seq
);
5918 if (is_gimple_omp_oacc (ctx
->stmt
))
5921 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5923 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5925 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5927 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5929 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5933 /* Generate code to implement the COPYPRIVATE clauses. */
5936 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5941 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5943 tree var
, new_var
, ref
, x
;
5945 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5947 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5950 var
= OMP_CLAUSE_DECL (c
);
5951 by_ref
= use_pointer_for_field (var
, NULL
);
5953 ref
= build_sender_ref (var
, ctx
);
5954 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5957 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5958 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5960 gimplify_assign (ref
, x
, slist
);
5962 ref
= build_receiver_ref (var
, false, ctx
);
5965 ref
= fold_convert_loc (clause_loc
,
5966 build_pointer_type (TREE_TYPE (new_var
)),
5968 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5970 if (is_reference (var
))
5972 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5973 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5974 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5976 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5977 gimplify_and_add (x
, rlist
);
5982 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5983 and REDUCTION from the sender (aka parent) side. */
5986 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5990 int ignored_looptemp
= 0;
5991 bool is_taskloop
= false;
5993 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5994 by GOMP_taskloop. */
5995 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5997 ignored_looptemp
= 2;
6001 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6003 tree val
, ref
, x
, var
;
6004 bool by_ref
, do_in
= false, do_out
= false;
6005 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6007 switch (OMP_CLAUSE_CODE (c
))
6009 case OMP_CLAUSE_PRIVATE
:
6010 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
6013 case OMP_CLAUSE_FIRSTPRIVATE
:
6014 case OMP_CLAUSE_COPYIN
:
6015 case OMP_CLAUSE_LASTPRIVATE
:
6016 case OMP_CLAUSE_REDUCTION
:
6018 case OMP_CLAUSE_SHARED
:
6019 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6022 case OMP_CLAUSE__LOOPTEMP_
:
6023 if (ignored_looptemp
)
6033 val
= OMP_CLAUSE_DECL (c
);
6034 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6035 && TREE_CODE (val
) == MEM_REF
)
6037 val
= TREE_OPERAND (val
, 0);
6038 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
6039 val
= TREE_OPERAND (val
, 0);
6040 if (TREE_CODE (val
) == INDIRECT_REF
6041 || TREE_CODE (val
) == ADDR_EXPR
)
6042 val
= TREE_OPERAND (val
, 0);
6043 if (is_variable_sized (val
))
6047 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6048 outer taskloop region. */
6049 omp_context
*ctx_for_o
= ctx
;
6051 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
6052 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6053 ctx_for_o
= ctx
->outer
;
6055 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
6057 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
6058 && is_global_var (var
))
6061 t
= omp_member_access_dummy_var (var
);
6064 var
= DECL_VALUE_EXPR (var
);
6065 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
6067 var
= unshare_and_remap (var
, t
, o
);
6069 var
= unshare_expr (var
);
6072 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
6074 /* Handle taskloop firstprivate/lastprivate, where the
6075 lastprivate on GIMPLE_OMP_TASK is represented as
6076 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6077 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
6078 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
6079 if (use_pointer_for_field (val
, ctx
))
6080 var
= build_fold_addr_expr (var
);
6081 gimplify_assign (x
, var
, ilist
);
6082 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
6086 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6087 || val
== OMP_CLAUSE_DECL (c
))
6088 && is_variable_sized (val
))
6090 by_ref
= use_pointer_for_field (val
, NULL
);
6092 switch (OMP_CLAUSE_CODE (c
))
6094 case OMP_CLAUSE_PRIVATE
:
6095 case OMP_CLAUSE_FIRSTPRIVATE
:
6096 case OMP_CLAUSE_COPYIN
:
6097 case OMP_CLAUSE__LOOPTEMP_
:
6101 case OMP_CLAUSE_LASTPRIVATE
:
6102 if (by_ref
|| is_reference (val
))
6104 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
6111 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
6116 case OMP_CLAUSE_REDUCTION
:
6118 if (val
== OMP_CLAUSE_DECL (c
))
6119 do_out
= !(by_ref
|| is_reference (val
));
6121 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
6130 ref
= build_sender_ref (val
, ctx
);
6131 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
6132 gimplify_assign (ref
, x
, ilist
);
6133 if (is_task_ctx (ctx
))
6134 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
6139 ref
= build_sender_ref (val
, ctx
);
6140 gimplify_assign (var
, ref
, olist
);
6145 /* Generate code to implement SHARED from the sender (aka parent)
6146 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6147 list things that got automatically shared. */
6150 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6152 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6154 if (ctx
->record_type
== NULL
)
6157 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6158 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6160 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6161 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6164 nvar
= maybe_lookup_decl (ovar
, ctx
);
6165 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6168 /* If CTX is a nested parallel directive. Find the immediately
6169 enclosing parallel or workshare construct that contains a
6170 mapping for OVAR. */
6171 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6173 t
= omp_member_access_dummy_var (var
);
6176 var
= DECL_VALUE_EXPR (var
);
6177 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6179 var
= unshare_and_remap (var
, t
, o
);
6181 var
= unshare_expr (var
);
6184 if (use_pointer_for_field (ovar
, ctx
))
6186 x
= build_sender_ref (ovar
, ctx
);
6187 var
= build_fold_addr_expr (var
);
6188 gimplify_assign (x
, var
, ilist
);
6192 x
= build_sender_ref (ovar
, ctx
);
6193 gimplify_assign (x
, var
, ilist
);
6195 if (!TREE_READONLY (var
)
6196 /* We don't need to receive a new reference to a result
6197 or parm decl. In fact we may not store to it as we will
6198 invalidate any pending RSO and generate wrong gimple
6200 && !((TREE_CODE (var
) == RESULT_DECL
6201 || TREE_CODE (var
) == PARM_DECL
)
6202 && DECL_BY_REFERENCE (var
)))
6204 x
= build_sender_ref (ovar
, ctx
);
6205 gimplify_assign (var
, x
, olist
);
6211 /* Emit an OpenACC head marker call, encapulating the partitioning and
6212 other information that must be processed by the target compiler.
6213 Return the maximum number of dimensions the associated loop might
6214 be partitioned over. */
6217 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6218 gimple_seq
*seq
, omp_context
*ctx
)
6220 unsigned levels
= 0;
6222 tree gang_static
= NULL_TREE
;
6223 auto_vec
<tree
, 5> args
;
6225 args
.quick_push (build_int_cst
6226 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6227 args
.quick_push (ddvar
);
6228 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6230 switch (OMP_CLAUSE_CODE (c
))
6232 case OMP_CLAUSE_GANG
:
6233 tag
|= OLF_DIM_GANG
;
6234 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6235 /* static:* is represented by -1, and we can ignore it, as
6236 scheduling is always static. */
6237 if (gang_static
&& integer_minus_onep (gang_static
))
6238 gang_static
= NULL_TREE
;
6242 case OMP_CLAUSE_WORKER
:
6243 tag
|= OLF_DIM_WORKER
;
6247 case OMP_CLAUSE_VECTOR
:
6248 tag
|= OLF_DIM_VECTOR
;
6252 case OMP_CLAUSE_SEQ
:
6256 case OMP_CLAUSE_AUTO
:
6260 case OMP_CLAUSE_INDEPENDENT
:
6261 tag
|= OLF_INDEPENDENT
;
6271 if (DECL_P (gang_static
))
6272 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6273 tag
|= OLF_GANG_STATIC
;
6276 /* In a parallel region, loops are implicitly INDEPENDENT. */
6277 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6278 if (!tgt
|| is_oacc_parallel (tgt
))
6279 tag
|= OLF_INDEPENDENT
;
6281 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6282 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6286 /* Ensure at least one level. */
6290 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6291 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6293 args
.quick_push (gang_static
);
6295 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6296 gimple_set_location (call
, loc
);
6297 gimple_set_lhs (call
, ddvar
);
6298 gimple_seq_add_stmt (seq
, call
);
6303 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6304 partitioning level of the enclosed region. */
6307 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6308 tree tofollow
, gimple_seq
*seq
)
6310 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6311 : IFN_UNIQUE_OACC_TAIL_MARK
);
6312 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6313 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6314 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6315 marker
, ddvar
, tofollow
);
6316 gimple_set_location (call
, loc
);
6317 gimple_set_lhs (call
, ddvar
);
6318 gimple_seq_add_stmt (seq
, call
);
6321 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6322 the loop clauses, from which we extract reductions. Initialize
6326 lower_oacc_head_tail (location_t loc
, tree clauses
,
6327 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6330 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6331 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6333 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6335 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6337 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6338 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6340 for (unsigned done
= 1; count
; count
--, done
++)
6342 gimple_seq fork_seq
= NULL
;
6343 gimple_seq join_seq
= NULL
;
6345 tree place
= build_int_cst (integer_type_node
, -1);
6346 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6347 fork_kind
, ddvar
, place
);
6348 gimple_set_location (fork
, loc
);
6349 gimple_set_lhs (fork
, ddvar
);
6351 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6352 join_kind
, ddvar
, place
);
6353 gimple_set_location (join
, loc
);
6354 gimple_set_lhs (join
, ddvar
);
6356 /* Mark the beginning of this level sequence. */
6358 lower_oacc_loop_marker (loc
, ddvar
, true,
6359 build_int_cst (integer_type_node
, count
),
6361 lower_oacc_loop_marker (loc
, ddvar
, false,
6362 build_int_cst (integer_type_node
, done
),
6365 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6366 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6368 /* Append this level to head. */
6369 gimple_seq_add_seq (head
, fork_seq
);
6370 /* Prepend it to tail. */
6371 gimple_seq_add_seq (&join_seq
, *tail
);
6377 /* Mark the end of the sequence. */
6378 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6379 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6382 /* A convenience function to build an empty GIMPLE_COND with just the
6386 gimple_build_cond_empty (tree cond
)
6388 enum tree_code pred_code
;
6391 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6392 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6395 /* Return true if a parallel REGION is within a declare target function or
6396 within a target region and is not a part of a gridified target. */
6399 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
6401 bool indirect
= false;
6402 for (region
= region
->outer
; region
; region
= region
->outer
)
6404 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6406 else if (region
->type
== GIMPLE_OMP_TARGET
)
6408 gomp_target
*tgt_stmt
6409 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
6411 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
6412 OMP_CLAUSE__GRIDDIM_
))
6419 if (lookup_attribute ("omp declare target",
6420 DECL_ATTRIBUTES (current_function_decl
)))
6426 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6429 /* Build the function calls to GOMP_parallel_start etc to actually
6430 generate the parallel operation. REGION is the parallel region
6431 being expanded. BB is the block where to insert the code. WS_ARGS
6432 will be set if this is a call to a combined parallel+workshare
6433 construct, it contains the list of additional arguments needed by
6434 the workshare construct. */
6437 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6438 gomp_parallel
*entry_stmt
,
6439 vec
<tree
, va_gc
> *ws_args
)
6441 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6442 gimple_stmt_iterator gsi
;
6444 enum built_in_function start_ix
;
6446 location_t clause_loc
;
6447 vec
<tree
, va_gc
> *args
;
6449 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6451 /* Determine what flavor of GOMP_parallel we will be
6453 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6454 if (is_combined_parallel (region
))
6456 switch (region
->inner
->type
)
6458 case GIMPLE_OMP_FOR
:
6459 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6460 switch (region
->inner
->sched_kind
)
6462 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6465 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6466 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6467 if (region
->inner
->sched_modifiers
6468 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6470 start_ix2
= 3 + region
->inner
->sched_kind
;
6475 start_ix2
= region
->inner
->sched_kind
;
6478 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6479 start_ix
= (enum built_in_function
) start_ix2
;
6481 case GIMPLE_OMP_SECTIONS
:
6482 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6489 /* By default, the value of NUM_THREADS is zero (selected at run time)
6490 and there is no conditional. */
6492 val
= build_int_cst (unsigned_type_node
, 0);
6493 flags
= build_int_cst (unsigned_type_node
, 0);
6495 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6497 cond
= OMP_CLAUSE_IF_EXPR (c
);
6499 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6502 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6503 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6506 clause_loc
= gimple_location (entry_stmt
);
6508 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6510 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6512 /* Ensure 'val' is of the correct type. */
6513 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6515 /* If we found the clause 'if (cond)', build either
6516 (cond != 0) or (cond ? val : 1u). */
6519 cond
= gimple_boolify (cond
);
6521 if (integer_zerop (val
))
6522 val
= fold_build2_loc (clause_loc
,
6523 EQ_EXPR
, unsigned_type_node
, cond
,
6524 build_int_cst (TREE_TYPE (cond
), 0));
6527 basic_block cond_bb
, then_bb
, else_bb
;
6528 edge e
, e_then
, e_else
;
6529 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6531 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6532 if (gimple_in_ssa_p (cfun
))
6534 tmp_then
= make_ssa_name (tmp_var
);
6535 tmp_else
= make_ssa_name (tmp_var
);
6536 tmp_join
= make_ssa_name (tmp_var
);
6545 e
= split_block_after_labels (bb
);
6550 then_bb
= create_empty_bb (cond_bb
);
6551 else_bb
= create_empty_bb (then_bb
);
6552 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6553 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6555 stmt
= gimple_build_cond_empty (cond
);
6556 gsi
= gsi_start_bb (cond_bb
);
6557 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6559 gsi
= gsi_start_bb (then_bb
);
6560 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6562 gsi
= gsi_start_bb (else_bb
);
6563 expand_omp_build_assign (&gsi
, tmp_else
,
6564 build_int_cst (unsigned_type_node
, 1),
6567 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6568 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6569 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6570 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6571 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6572 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6574 if (gimple_in_ssa_p (cfun
))
6576 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6577 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6578 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6584 gsi
= gsi_start_bb (bb
);
6585 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6586 false, GSI_CONTINUE_LINKING
);
6589 gsi
= gsi_last_bb (bb
);
6590 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6592 t1
= null_pointer_node
;
6594 t1
= build_fold_addr_expr (t
);
6595 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
6596 t2
= build_fold_addr_expr (child_fndecl
);
6598 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6599 args
->quick_push (t2
);
6600 args
->quick_push (t1
);
6601 args
->quick_push (val
);
6603 args
->splice (*ws_args
);
6604 args
->quick_push (flags
);
6606 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6607 builtin_decl_explicit (start_ix
), args
);
6609 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6610 false, GSI_CONTINUE_LINKING
);
6612 if (hsa_gen_requested_p ()
6613 && parallel_needs_hsa_kernel_p (region
))
6615 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
6616 hsa_register_kernel (child_cnode
);
6620 /* Insert a function call whose name is FUNC_NAME with the information from
6621 ENTRY_STMT into the basic_block BB. */
6624 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6625 vec
<tree
, va_gc
> *ws_args
)
6628 gimple_stmt_iterator gsi
;
6629 vec
<tree
, va_gc
> *args
;
6631 gcc_assert (vec_safe_length (ws_args
) == 2);
6632 tree func_name
= (*ws_args
)[0];
6633 tree grain
= (*ws_args
)[1];
6635 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6636 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6637 gcc_assert (count
!= NULL_TREE
);
6638 count
= OMP_CLAUSE_OPERAND (count
, 0);
6640 gsi
= gsi_last_bb (bb
);
6641 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6643 t1
= null_pointer_node
;
6645 t1
= build_fold_addr_expr (t
);
6646 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6648 vec_alloc (args
, 4);
6649 args
->quick_push (t2
);
6650 args
->quick_push (t1
);
6651 args
->quick_push (count
);
6652 args
->quick_push (grain
);
6653 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6655 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6656 GSI_CONTINUE_LINKING
);
6659 /* Build the function call to GOMP_task to actually
6660 generate the task operation. BB is the block where to insert the code. */
6663 expand_task_call (struct omp_region
*region
, basic_block bb
,
6664 gomp_task
*entry_stmt
)
6667 gimple_stmt_iterator gsi
;
6668 location_t loc
= gimple_location (entry_stmt
);
6670 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6672 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6673 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6674 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6675 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6676 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6677 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6680 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6681 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6682 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6684 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6685 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6686 tree num_tasks
= NULL_TREE
;
6690 gimple
*g
= last_stmt (region
->outer
->entry
);
6691 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6692 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6693 struct omp_for_data fd
;
6694 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6695 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6696 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6697 OMP_CLAUSE__LOOPTEMP_
);
6698 startvar
= OMP_CLAUSE_DECL (startvar
);
6699 endvar
= OMP_CLAUSE_DECL (endvar
);
6700 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6701 if (fd
.loop
.cond_code
== LT_EXPR
)
6702 iflags
|= GOMP_TASK_FLAG_UP
;
6703 tree tclauses
= gimple_omp_for_clauses (g
);
6704 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6706 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6709 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6712 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6713 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6716 num_tasks
= integer_zero_node
;
6718 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6719 if (ifc
== NULL_TREE
)
6720 iflags
|= GOMP_TASK_FLAG_IF
;
6721 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6722 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6723 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6726 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6728 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6730 tree cond
= boolean_true_node
;
6735 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6736 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6737 build_int_cst (unsigned_type_node
,
6739 build_int_cst (unsigned_type_node
, 0));
6740 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6744 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6749 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6750 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6751 build_int_cst (unsigned_type_node
,
6752 GOMP_TASK_FLAG_FINAL
),
6753 build_int_cst (unsigned_type_node
, 0));
6754 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6757 depend
= OMP_CLAUSE_DECL (depend
);
6759 depend
= build_int_cst (ptr_type_node
, 0);
6761 priority
= fold_convert (integer_type_node
,
6762 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6764 priority
= integer_zero_node
;
6766 gsi
= gsi_last_bb (bb
);
6767 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6769 t2
= null_pointer_node
;
6771 t2
= build_fold_addr_expr_loc (loc
, t
);
6772 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6773 t
= gimple_omp_task_copy_fn (entry_stmt
);
6775 t3
= null_pointer_node
;
6777 t3
= build_fold_addr_expr_loc (loc
, t
);
6780 t
= build_call_expr (ull
6781 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6782 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6784 gimple_omp_task_arg_size (entry_stmt
),
6785 gimple_omp_task_arg_align (entry_stmt
), flags
,
6786 num_tasks
, priority
, startvar
, endvar
, step
);
6788 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6790 gimple_omp_task_arg_size (entry_stmt
),
6791 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6794 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6795 false, GSI_CONTINUE_LINKING
);
6799 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6800 catch handler and return it. This prevents programs from violating the
6801 structured block semantics with throws. */
6804 maybe_catch_exception (gimple_seq body
)
6809 if (!flag_exceptions
)
6812 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6813 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6815 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6817 g
= gimple_build_eh_must_not_throw (decl
);
6818 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6821 return gimple_seq_alloc_with_stmt (g
);
6824 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6827 vec2chain (vec
<tree
, va_gc
> *v
)
6829 tree chain
= NULL_TREE
, t
;
6832 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6834 DECL_CHAIN (t
) = chain
;
6842 /* Remove barriers in REGION->EXIT's block. Note that this is only
6843 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6844 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6845 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6849 remove_exit_barrier (struct omp_region
*region
)
6851 gimple_stmt_iterator gsi
;
6852 basic_block exit_bb
;
6856 int any_addressable_vars
= -1;
6858 exit_bb
= region
->exit
;
6860 /* If the parallel region doesn't return, we don't have REGION->EXIT
6865 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6866 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6867 statements that can appear in between are extremely limited -- no
6868 memory operations at all. Here, we allow nothing at all, so the
6869 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6870 gsi
= gsi_last_bb (exit_bb
);
6871 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6873 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6876 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6878 gsi
= gsi_last_bb (e
->src
);
6879 if (gsi_end_p (gsi
))
6881 stmt
= gsi_stmt (gsi
);
6882 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6883 && !gimple_omp_return_nowait_p (stmt
))
6885 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6886 in many cases. If there could be tasks queued, the barrier
6887 might be needed to let the tasks run before some local
6888 variable of the parallel that the task uses as shared
6889 runs out of scope. The task can be spawned either
6890 from within current function (this would be easy to check)
6891 or from some function it calls and gets passed an address
6892 of such a variable. */
6893 if (any_addressable_vars
< 0)
6895 gomp_parallel
*parallel_stmt
6896 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6897 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6898 tree local_decls
, block
, decl
;
6901 any_addressable_vars
= 0;
6902 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6903 if (TREE_ADDRESSABLE (decl
))
6905 any_addressable_vars
= 1;
6908 for (block
= gimple_block (stmt
);
6909 !any_addressable_vars
6911 && TREE_CODE (block
) == BLOCK
;
6912 block
= BLOCK_SUPERCONTEXT (block
))
6914 for (local_decls
= BLOCK_VARS (block
);
6916 local_decls
= DECL_CHAIN (local_decls
))
6917 if (TREE_ADDRESSABLE (local_decls
))
6919 any_addressable_vars
= 1;
6922 if (block
== gimple_block (parallel_stmt
))
6926 if (!any_addressable_vars
)
6927 gimple_omp_return_set_nowait (stmt
);
6933 remove_exit_barriers (struct omp_region
*region
)
6935 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6936 remove_exit_barrier (region
);
6940 region
= region
->inner
;
6941 remove_exit_barriers (region
);
6942 while (region
->next
)
6944 region
= region
->next
;
6945 remove_exit_barriers (region
);
6950 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6951 calls. These can't be declared as const functions, but
6952 within one parallel body they are constant, so they can be
6953 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6954 which are declared const. Similarly for task body, except
6955 that in untied task omp_get_thread_num () can change at any task
6956 scheduling point. */
6959 optimize_omp_library_calls (gimple
*entry_stmt
)
6962 gimple_stmt_iterator gsi
;
6963 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6964 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6965 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6966 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6967 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6968 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6969 OMP_CLAUSE_UNTIED
) != NULL
);
6971 FOR_EACH_BB_FN (bb
, cfun
)
6972 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6974 gimple
*call
= gsi_stmt (gsi
);
6977 if (is_gimple_call (call
)
6978 && (decl
= gimple_call_fndecl (call
))
6979 && DECL_EXTERNAL (decl
)
6980 && TREE_PUBLIC (decl
)
6981 && DECL_INITIAL (decl
) == NULL
)
6985 if (DECL_NAME (decl
) == thr_num_id
)
6987 /* In #pragma omp task untied omp_get_thread_num () can change
6988 during the execution of the task region. */
6991 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6993 else if (DECL_NAME (decl
) == num_thr_id
)
6994 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6998 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6999 || gimple_call_num_args (call
) != 0)
7002 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
7005 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
7006 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
7007 TREE_TYPE (TREE_TYPE (built_in
))))
7010 gimple_call_set_fndecl (call
, built_in
);
7015 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7019 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
7023 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7024 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
7027 if (TREE_CODE (t
) == ADDR_EXPR
)
7028 recompute_tree_invariant_for_addr_expr (t
);
7030 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
7034 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7037 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
7040 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
7041 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
7042 !after
, after
? GSI_CONTINUE_LINKING
7044 gimple
*stmt
= gimple_build_assign (to
, from
);
7046 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
7048 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
7049 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
7050 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
7052 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
7053 gimple_regimplify_operands (stmt
, &gsi
);
7057 /* Expand the OpenMP parallel or task directive starting at REGION. */
7060 expand_omp_taskreg (struct omp_region
*region
)
7062 basic_block entry_bb
, exit_bb
, new_bb
;
7063 struct function
*child_cfun
;
7064 tree child_fn
, block
, t
;
7065 gimple_stmt_iterator gsi
;
7066 gimple
*entry_stmt
, *stmt
;
7068 vec
<tree
, va_gc
> *ws_args
;
7070 entry_stmt
= last_stmt (region
->entry
);
7071 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
7072 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7074 entry_bb
= region
->entry
;
7075 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
7076 exit_bb
= region
->cont
;
7078 exit_bb
= region
->exit
;
7082 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
7083 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
7084 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
7087 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7088 and the inner statement contains the name of the built-in function
7090 ws_args
= region
->inner
->ws_args
;
7091 else if (is_combined_parallel (region
))
7092 ws_args
= region
->ws_args
;
7096 if (child_cfun
->cfg
)
7098 /* Due to inlining, it may happen that we have already outlined
7099 the region, in which case all we need to do is make the
7100 sub-graph unreachable and emit the parallel call. */
7101 edge entry_succ_e
, exit_succ_e
;
7103 entry_succ_e
= single_succ_edge (entry_bb
);
7105 gsi
= gsi_last_bb (entry_bb
);
7106 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
7107 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
7108 gsi_remove (&gsi
, true);
7113 exit_succ_e
= single_succ_edge (exit_bb
);
7114 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
7116 remove_edge_and_dominated_blocks (entry_succ_e
);
7120 unsigned srcidx
, dstidx
, num
;
7122 /* If the parallel region needs data sent from the parent
7123 function, then the very first statement (except possible
7124 tree profile counter updates) of the parallel body
7125 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7126 &.OMP_DATA_O is passed as an argument to the child function,
7127 we need to replace it with the argument as seen by the child
7130 In most cases, this will end up being the identity assignment
7131 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7132 a function call that has been inlined, the original PARM_DECL
7133 .OMP_DATA_I may have been converted into a different local
7134 variable. In which case, we need to keep the assignment. */
7135 if (gimple_omp_taskreg_data_arg (entry_stmt
))
7137 basic_block entry_succ_bb
7138 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
7139 : FALLTHRU_EDGE (entry_bb
)->dest
;
7141 gimple
*parcopy_stmt
= NULL
;
7143 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7147 gcc_assert (!gsi_end_p (gsi
));
7148 stmt
= gsi_stmt (gsi
);
7149 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7152 if (gimple_num_ops (stmt
) == 2)
7154 tree arg
= gimple_assign_rhs1 (stmt
);
7156 /* We're ignore the subcode because we're
7157 effectively doing a STRIP_NOPS. */
7159 if (TREE_CODE (arg
) == ADDR_EXPR
7160 && TREE_OPERAND (arg
, 0)
7161 == gimple_omp_taskreg_data_arg (entry_stmt
))
7163 parcopy_stmt
= stmt
;
7169 gcc_assert (parcopy_stmt
!= NULL
);
7170 arg
= DECL_ARGUMENTS (child_fn
);
7172 if (!gimple_in_ssa_p (cfun
))
7174 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
7175 gsi_remove (&gsi
, true);
7178 /* ?? Is setting the subcode really necessary ?? */
7179 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
7180 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7185 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
7186 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
7187 /* We'd like to set the rhs to the default def in the child_fn,
7188 but it's too early to create ssa names in the child_fn.
7189 Instead, we set the rhs to the parm. In
7190 move_sese_region_to_fn, we introduce a default def for the
7191 parm, map the parm to it's default def, and once we encounter
7192 this stmt, replace the parm with the default def. */
7193 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7194 update_stmt (parcopy_stmt
);
7198 /* Declare local variables needed in CHILD_CFUN. */
7199 block
= DECL_INITIAL (child_fn
);
7200 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7201 /* The gimplifier could record temporaries in parallel/task block
7202 rather than in containing function's local_decls chain,
7203 which would mean cgraph missed finalizing them. Do it now. */
7204 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7205 if (TREE_CODE (t
) == VAR_DECL
7207 && !DECL_EXTERNAL (t
))
7208 varpool_node::finalize_decl (t
);
7209 DECL_SAVED_TREE (child_fn
) = NULL
;
7210 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7211 gimple_set_body (child_fn
, NULL
);
7212 TREE_USED (block
) = 1;
7214 /* Reset DECL_CONTEXT on function arguments. */
7215 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7216 DECL_CONTEXT (t
) = child_fn
;
7218 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7219 so that it can be moved to the child function. */
7220 gsi
= gsi_last_bb (entry_bb
);
7221 stmt
= gsi_stmt (gsi
);
7222 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7223 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7224 e
= split_block (entry_bb
, stmt
);
7225 gsi_remove (&gsi
, true);
7228 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7229 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7232 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7233 gcc_assert (e2
->dest
== region
->exit
);
7234 remove_edge (BRANCH_EDGE (entry_bb
));
7235 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7236 gsi
= gsi_last_bb (region
->exit
);
7237 gcc_assert (!gsi_end_p (gsi
)
7238 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7239 gsi_remove (&gsi
, true);
7242 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7245 gsi
= gsi_last_bb (exit_bb
);
7246 gcc_assert (!gsi_end_p (gsi
)
7247 && (gimple_code (gsi_stmt (gsi
))
7248 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7249 stmt
= gimple_build_return (NULL
);
7250 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7251 gsi_remove (&gsi
, true);
7254 /* Move the parallel region into CHILD_CFUN. */
7256 if (gimple_in_ssa_p (cfun
))
7258 init_tree_ssa (child_cfun
);
7259 init_ssa_operands (child_cfun
);
7260 child_cfun
->gimple_df
->in_ssa_p
= true;
7264 block
= gimple_block (entry_stmt
);
7266 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7268 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7271 basic_block dest_bb
= e2
->dest
;
7273 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7275 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7277 /* When the OMP expansion process cannot guarantee an up-to-date
7278 loop tree arrange for the child function to fixup loops. */
7279 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7280 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7282 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7283 num
= vec_safe_length (child_cfun
->local_decls
);
7284 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7286 t
= (*child_cfun
->local_decls
)[srcidx
];
7287 if (DECL_CONTEXT (t
) == cfun
->decl
)
7289 if (srcidx
!= dstidx
)
7290 (*child_cfun
->local_decls
)[dstidx
] = t
;
7294 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7296 /* Inform the callgraph about the new function. */
7297 child_cfun
->curr_properties
= cfun
->curr_properties
;
7298 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7299 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7300 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7301 node
->parallelized_function
= 1;
7302 cgraph_node::add_new_function (child_fn
, true);
7304 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7305 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7307 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7308 fixed in a following pass. */
7309 push_cfun (child_cfun
);
7311 assign_assembler_name_if_neeeded (child_fn
);
7314 optimize_omp_library_calls (entry_stmt
);
7315 cgraph_edge::rebuild_edges ();
7317 /* Some EH regions might become dead, see PR34608. If
7318 pass_cleanup_cfg isn't the first pass to happen with the
7319 new child, these dead EH edges might cause problems.
7320 Clean them up now. */
7321 if (flag_exceptions
)
7324 bool changed
= false;
7326 FOR_EACH_BB_FN (bb
, cfun
)
7327 changed
|= gimple_purge_dead_eh_edges (bb
);
7329 cleanup_tree_cfg ();
7331 if (gimple_in_ssa_p (cfun
))
7332 update_ssa (TODO_update_ssa
);
7333 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7334 verify_loop_structure ();
7337 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7339 omp_any_child_fn_dumped
= true;
7340 dump_function_header (dump_file
, child_fn
, dump_flags
);
7341 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7345 /* Emit a library call to launch the children threads. */
7347 expand_cilk_for_call (new_bb
,
7348 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7349 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7350 expand_parallel_call (region
, new_bb
,
7351 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7353 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7354 if (gimple_in_ssa_p (cfun
))
7355 update_ssa (TODO_update_ssa_only_virtuals
);
7358 /* Information about members of an OpenACC collapsed loop nest. */
7360 struct oacc_collapse
7362 tree base
; /* Base value. */
7363 tree iters
; /* Number of steps. */
7364 tree step
; /* step size. */
7367 /* Helper for expand_oacc_for. Determine collapsed loop information.
7368 Fill in COUNTS array. Emit any initialization code before GSI.
7369 Return the calculated outer loop bound of BOUND_TYPE. */
7372 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7373 gimple_stmt_iterator
*gsi
,
7374 oacc_collapse
*counts
, tree bound_type
)
7376 tree total
= build_int_cst (bound_type
, 1);
7379 gcc_assert (integer_onep (fd
->loop
.step
));
7380 gcc_assert (integer_zerop (fd
->loop
.n1
));
7382 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7384 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7386 tree iter_type
= TREE_TYPE (loop
->v
);
7387 tree diff_type
= iter_type
;
7388 tree plus_type
= iter_type
;
7390 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7392 if (POINTER_TYPE_P (iter_type
))
7393 plus_type
= sizetype
;
7394 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7395 diff_type
= signed_type_for (diff_type
);
7399 tree s
= loop
->step
;
7400 bool up
= loop
->cond_code
== LT_EXPR
;
7401 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7405 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7406 true, GSI_SAME_STMT
);
7407 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7408 true, GSI_SAME_STMT
);
7410 /* Convert the step, avoiding possible unsigned->signed overflow. */
7411 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7413 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7414 s
= fold_convert (diff_type
, s
);
7416 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7417 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7418 true, GSI_SAME_STMT
);
7420 /* Determine the range, avoiding possible unsigned->signed overflow. */
7421 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7422 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7423 fold_convert (plus_type
, negating
? b
: e
),
7424 fold_convert (plus_type
, negating
? e
: b
));
7425 expr
= fold_convert (diff_type
, expr
);
7427 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7428 tree range
= force_gimple_operand_gsi
7429 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7431 /* Determine number of iterations. */
7432 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7433 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7434 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7436 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7437 true, GSI_SAME_STMT
);
7439 counts
[ix
].base
= b
;
7440 counts
[ix
].iters
= iters
;
7441 counts
[ix
].step
= s
;
7443 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7444 fold_convert (bound_type
, iters
));
7450 /* Emit initializers for collapsed loop members. IVAR is the outer
7451 loop iteration variable, from which collapsed loop iteration values
7452 are calculated. COUNTS array has been initialized by
7453 expand_oacc_collapse_inits. */
7456 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7457 gimple_stmt_iterator
*gsi
,
7458 const oacc_collapse
*counts
, tree ivar
)
7460 tree ivar_type
= TREE_TYPE (ivar
);
7462 /* The most rapidly changing iteration variable is the innermost
7464 for (int ix
= fd
->collapse
; ix
--;)
7466 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7467 const oacc_collapse
*collapse
= &counts
[ix
];
7468 tree iter_type
= TREE_TYPE (loop
->v
);
7469 tree diff_type
= TREE_TYPE (collapse
->step
);
7470 tree plus_type
= iter_type
;
7471 enum tree_code plus_code
= PLUS_EXPR
;
7474 if (POINTER_TYPE_P (iter_type
))
7476 plus_code
= POINTER_PLUS_EXPR
;
7477 plus_type
= sizetype
;
7480 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7481 fold_convert (ivar_type
, collapse
->iters
));
7482 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7484 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7485 fold_convert (plus_type
, expr
));
7486 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7487 true, GSI_SAME_STMT
);
7488 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7489 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7493 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7494 fold_convert (ivar_type
, collapse
->iters
));
7495 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7496 true, GSI_SAME_STMT
);
7502 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7503 of the combined collapse > 1 loop constructs, generate code like:
7504 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7509 count3 = (adj + N32 - N31) / STEP3;
7510 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7515 count2 = (adj + N22 - N21) / STEP2;
7516 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7521 count1 = (adj + N12 - N11) / STEP1;
7522 count = count1 * count2 * count3;
7523 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7525 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7526 of the combined loop constructs, just initialize COUNTS array
7527 from the _looptemp_ clauses. */
7529 /* NOTE: It *could* be better to moosh all of the BBs together,
7530 creating one larger BB with all the computation and the unexpected
7531 jump at the end. I.e.
7533 bool zero3, zero2, zero1, zero;
7536 count3 = (N32 - N31) /[cl] STEP3;
7538 count2 = (N22 - N21) /[cl] STEP2;
7540 count1 = (N12 - N11) /[cl] STEP1;
7541 zero = zero3 || zero2 || zero1;
7542 count = count1 * count2 * count3;
7543 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7545 After all, we expect the zero=false, and thus we expect to have to
7546 evaluate all of the comparison expressions, so short-circuiting
7547 oughtn't be a win. Since the condition isn't protecting a
7548 denominator, we're not concerned about divide-by-zero, so we can
7549 fully evaluate count even if a numerator turned out to be wrong.
7551 It seems like putting this all together would create much better
7552 scheduling opportunities, and less pressure on the chip's branch
7556 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7557 basic_block
&entry_bb
, tree
*counts
,
7558 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7559 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7560 basic_block
&l2_dom_bb
)
7562 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7566 /* Collapsed loops need work for expansion into SSA form. */
7567 gcc_assert (!gimple_in_ssa_p (cfun
));
7569 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7570 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7572 gcc_assert (fd
->ordered
== 0);
7573 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7574 isn't supposed to be handled, as the inner loop doesn't
7576 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7577 OMP_CLAUSE__LOOPTEMP_
);
7578 gcc_assert (innerc
);
7579 for (i
= 0; i
< fd
->collapse
; i
++)
7581 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7582 OMP_CLAUSE__LOOPTEMP_
);
7583 gcc_assert (innerc
);
7585 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7587 counts
[0] = NULL_TREE
;
7592 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7594 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7595 counts
[i
] = NULL_TREE
;
7596 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7597 fold_convert (itype
, fd
->loops
[i
].n1
),
7598 fold_convert (itype
, fd
->loops
[i
].n2
));
7599 if (t
&& integer_zerop (t
))
7601 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7602 counts
[i
] = build_int_cst (type
, 0);
7606 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7608 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7610 if (i
>= fd
->collapse
&& counts
[i
])
7612 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7613 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7614 fold_convert (itype
, fd
->loops
[i
].n1
),
7615 fold_convert (itype
, fd
->loops
[i
].n2
)))
7616 == NULL_TREE
|| !integer_onep (t
)))
7620 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7621 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7622 true, GSI_SAME_STMT
);
7623 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7624 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7625 true, GSI_SAME_STMT
);
7626 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7627 NULL_TREE
, NULL_TREE
);
7628 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7629 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7630 expand_omp_regimplify_p
, NULL
, NULL
)
7631 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7632 expand_omp_regimplify_p
, NULL
, NULL
))
7634 *gsi
= gsi_for_stmt (cond_stmt
);
7635 gimple_regimplify_operands (cond_stmt
, gsi
);
7637 e
= split_block (entry_bb
, cond_stmt
);
7638 basic_block
&zero_iter_bb
7639 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7640 int &first_zero_iter
7641 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7642 if (zero_iter_bb
== NULL
)
7644 gassign
*assign_stmt
;
7645 first_zero_iter
= i
;
7646 zero_iter_bb
= create_empty_bb (entry_bb
);
7647 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7648 *gsi
= gsi_after_labels (zero_iter_bb
);
7649 if (i
< fd
->collapse
)
7650 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7651 build_zero_cst (type
));
7654 counts
[i
] = create_tmp_reg (type
, ".count");
7656 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7658 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7659 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7662 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7663 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7664 e
->flags
= EDGE_TRUE_VALUE
;
7665 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7666 if (l2_dom_bb
== NULL
)
7667 l2_dom_bb
= entry_bb
;
7669 *gsi
= gsi_last_bb (entry_bb
);
7672 if (POINTER_TYPE_P (itype
))
7673 itype
= signed_type_for (itype
);
7674 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7676 t
= fold_build2 (PLUS_EXPR
, itype
,
7677 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7678 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7679 fold_convert (itype
, fd
->loops
[i
].n2
));
7680 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7681 fold_convert (itype
, fd
->loops
[i
].n1
));
7682 /* ?? We could probably use CEIL_DIV_EXPR instead of
7683 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7684 generate the same code in the end because generically we
7685 don't know that the values involved must be negative for
7687 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7688 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7689 fold_build1 (NEGATE_EXPR
, itype
, t
),
7690 fold_build1 (NEGATE_EXPR
, itype
,
7691 fold_convert (itype
,
7692 fd
->loops
[i
].step
)));
7694 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7695 fold_convert (itype
, fd
->loops
[i
].step
));
7696 t
= fold_convert (type
, t
);
7697 if (TREE_CODE (t
) == INTEGER_CST
)
7701 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7702 counts
[i
] = create_tmp_reg (type
, ".count");
7703 expand_omp_build_assign (gsi
, counts
[i
], t
);
7705 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7710 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7711 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7717 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7719 V3 = N31 + (T % count3) * STEP3;
7721 V2 = N21 + (T % count2) * STEP2;
7723 V1 = N11 + T * STEP1;
7724 if this loop doesn't have an inner loop construct combined with it.
7725 If it does have an inner loop construct combined with it and the
7726 iteration count isn't known constant, store values from counts array
7727 into its _looptemp_ temporaries instead. */
7730 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7731 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7734 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7736 /* If fd->loop.n2 is constant, then no propagation of the counts
7737 is needed, they are constant. */
7738 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7741 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7742 ? gimple_omp_taskreg_clauses (inner_stmt
)
7743 : gimple_omp_for_clauses (inner_stmt
);
7744 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7745 isn't supposed to be handled, as the inner loop doesn't
7747 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7748 gcc_assert (innerc
);
7749 for (i
= 0; i
< fd
->collapse
; i
++)
7751 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7752 OMP_CLAUSE__LOOPTEMP_
);
7753 gcc_assert (innerc
);
7756 tree tem
= OMP_CLAUSE_DECL (innerc
);
7757 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7758 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7759 false, GSI_CONTINUE_LINKING
);
7760 gassign
*stmt
= gimple_build_assign (tem
, t
);
7761 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7767 tree type
= TREE_TYPE (fd
->loop
.v
);
7768 tree tem
= create_tmp_reg (type
, ".tem");
7769 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7770 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7772 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7774 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7776 if (POINTER_TYPE_P (vtype
))
7777 itype
= signed_type_for (vtype
);
7779 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7782 t
= fold_convert (itype
, t
);
7783 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7784 fold_convert (itype
, fd
->loops
[i
].step
));
7785 if (POINTER_TYPE_P (vtype
))
7786 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7788 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7789 t
= force_gimple_operand_gsi (gsi
, t
,
7790 DECL_P (fd
->loops
[i
].v
)
7791 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7793 GSI_CONTINUE_LINKING
);
7794 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7795 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7798 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7799 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7800 false, GSI_CONTINUE_LINKING
);
7801 stmt
= gimple_build_assign (tem
, t
);
7802 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7808 /* Helper function for expand_omp_for_*. Generate code like:
7811 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7815 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7822 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7823 basic_block body_bb
)
7825 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7827 gimple_stmt_iterator gsi
;
7833 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7835 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7837 bb
= create_empty_bb (last_bb
);
7838 add_bb_to_loop (bb
, last_bb
->loop_father
);
7839 gsi
= gsi_start_bb (bb
);
7841 if (i
< fd
->collapse
- 1)
7843 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7844 e
->probability
= REG_BR_PROB_BASE
/ 8;
7846 t
= fd
->loops
[i
+ 1].n1
;
7847 t
= force_gimple_operand_gsi (&gsi
, t
,
7848 DECL_P (fd
->loops
[i
+ 1].v
)
7849 && TREE_ADDRESSABLE (fd
->loops
[i
7852 GSI_CONTINUE_LINKING
);
7853 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7854 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7859 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7861 if (POINTER_TYPE_P (vtype
))
7862 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7864 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7865 t
= force_gimple_operand_gsi (&gsi
, t
,
7866 DECL_P (fd
->loops
[i
].v
)
7867 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7868 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7869 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7870 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7874 t
= fd
->loops
[i
].n2
;
7875 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7876 false, GSI_CONTINUE_LINKING
);
7877 tree v
= fd
->loops
[i
].v
;
7878 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7879 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7880 false, GSI_CONTINUE_LINKING
);
7881 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7882 stmt
= gimple_build_cond_empty (t
);
7883 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7884 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7885 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7888 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7896 /* Expand #pragma omp ordered depend(source). */
7899 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7900 tree
*counts
, location_t loc
)
7902 enum built_in_function source_ix
7903 = fd
->iter_type
== long_integer_type_node
7904 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7906 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7907 build_fold_addr_expr (counts
[fd
->ordered
]));
7908 gimple_set_location (g
, loc
);
7909 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7912 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7915 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7916 tree
*counts
, tree c
, location_t loc
)
7918 auto_vec
<tree
, 10> args
;
7919 enum built_in_function sink_ix
7920 = fd
->iter_type
== long_integer_type_node
7921 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7922 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7924 gimple_stmt_iterator gsi2
= *gsi
;
7925 bool warned_step
= false;
7927 for (i
= 0; i
< fd
->ordered
; i
++)
7929 off
= TREE_PURPOSE (deps
);
7930 if (!integer_zerop (off
))
7932 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7933 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7934 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7935 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7936 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7937 "lexically later iteration");
7940 deps
= TREE_CHAIN (deps
);
7942 /* If all offsets corresponding to the collapsed loops are zero,
7943 this depend clause can be ignored. FIXME: but there is still a
7944 flush needed. We need to emit one __sync_synchronize () for it
7945 though (perhaps conditionally)? Solve this together with the
7946 conservative dependence folding optimization.
7947 if (i >= fd->collapse)
7950 deps
= OMP_CLAUSE_DECL (c
);
7952 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7953 edge e2
= split_block_after_labels (e1
->dest
);
7955 *gsi
= gsi_after_labels (e1
->dest
);
7956 for (i
= 0; i
< fd
->ordered
; i
++)
7958 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7959 if (POINTER_TYPE_P (itype
))
7962 deps
= TREE_CHAIN (deps
);
7963 off
= TREE_PURPOSE (deps
);
7964 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7966 if (integer_zerop (off
))
7967 t
= boolean_true_node
;
7971 tree co
= fold_convert_loc (loc
, itype
, off
);
7972 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7974 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7975 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7976 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7977 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7980 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7981 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7982 fd
->loops
[i
].v
, co
);
7984 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7985 fd
->loops
[i
].v
, co
);
7986 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7988 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7989 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7992 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7995 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7996 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7999 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
8003 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
8007 off
= fold_convert_loc (loc
, itype
, off
);
8009 if (fd
->loops
[i
].cond_code
== LT_EXPR
8010 ? !integer_onep (fd
->loops
[i
].step
)
8011 : !integer_minus_onep (fd
->loops
[i
].step
))
8013 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8014 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
8015 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8018 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
8019 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
8020 build_int_cst (itype
, 0));
8021 if (integer_zerop (t
) && !warned_step
)
8023 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
8024 "in the iteration space");
8027 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
8031 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8037 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8038 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8039 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
8041 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8042 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
8043 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8046 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
8047 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8048 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
8049 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
8050 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8053 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
8055 if (i
< fd
->collapse
- 1)
8057 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
8062 off
= unshare_expr (off
);
8063 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
8064 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
8065 true, GSI_SAME_STMT
);
8068 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
8069 gimple_set_location (g
, loc
);
8070 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
8072 *gsi
= gsi_last_bb (e1
->src
);
8073 cond
= unshare_expr (cond
);
8074 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
8075 GSI_CONTINUE_LINKING
);
8076 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
8077 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
8078 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8079 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8080 e1
->flags
= EDGE_TRUE_VALUE
;
8081 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
8083 *gsi
= gsi_after_labels (e2
->dest
);
8086 /* Expand all #pragma omp ordered depend(source) and
8087 #pragma omp ordered depend(sink:...) constructs in the current
8088 #pragma omp for ordered(n) region. */
8091 expand_omp_ordered_source_sink (struct omp_region
*region
,
8092 struct omp_for_data
*fd
, tree
*counts
,
8093 basic_block cont_bb
)
8095 struct omp_region
*inner
;
8097 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
8098 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
8099 counts
[i
] = NULL_TREE
;
8100 else if (i
>= fd
->collapse
&& !cont_bb
)
8101 counts
[i
] = build_zero_cst (fd
->iter_type
);
8102 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
8103 && integer_onep (fd
->loops
[i
].step
))
8104 counts
[i
] = NULL_TREE
;
8106 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
8108 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
8109 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
8110 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
8112 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
8113 if (inner
->type
== GIMPLE_OMP_ORDERED
)
8115 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
8116 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
8117 location_t loc
= gimple_location (ord_stmt
);
8119 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8120 c
; c
= OMP_CLAUSE_CHAIN (c
))
8121 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
8124 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
8125 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8126 c
; c
= OMP_CLAUSE_CHAIN (c
))
8127 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8128 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
8129 gsi_remove (&gsi
, true);
8133 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8137 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
8138 basic_block cont_bb
, basic_block body_bb
,
8139 bool ordered_lastprivate
)
8141 if (fd
->ordered
== fd
->collapse
)
8146 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8147 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8149 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8150 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
8151 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
8152 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8153 size_int (i
- fd
->collapse
+ 1),
8154 NULL_TREE
, NULL_TREE
);
8155 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8160 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
8162 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
8163 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8164 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8165 fold_convert (type
, fd
->loops
[i
].n1
));
8167 expand_omp_build_assign (&gsi
, counts
[i
],
8168 build_zero_cst (fd
->iter_type
));
8169 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8170 size_int (i
- fd
->collapse
+ 1),
8171 NULL_TREE
, NULL_TREE
);
8172 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8173 if (!gsi_end_p (gsi
))
8176 gsi
= gsi_last_bb (body_bb
);
8177 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
8178 basic_block new_body
= e1
->dest
;
8179 if (body_bb
== cont_bb
)
8182 basic_block new_header
;
8183 if (EDGE_COUNT (cont_bb
->preds
) > 0)
8185 gsi
= gsi_last_bb (cont_bb
);
8186 if (POINTER_TYPE_P (type
))
8187 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8188 fold_convert (sizetype
,
8189 fd
->loops
[i
].step
));
8191 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8192 fold_convert (type
, fd
->loops
[i
].step
));
8193 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8196 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8197 build_int_cst (fd
->iter_type
, 1));
8198 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8203 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8204 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8205 t
= fold_convert (fd
->iter_type
, t
);
8206 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8207 true, GSI_SAME_STMT
);
8209 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8210 size_int (i
- fd
->collapse
+ 1),
8211 NULL_TREE
, NULL_TREE
);
8212 expand_omp_build_assign (&gsi
, aref
, t
);
8214 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8215 new_header
= e2
->dest
;
8218 new_header
= cont_bb
;
8219 gsi
= gsi_after_labels (new_header
);
8220 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8221 true, GSI_SAME_STMT
);
8223 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8224 true, NULL_TREE
, true, GSI_SAME_STMT
);
8225 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8226 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8227 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8230 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8231 e3
->flags
= EDGE_FALSE_VALUE
;
8232 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8233 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8234 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8236 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8237 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8241 struct loop
*loop
= alloc_loop ();
8242 loop
->header
= new_header
;
8243 loop
->latch
= e2
->src
;
8244 add_loop (loop
, body_bb
->loop_father
);
8248 /* If there are any lastprivate clauses and it is possible some loops
8249 might have zero iterations, ensure all the decls are initialized,
8250 otherwise we could crash evaluating C++ class iterators with lastprivate
8252 bool need_inits
= false;
8253 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8256 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8257 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8258 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8259 fold_convert (type
, fd
->loops
[i
].n1
));
8263 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8264 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8266 fold_convert (type
, fd
->loops
[i
].n1
),
8267 fold_convert (type
, fd
->loops
[i
].n2
));
8268 if (!integer_onep (this_cond
))
8276 /* A subroutine of expand_omp_for. Generate code for a parallel
8277 loop with any schedule. Given parameters:
8279 for (V = N1; V cond N2; V += STEP) BODY;
8281 where COND is "<" or ">", we generate pseudocode
8283 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8284 if (more) goto L0; else goto L3;
8291 if (V cond iend) goto L1; else goto L2;
8293 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8296 If this is a combined omp parallel loop, instead of the call to
8297 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8298 If this is gimple_omp_for_combined_p loop, then instead of assigning
8299 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8300 inner GIMPLE_OMP_FOR and V += STEP; and
8301 if (V cond iend) goto L1; else goto L2; are removed.
8303 For collapsed loops, given parameters:
8305 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8306 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8307 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8310 we generate pseudocode
8312 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8317 count3 = (adj + N32 - N31) / STEP3;
8318 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8323 count2 = (adj + N22 - N21) / STEP2;
8324 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8329 count1 = (adj + N12 - N11) / STEP1;
8330 count = count1 * count2 * count3;
8335 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8336 if (more) goto L0; else goto L3;
8340 V3 = N31 + (T % count3) * STEP3;
8342 V2 = N21 + (T % count2) * STEP2;
8344 V1 = N11 + T * STEP1;
8349 if (V < iend) goto L10; else goto L2;
8352 if (V3 cond3 N32) goto L1; else goto L11;
8356 if (V2 cond2 N22) goto L1; else goto L12;
8362 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8368 expand_omp_for_generic (struct omp_region
*region
,
8369 struct omp_for_data
*fd
,
8370 enum built_in_function start_fn
,
8371 enum built_in_function next_fn
,
8374 tree type
, istart0
, iend0
, iend
;
8375 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8376 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8377 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8378 gimple_stmt_iterator gsi
;
8379 gassign
*assign_stmt
;
8380 bool in_combined_parallel
= is_combined_parallel (region
);
8381 bool broken_loop
= region
->cont
== NULL
;
8383 tree
*counts
= NULL
;
8385 bool ordered_lastprivate
= false;
8387 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8388 gcc_assert (fd
->iter_type
== long_integer_type_node
8389 || !in_combined_parallel
);
8391 entry_bb
= region
->entry
;
8392 cont_bb
= region
->cont
;
8394 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8395 gcc_assert (broken_loop
8396 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8397 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8398 l1_bb
= single_succ (l0_bb
);
8401 l2_bb
= create_empty_bb (cont_bb
);
8402 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8403 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8405 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8409 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8410 exit_bb
= region
->exit
;
8412 gsi
= gsi_last_bb (entry_bb
);
8414 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8416 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8417 OMP_CLAUSE_LASTPRIVATE
))
8418 ordered_lastprivate
= false;
8419 if (fd
->collapse
> 1 || fd
->ordered
)
8421 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8422 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8424 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8425 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8426 zero_iter1_bb
, first_zero_iter1
,
8427 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8431 /* Some counts[i] vars might be uninitialized if
8432 some loop has zero iterations. But the body shouldn't
8433 be executed in that case, so just avoid uninit warnings. */
8434 for (i
= first_zero_iter1
;
8435 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8436 if (SSA_VAR_P (counts
[i
]))
8437 TREE_NO_WARNING (counts
[i
]) = 1;
8439 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8441 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8442 gsi
= gsi_last_bb (entry_bb
);
8443 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8444 get_immediate_dominator (CDI_DOMINATORS
,
8449 /* Some counts[i] vars might be uninitialized if
8450 some loop has zero iterations. But the body shouldn't
8451 be executed in that case, so just avoid uninit warnings. */
8452 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8453 if (SSA_VAR_P (counts
[i
]))
8454 TREE_NO_WARNING (counts
[i
]) = 1;
8456 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8460 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8462 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8463 gsi
= gsi_last_bb (entry_bb
);
8464 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8465 get_immediate_dominator
8466 (CDI_DOMINATORS
, zero_iter2_bb
));
8469 if (fd
->collapse
== 1)
8471 counts
[0] = fd
->loop
.n2
;
8472 fd
->loop
= fd
->loops
[0];
8476 type
= TREE_TYPE (fd
->loop
.v
);
8477 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8478 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8479 TREE_ADDRESSABLE (istart0
) = 1;
8480 TREE_ADDRESSABLE (iend0
) = 1;
8482 /* See if we need to bias by LLONG_MIN. */
8483 if (fd
->iter_type
== long_long_unsigned_type_node
8484 && TREE_CODE (type
) == INTEGER_TYPE
8485 && !TYPE_UNSIGNED (type
)
8486 && fd
->ordered
== 0)
8490 if (fd
->loop
.cond_code
== LT_EXPR
)
8493 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8497 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8500 if (TREE_CODE (n1
) != INTEGER_CST
8501 || TREE_CODE (n2
) != INTEGER_CST
8502 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8503 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8506 gimple_stmt_iterator gsif
= gsi
;
8509 tree arr
= NULL_TREE
;
8510 if (in_combined_parallel
)
8512 gcc_assert (fd
->ordered
== 0);
8513 /* In a combined parallel loop, emit a call to
8514 GOMP_loop_foo_next. */
8515 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8516 build_fold_addr_expr (istart0
),
8517 build_fold_addr_expr (iend0
));
8521 tree t0
, t1
, t2
, t3
, t4
;
8522 /* If this is not a combined parallel loop, emit a call to
8523 GOMP_loop_foo_start in ENTRY_BB. */
8524 t4
= build_fold_addr_expr (iend0
);
8525 t3
= build_fold_addr_expr (istart0
);
8528 t0
= build_int_cst (unsigned_type_node
,
8529 fd
->ordered
- fd
->collapse
+ 1);
8530 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8532 - fd
->collapse
+ 1),
8534 DECL_NAMELESS (arr
) = 1;
8535 TREE_ADDRESSABLE (arr
) = 1;
8536 TREE_STATIC (arr
) = 1;
8537 vec
<constructor_elt
, va_gc
> *v
;
8538 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8541 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8544 if (idx
== 0 && fd
->collapse
> 1)
8547 c
= counts
[idx
+ fd
->collapse
- 1];
8548 tree purpose
= size_int (idx
);
8549 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8550 if (TREE_CODE (c
) != INTEGER_CST
)
8551 TREE_STATIC (arr
) = 0;
8554 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8555 if (!TREE_STATIC (arr
))
8556 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8557 void_type_node
, arr
),
8558 true, NULL_TREE
, true, GSI_SAME_STMT
);
8559 t1
= build_fold_addr_expr (arr
);
8564 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8567 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8570 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8571 OMP_CLAUSE__LOOPTEMP_
);
8572 gcc_assert (innerc
);
8573 t0
= OMP_CLAUSE_DECL (innerc
);
8574 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8575 OMP_CLAUSE__LOOPTEMP_
);
8576 gcc_assert (innerc
);
8577 t1
= OMP_CLAUSE_DECL (innerc
);
8579 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8580 && TYPE_PRECISION (TREE_TYPE (t0
))
8581 != TYPE_PRECISION (fd
->iter_type
))
8583 /* Avoid casting pointers to integer of a different size. */
8584 tree itype
= signed_type_for (type
);
8585 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8586 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8590 t1
= fold_convert (fd
->iter_type
, t1
);
8591 t0
= fold_convert (fd
->iter_type
, t0
);
8595 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8596 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8599 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8603 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8604 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8606 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8607 5, t0
, t1
, t
, t3
, t4
);
8609 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8610 6, t0
, t1
, t2
, t
, t3
, t4
);
8612 else if (fd
->ordered
)
8613 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8616 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8617 5, t0
, t1
, t2
, t3
, t4
);
8625 /* The GOMP_loop_ull_*start functions have additional boolean
8626 argument, true for < loops and false for > loops.
8627 In Fortran, the C bool type can be different from
8628 boolean_type_node. */
8629 bfn_decl
= builtin_decl_explicit (start_fn
);
8630 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8631 t5
= build_int_cst (c_bool_type
,
8632 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8635 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8636 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8637 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8638 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8641 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8642 6, t5
, t0
, t1
, t2
, t3
, t4
);
8645 if (TREE_TYPE (t
) != boolean_type_node
)
8646 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8647 t
, build_int_cst (TREE_TYPE (t
), 0));
8648 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8649 true, GSI_SAME_STMT
);
8650 if (arr
&& !TREE_STATIC (arr
))
8652 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8653 TREE_THIS_VOLATILE (clobber
) = 1;
8654 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8657 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8659 /* Remove the GIMPLE_OMP_FOR statement. */
8660 gsi_remove (&gsi
, true);
8662 if (gsi_end_p (gsif
))
8663 gsif
= gsi_after_labels (gsi_bb (gsif
));
8666 /* Iteration setup for sequential loop goes in L0_BB. */
8667 tree startvar
= fd
->loop
.v
;
8668 tree endvar
= NULL_TREE
;
8670 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8672 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8673 && gimple_omp_for_kind (inner_stmt
)
8674 == GF_OMP_FOR_KIND_SIMD
);
8675 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8676 OMP_CLAUSE__LOOPTEMP_
);
8677 gcc_assert (innerc
);
8678 startvar
= OMP_CLAUSE_DECL (innerc
);
8679 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8680 OMP_CLAUSE__LOOPTEMP_
);
8681 gcc_assert (innerc
);
8682 endvar
= OMP_CLAUSE_DECL (innerc
);
8685 gsi
= gsi_start_bb (l0_bb
);
8687 if (fd
->ordered
&& fd
->collapse
== 1)
8688 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8689 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8691 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8692 if (fd
->ordered
&& fd
->collapse
== 1)
8694 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8695 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8696 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8699 t
= fold_convert (TREE_TYPE (startvar
), t
);
8700 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8706 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8707 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8708 t
= fold_convert (TREE_TYPE (startvar
), t
);
8710 t
= force_gimple_operand_gsi (&gsi
, t
,
8712 && TREE_ADDRESSABLE (startvar
),
8713 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8714 assign_stmt
= gimple_build_assign (startvar
, t
);
8715 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8718 if (fd
->ordered
&& fd
->collapse
== 1)
8719 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8720 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8722 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8723 if (fd
->ordered
&& fd
->collapse
== 1)
8725 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8726 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8727 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8730 t
= fold_convert (TREE_TYPE (startvar
), t
);
8731 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8737 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8738 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8739 t
= fold_convert (TREE_TYPE (startvar
), t
);
8741 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8742 false, GSI_CONTINUE_LINKING
);
8745 assign_stmt
= gimple_build_assign (endvar
, iend
);
8746 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8747 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8748 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8750 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8751 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8753 /* Handle linear clause adjustments. */
8754 tree itercnt
= NULL_TREE
;
8755 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8756 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8757 c
; c
= OMP_CLAUSE_CHAIN (c
))
8758 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8759 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8761 tree d
= OMP_CLAUSE_DECL (c
);
8762 bool is_ref
= is_reference (d
);
8763 tree t
= d
, a
, dest
;
8765 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8766 tree type
= TREE_TYPE (t
);
8767 if (POINTER_TYPE_P (type
))
8769 dest
= unshare_expr (t
);
8770 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8771 expand_omp_build_assign (&gsif
, v
, t
);
8772 if (itercnt
== NULL_TREE
)
8775 tree n1
= fd
->loop
.n1
;
8776 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8779 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8781 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8783 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8785 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8786 itercnt
, fd
->loop
.step
);
8787 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8789 GSI_CONTINUE_LINKING
);
8791 a
= fold_build2 (MULT_EXPR
, type
,
8792 fold_convert (type
, itercnt
),
8793 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8794 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8795 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8796 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8797 false, GSI_CONTINUE_LINKING
);
8798 assign_stmt
= gimple_build_assign (dest
, t
);
8799 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8801 if (fd
->collapse
> 1)
8802 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8806 /* Until now, counts array contained number of iterations or
8807 variable containing it for ith loop. From now on, we need
8808 those counts only for collapsed loops, and only for the 2nd
8809 till the last collapsed one. Move those one element earlier,
8810 we'll use counts[fd->collapse - 1] for the first source/sink
8811 iteration counter and so on and counts[fd->ordered]
8812 as the array holding the current counter values for
8814 if (fd
->collapse
> 1)
8815 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8819 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8821 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8823 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8824 fold_convert (type
, fd
->loops
[i
].n1
),
8825 fold_convert (type
, fd
->loops
[i
].n2
));
8826 if (!integer_onep (this_cond
))
8829 if (i
< fd
->ordered
)
8832 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8833 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8834 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8835 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8836 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8837 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8838 make_edge (cont_bb
, l1_bb
, 0);
8839 l2_bb
= create_empty_bb (cont_bb
);
8840 broken_loop
= false;
8843 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8844 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8845 ordered_lastprivate
);
8846 if (counts
[fd
->collapse
- 1])
8848 gcc_assert (fd
->collapse
== 1);
8849 gsi
= gsi_last_bb (l0_bb
);
8850 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8852 gsi
= gsi_last_bb (cont_bb
);
8853 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8854 build_int_cst (fd
->iter_type
, 1));
8855 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8856 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8857 size_zero_node
, NULL_TREE
, NULL_TREE
);
8858 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8859 t
= counts
[fd
->collapse
- 1];
8861 else if (fd
->collapse
> 1)
8865 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8866 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8867 t
= fold_convert (fd
->iter_type
, t
);
8869 gsi
= gsi_last_bb (l0_bb
);
8870 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8871 size_zero_node
, NULL_TREE
, NULL_TREE
);
8872 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8873 false, GSI_CONTINUE_LINKING
);
8874 expand_omp_build_assign (&gsi
, aref
, t
, true);
8879 /* Code to control the increment and predicate for the sequential
8880 loop goes in the CONT_BB. */
8881 gsi
= gsi_last_bb (cont_bb
);
8882 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8883 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8884 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8885 vback
= gimple_omp_continue_control_def (cont_stmt
);
8887 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8889 if (POINTER_TYPE_P (type
))
8890 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8892 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8893 t
= force_gimple_operand_gsi (&gsi
, t
,
8895 && TREE_ADDRESSABLE (vback
),
8896 NULL_TREE
, true, GSI_SAME_STMT
);
8897 assign_stmt
= gimple_build_assign (vback
, t
);
8898 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8900 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8902 if (fd
->collapse
> 1)
8906 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8907 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8908 t
= fold_convert (fd
->iter_type
, t
);
8910 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8911 counts
[fd
->ordered
], size_zero_node
,
8912 NULL_TREE
, NULL_TREE
);
8913 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8914 true, GSI_SAME_STMT
);
8915 expand_omp_build_assign (&gsi
, aref
, t
);
8918 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8919 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8921 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8922 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8925 /* Remove GIMPLE_OMP_CONTINUE. */
8926 gsi_remove (&gsi
, true);
8928 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8929 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8931 /* Emit code to get the next parallel iteration in L2_BB. */
8932 gsi
= gsi_start_bb (l2_bb
);
8934 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8935 build_fold_addr_expr (istart0
),
8936 build_fold_addr_expr (iend0
));
8937 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8938 false, GSI_CONTINUE_LINKING
);
8939 if (TREE_TYPE (t
) != boolean_type_node
)
8940 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8941 t
, build_int_cst (TREE_TYPE (t
), 0));
8942 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8943 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8946 /* Add the loop cleanup function. */
8947 gsi
= gsi_last_bb (exit_bb
);
8948 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8949 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8950 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8951 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8953 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8954 gcall
*call_stmt
= gimple_build_call (t
, 0);
8955 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8956 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8957 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8960 tree arr
= counts
[fd
->ordered
];
8961 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8962 TREE_THIS_VOLATILE (clobber
) = 1;
8963 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8966 gsi_remove (&gsi
, true);
8968 /* Connect the new blocks. */
8969 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8970 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8976 e
= find_edge (cont_bb
, l3_bb
);
8977 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8979 phis
= phi_nodes (l3_bb
);
8980 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8982 gimple
*phi
= gsi_stmt (gsi
);
8983 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8984 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8988 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8989 e
= find_edge (cont_bb
, l1_bb
);
8992 e
= BRANCH_EDGE (cont_bb
);
8993 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8995 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9000 else if (fd
->collapse
> 1)
9003 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9006 e
->flags
= EDGE_TRUE_VALUE
;
9009 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9010 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
9014 e
= find_edge (cont_bb
, l2_bb
);
9015 e
->flags
= EDGE_FALLTHRU
;
9017 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9019 if (gimple_in_ssa_p (cfun
))
9021 /* Add phis to the outer loop that connect to the phis in the inner,
9022 original loop, and move the loop entry value of the inner phi to
9023 the loop entry value of the outer phi. */
9025 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
9027 source_location locus
;
9029 gphi
*exit_phi
= psi
.phi ();
9031 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
9032 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
9034 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
9035 edge latch_to_l1
= find_edge (latch
, l1_bb
);
9037 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
9039 tree t
= gimple_phi_result (exit_phi
);
9040 tree new_res
= copy_ssa_name (t
, NULL
);
9041 nphi
= create_phi_node (new_res
, l0_bb
);
9043 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
9044 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
9045 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
9046 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
9047 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
9049 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
9050 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
9052 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
9056 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
9057 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
9058 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
9059 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
9060 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
9061 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
9062 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
9063 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
9065 /* We enter expand_omp_for_generic with a loop. This original loop may
9066 have its own loop struct, or it may be part of an outer loop struct
9067 (which may be the fake loop). */
9068 struct loop
*outer_loop
= entry_bb
->loop_father
;
9069 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
9071 add_bb_to_loop (l2_bb
, outer_loop
);
9073 /* We've added a new loop around the original loop. Allocate the
9074 corresponding loop struct. */
9075 struct loop
*new_loop
= alloc_loop ();
9076 new_loop
->header
= l0_bb
;
9077 new_loop
->latch
= l2_bb
;
9078 add_loop (new_loop
, outer_loop
);
9080 /* Allocate a loop structure for the original loop unless we already
9082 if (!orig_loop_has_loop_struct
9083 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9085 struct loop
*orig_loop
= alloc_loop ();
9086 orig_loop
->header
= l1_bb
;
9087 /* The loop may have multiple latches. */
9088 add_loop (orig_loop
, new_loop
);
9094 /* A subroutine of expand_omp_for. Generate code for a parallel
9095 loop with static schedule and no specified chunk size. Given
9098 for (V = N1; V cond N2; V += STEP) BODY;
9100 where COND is "<" or ">", we generate pseudocode
9102 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9107 if ((__typeof (V)) -1 > 0 && cond is >)
9108 n = -(adj + N2 - N1) / -STEP;
9110 n = (adj + N2 - N1) / STEP;
9113 if (threadid < tt) goto L3; else goto L4;
9118 s0 = q * threadid + tt;
9121 if (s0 >= e0) goto L2; else goto L0;
9127 if (V cond e) goto L1;
9132 expand_omp_for_static_nochunk (struct omp_region
*region
,
9133 struct omp_for_data
*fd
,
9136 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
9137 tree type
, itype
, vmain
, vback
;
9138 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
9139 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
9141 gimple_stmt_iterator gsi
;
9143 bool broken_loop
= region
->cont
== NULL
;
9144 tree
*counts
= NULL
;
9147 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9148 if (POINTER_TYPE_P (type
))
9149 itype
= signed_type_for (type
);
9151 entry_bb
= region
->entry
;
9152 cont_bb
= region
->cont
;
9153 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9154 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9155 gcc_assert (broken_loop
9156 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
9157 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
9158 body_bb
= single_succ (seq_start_bb
);
9161 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9162 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9163 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9165 exit_bb
= region
->exit
;
9167 /* Iteration space partitioning goes in ENTRY_BB. */
9168 gsi
= gsi_last_bb (entry_bb
);
9169 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9171 if (fd
->collapse
> 1)
9173 int first_zero_iter
= -1, dummy
= -1;
9174 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9176 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9177 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9178 fin_bb
, first_zero_iter
,
9179 dummy_bb
, dummy
, l2_dom_bb
);
9182 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9183 t
= integer_one_node
;
9185 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9186 fold_convert (type
, fd
->loop
.n1
),
9187 fold_convert (type
, fd
->loop
.n2
));
9188 if (fd
->collapse
== 1
9189 && TYPE_UNSIGNED (type
)
9190 && (t
== NULL_TREE
|| !integer_onep (t
)))
9192 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9193 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9194 true, GSI_SAME_STMT
);
9195 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9196 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9197 true, GSI_SAME_STMT
);
9198 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9199 NULL_TREE
, NULL_TREE
);
9200 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9201 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9202 expand_omp_regimplify_p
, NULL
, NULL
)
9203 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9204 expand_omp_regimplify_p
, NULL
, NULL
))
9206 gsi
= gsi_for_stmt (cond_stmt
);
9207 gimple_regimplify_operands (cond_stmt
, &gsi
);
9209 ep
= split_block (entry_bb
, cond_stmt
);
9210 ep
->flags
= EDGE_TRUE_VALUE
;
9211 entry_bb
= ep
->dest
;
9212 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9213 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9214 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9215 if (gimple_in_ssa_p (cfun
))
9217 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9218 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9219 !gsi_end_p (gpi
); gsi_next (&gpi
))
9221 gphi
*phi
= gpi
.phi ();
9222 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9223 ep
, UNKNOWN_LOCATION
);
9226 gsi
= gsi_last_bb (entry_bb
);
9229 switch (gimple_omp_for_kind (fd
->for_stmt
))
9231 case GF_OMP_FOR_KIND_FOR
:
9232 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9233 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9235 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9236 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9237 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9242 nthreads
= build_call_expr (nthreads
, 0);
9243 nthreads
= fold_convert (itype
, nthreads
);
9244 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9245 true, GSI_SAME_STMT
);
9246 threadid
= build_call_expr (threadid
, 0);
9247 threadid
= fold_convert (itype
, threadid
);
9248 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9249 true, GSI_SAME_STMT
);
9253 step
= fd
->loop
.step
;
9254 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9256 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9257 OMP_CLAUSE__LOOPTEMP_
);
9258 gcc_assert (innerc
);
9259 n1
= OMP_CLAUSE_DECL (innerc
);
9260 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9261 OMP_CLAUSE__LOOPTEMP_
);
9262 gcc_assert (innerc
);
9263 n2
= OMP_CLAUSE_DECL (innerc
);
9265 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9266 true, NULL_TREE
, true, GSI_SAME_STMT
);
9267 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9268 true, NULL_TREE
, true, GSI_SAME_STMT
);
9269 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9270 true, NULL_TREE
, true, GSI_SAME_STMT
);
9272 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9273 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9274 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9275 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9276 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9277 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9278 fold_build1 (NEGATE_EXPR
, itype
, t
),
9279 fold_build1 (NEGATE_EXPR
, itype
, step
));
9281 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9282 t
= fold_convert (itype
, t
);
9283 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9285 q
= create_tmp_reg (itype
, "q");
9286 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9287 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9288 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9290 tt
= create_tmp_reg (itype
, "tt");
9291 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9292 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9293 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9295 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9296 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9297 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9299 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9300 gsi
= gsi_last_bb (second_bb
);
9301 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9303 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9305 gassign
*assign_stmt
9306 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9307 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9309 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9310 gsi
= gsi_last_bb (third_bb
);
9311 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9313 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9314 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9315 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9317 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9318 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9320 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9321 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9323 /* Remove the GIMPLE_OMP_FOR statement. */
9324 gsi_remove (&gsi
, true);
9326 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9327 gsi
= gsi_start_bb (seq_start_bb
);
9329 tree startvar
= fd
->loop
.v
;
9330 tree endvar
= NULL_TREE
;
9332 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9334 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9335 ? gimple_omp_parallel_clauses (inner_stmt
)
9336 : gimple_omp_for_clauses (inner_stmt
);
9337 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9338 gcc_assert (innerc
);
9339 startvar
= OMP_CLAUSE_DECL (innerc
);
9340 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9341 OMP_CLAUSE__LOOPTEMP_
);
9342 gcc_assert (innerc
);
9343 endvar
= OMP_CLAUSE_DECL (innerc
);
9344 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9345 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9348 for (i
= 1; i
< fd
->collapse
; i
++)
9350 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9351 OMP_CLAUSE__LOOPTEMP_
);
9352 gcc_assert (innerc
);
9354 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9355 OMP_CLAUSE__LOOPTEMP_
);
9358 /* If needed (distribute parallel for with lastprivate),
9359 propagate down the total number of iterations. */
9360 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9362 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9363 GSI_CONTINUE_LINKING
);
9364 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9365 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9369 t
= fold_convert (itype
, s0
);
9370 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9371 if (POINTER_TYPE_P (type
))
9372 t
= fold_build_pointer_plus (n1
, t
);
9374 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9375 t
= fold_convert (TREE_TYPE (startvar
), t
);
9376 t
= force_gimple_operand_gsi (&gsi
, t
,
9378 && TREE_ADDRESSABLE (startvar
),
9379 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9380 assign_stmt
= gimple_build_assign (startvar
, t
);
9381 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9383 t
= fold_convert (itype
, e0
);
9384 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9385 if (POINTER_TYPE_P (type
))
9386 t
= fold_build_pointer_plus (n1
, t
);
9388 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9389 t
= fold_convert (TREE_TYPE (startvar
), t
);
9390 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9391 false, GSI_CONTINUE_LINKING
);
9394 assign_stmt
= gimple_build_assign (endvar
, e
);
9395 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9396 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9397 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9399 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9400 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9402 /* Handle linear clause adjustments. */
9403 tree itercnt
= NULL_TREE
;
9404 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9405 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9406 c
; c
= OMP_CLAUSE_CHAIN (c
))
9407 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9408 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9410 tree d
= OMP_CLAUSE_DECL (c
);
9411 bool is_ref
= is_reference (d
);
9412 tree t
= d
, a
, dest
;
9414 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9415 if (itercnt
== NULL_TREE
)
9417 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9419 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9420 fold_convert (itype
, n1
),
9421 fold_convert (itype
, fd
->loop
.n1
));
9422 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9423 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9424 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9426 GSI_CONTINUE_LINKING
);
9431 tree type
= TREE_TYPE (t
);
9432 if (POINTER_TYPE_P (type
))
9434 a
= fold_build2 (MULT_EXPR
, type
,
9435 fold_convert (type
, itercnt
),
9436 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9437 dest
= unshare_expr (t
);
9438 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9439 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9440 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9441 false, GSI_CONTINUE_LINKING
);
9442 assign_stmt
= gimple_build_assign (dest
, t
);
9443 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9445 if (fd
->collapse
> 1)
9446 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9450 /* The code controlling the sequential loop replaces the
9451 GIMPLE_OMP_CONTINUE. */
9452 gsi
= gsi_last_bb (cont_bb
);
9453 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9454 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9455 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9456 vback
= gimple_omp_continue_control_def (cont_stmt
);
9458 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9460 if (POINTER_TYPE_P (type
))
9461 t
= fold_build_pointer_plus (vmain
, step
);
9463 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9464 t
= force_gimple_operand_gsi (&gsi
, t
,
9466 && TREE_ADDRESSABLE (vback
),
9467 NULL_TREE
, true, GSI_SAME_STMT
);
9468 assign_stmt
= gimple_build_assign (vback
, t
);
9469 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9471 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9472 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9474 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9477 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9478 gsi_remove (&gsi
, true);
9480 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9481 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9484 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9485 gsi
= gsi_last_bb (exit_bb
);
9486 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9488 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9489 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9491 gsi_remove (&gsi
, true);
9493 /* Connect all the blocks. */
9494 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9495 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9496 ep
= find_edge (entry_bb
, second_bb
);
9497 ep
->flags
= EDGE_TRUE_VALUE
;
9498 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9499 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9500 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9504 ep
= find_edge (cont_bb
, body_bb
);
9507 ep
= BRANCH_EDGE (cont_bb
);
9508 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9510 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9515 else if (fd
->collapse
> 1)
9518 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9521 ep
->flags
= EDGE_TRUE_VALUE
;
9522 find_edge (cont_bb
, fin_bb
)->flags
9523 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9526 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9527 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9528 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9530 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9531 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9532 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9533 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9535 struct loop
*loop
= body_bb
->loop_father
;
9536 if (loop
!= entry_bb
->loop_father
)
9538 gcc_assert (loop
->header
== body_bb
);
9539 gcc_assert (broken_loop
9540 || loop
->latch
== region
->cont
9541 || single_pred (loop
->latch
) == region
->cont
);
9545 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9547 loop
= alloc_loop ();
9548 loop
->header
= body_bb
;
9549 if (collapse_bb
== NULL
)
9550 loop
->latch
= cont_bb
;
9551 add_loop (loop
, body_bb
->loop_father
);
9555 /* Return phi in E->DEST with ARG on edge E. */
9558 find_phi_with_arg_on_edge (tree arg
, edge e
)
9560 basic_block bb
= e
->dest
;
9562 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9566 gphi
*phi
= gpi
.phi ();
9567 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9574 /* A subroutine of expand_omp_for. Generate code for a parallel
9575 loop with static schedule and a specified chunk size. Given
9578 for (V = N1; V cond N2; V += STEP) BODY;
9580 where COND is "<" or ">", we generate pseudocode
9582 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9587 if ((__typeof (V)) -1 > 0 && cond is >)
9588 n = -(adj + N2 - N1) / -STEP;
9590 n = (adj + N2 - N1) / STEP;
9592 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9593 here so that V is defined
9594 if the loop is not entered
9596 s0 = (trip * nthreads + threadid) * CHUNK;
9597 e0 = min(s0 + CHUNK, n);
9598 if (s0 < n) goto L1; else goto L4;
9605 if (V cond e) goto L2; else goto L3;
9613 expand_omp_for_static_chunk (struct omp_region
*region
,
9614 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9616 tree n
, s0
, e0
, e
, t
;
9617 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9618 tree type
, itype
, vmain
, vback
, vextra
;
9619 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9620 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9621 gimple_stmt_iterator gsi
;
9623 bool broken_loop
= region
->cont
== NULL
;
9624 tree
*counts
= NULL
;
9627 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9628 if (POINTER_TYPE_P (type
))
9629 itype
= signed_type_for (type
);
9631 entry_bb
= region
->entry
;
9632 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9634 iter_part_bb
= se
->dest
;
9635 cont_bb
= region
->cont
;
9636 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9637 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9638 gcc_assert (broken_loop
9639 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9640 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9641 body_bb
= single_succ (seq_start_bb
);
9644 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9645 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9646 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9647 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9649 exit_bb
= region
->exit
;
9651 /* Trip and adjustment setup goes in ENTRY_BB. */
9652 gsi
= gsi_last_bb (entry_bb
);
9653 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9655 if (fd
->collapse
> 1)
9657 int first_zero_iter
= -1, dummy
= -1;
9658 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9660 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9661 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9662 fin_bb
, first_zero_iter
,
9663 dummy_bb
, dummy
, l2_dom_bb
);
9666 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9667 t
= integer_one_node
;
9669 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9670 fold_convert (type
, fd
->loop
.n1
),
9671 fold_convert (type
, fd
->loop
.n2
));
9672 if (fd
->collapse
== 1
9673 && TYPE_UNSIGNED (type
)
9674 && (t
== NULL_TREE
|| !integer_onep (t
)))
9676 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9677 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9678 true, GSI_SAME_STMT
);
9679 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9680 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9681 true, GSI_SAME_STMT
);
9682 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9683 NULL_TREE
, NULL_TREE
);
9684 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9685 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9686 expand_omp_regimplify_p
, NULL
, NULL
)
9687 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9688 expand_omp_regimplify_p
, NULL
, NULL
))
9690 gsi
= gsi_for_stmt (cond_stmt
);
9691 gimple_regimplify_operands (cond_stmt
, &gsi
);
9693 se
= split_block (entry_bb
, cond_stmt
);
9694 se
->flags
= EDGE_TRUE_VALUE
;
9695 entry_bb
= se
->dest
;
9696 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9697 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9698 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9699 if (gimple_in_ssa_p (cfun
))
9701 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9702 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9703 !gsi_end_p (gpi
); gsi_next (&gpi
))
9705 gphi
*phi
= gpi
.phi ();
9706 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9707 se
, UNKNOWN_LOCATION
);
9710 gsi
= gsi_last_bb (entry_bb
);
9713 switch (gimple_omp_for_kind (fd
->for_stmt
))
9715 case GF_OMP_FOR_KIND_FOR
:
9716 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9717 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9719 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9720 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9721 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9726 nthreads
= build_call_expr (nthreads
, 0);
9727 nthreads
= fold_convert (itype
, nthreads
);
9728 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9729 true, GSI_SAME_STMT
);
9730 threadid
= build_call_expr (threadid
, 0);
9731 threadid
= fold_convert (itype
, threadid
);
9732 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9733 true, GSI_SAME_STMT
);
9737 step
= fd
->loop
.step
;
9738 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9740 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9741 OMP_CLAUSE__LOOPTEMP_
);
9742 gcc_assert (innerc
);
9743 n1
= OMP_CLAUSE_DECL (innerc
);
9744 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9745 OMP_CLAUSE__LOOPTEMP_
);
9746 gcc_assert (innerc
);
9747 n2
= OMP_CLAUSE_DECL (innerc
);
9749 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9750 true, NULL_TREE
, true, GSI_SAME_STMT
);
9751 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9752 true, NULL_TREE
, true, GSI_SAME_STMT
);
9753 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9754 true, NULL_TREE
, true, GSI_SAME_STMT
);
9755 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9756 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9758 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9761 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9762 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9763 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9764 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9765 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9766 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9767 fold_build1 (NEGATE_EXPR
, itype
, t
),
9768 fold_build1 (NEGATE_EXPR
, itype
, step
));
9770 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9771 t
= fold_convert (itype
, t
);
9772 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9773 true, GSI_SAME_STMT
);
9775 trip_var
= create_tmp_reg (itype
, ".trip");
9776 if (gimple_in_ssa_p (cfun
))
9778 trip_init
= make_ssa_name (trip_var
);
9779 trip_main
= make_ssa_name (trip_var
);
9780 trip_back
= make_ssa_name (trip_var
);
9784 trip_init
= trip_var
;
9785 trip_main
= trip_var
;
9786 trip_back
= trip_var
;
9789 gassign
*assign_stmt
9790 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9791 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9793 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9794 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9795 if (POINTER_TYPE_P (type
))
9796 t
= fold_build_pointer_plus (n1
, t
);
9798 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9799 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9800 true, GSI_SAME_STMT
);
9802 /* Remove the GIMPLE_OMP_FOR. */
9803 gsi_remove (&gsi
, true);
9805 gimple_stmt_iterator gsif
= gsi
;
9807 /* Iteration space partitioning goes in ITER_PART_BB. */
9808 gsi
= gsi_last_bb (iter_part_bb
);
9810 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9811 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9812 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9813 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9814 false, GSI_CONTINUE_LINKING
);
9816 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9817 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9818 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9819 false, GSI_CONTINUE_LINKING
);
9821 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9822 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9824 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9825 gsi
= gsi_start_bb (seq_start_bb
);
9827 tree startvar
= fd
->loop
.v
;
9828 tree endvar
= NULL_TREE
;
9830 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9832 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9833 ? gimple_omp_parallel_clauses (inner_stmt
)
9834 : gimple_omp_for_clauses (inner_stmt
);
9835 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9836 gcc_assert (innerc
);
9837 startvar
= OMP_CLAUSE_DECL (innerc
);
9838 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9839 OMP_CLAUSE__LOOPTEMP_
);
9840 gcc_assert (innerc
);
9841 endvar
= OMP_CLAUSE_DECL (innerc
);
9842 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9843 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9846 for (i
= 1; i
< fd
->collapse
; i
++)
9848 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9849 OMP_CLAUSE__LOOPTEMP_
);
9850 gcc_assert (innerc
);
9852 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9853 OMP_CLAUSE__LOOPTEMP_
);
9856 /* If needed (distribute parallel for with lastprivate),
9857 propagate down the total number of iterations. */
9858 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9860 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9861 GSI_CONTINUE_LINKING
);
9862 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9863 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9868 t
= fold_convert (itype
, s0
);
9869 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9870 if (POINTER_TYPE_P (type
))
9871 t
= fold_build_pointer_plus (n1
, t
);
9873 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9874 t
= fold_convert (TREE_TYPE (startvar
), t
);
9875 t
= force_gimple_operand_gsi (&gsi
, t
,
9877 && TREE_ADDRESSABLE (startvar
),
9878 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9879 assign_stmt
= gimple_build_assign (startvar
, t
);
9880 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9882 t
= fold_convert (itype
, e0
);
9883 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9884 if (POINTER_TYPE_P (type
))
9885 t
= fold_build_pointer_plus (n1
, t
);
9887 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9888 t
= fold_convert (TREE_TYPE (startvar
), t
);
9889 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9890 false, GSI_CONTINUE_LINKING
);
9893 assign_stmt
= gimple_build_assign (endvar
, e
);
9894 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9895 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9896 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9898 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9899 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9901 /* Handle linear clause adjustments. */
9902 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9903 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9904 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9905 c
; c
= OMP_CLAUSE_CHAIN (c
))
9906 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9907 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9909 tree d
= OMP_CLAUSE_DECL (c
);
9910 bool is_ref
= is_reference (d
);
9911 tree t
= d
, a
, dest
;
9913 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9914 tree type
= TREE_TYPE (t
);
9915 if (POINTER_TYPE_P (type
))
9917 dest
= unshare_expr (t
);
9918 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9919 expand_omp_build_assign (&gsif
, v
, t
);
9920 if (itercnt
== NULL_TREE
)
9922 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9925 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9926 fold_convert (itype
, fd
->loop
.n1
));
9927 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9930 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9933 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9934 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9936 GSI_CONTINUE_LINKING
);
9941 a
= fold_build2 (MULT_EXPR
, type
,
9942 fold_convert (type
, itercnt
),
9943 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9944 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9945 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9946 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9947 false, GSI_CONTINUE_LINKING
);
9948 assign_stmt
= gimple_build_assign (dest
, t
);
9949 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9951 if (fd
->collapse
> 1)
9952 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9956 /* The code controlling the sequential loop goes in CONT_BB,
9957 replacing the GIMPLE_OMP_CONTINUE. */
9958 gsi
= gsi_last_bb (cont_bb
);
9959 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9960 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9961 vback
= gimple_omp_continue_control_def (cont_stmt
);
9963 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9965 if (POINTER_TYPE_P (type
))
9966 t
= fold_build_pointer_plus (vmain
, step
);
9968 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9969 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9970 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9971 true, GSI_SAME_STMT
);
9972 assign_stmt
= gimple_build_assign (vback
, t
);
9973 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9975 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9976 t
= build2 (EQ_EXPR
, boolean_type_node
,
9977 build_int_cst (itype
, 0),
9978 build_int_cst (itype
, 1));
9980 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9981 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9983 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9986 /* Remove GIMPLE_OMP_CONTINUE. */
9987 gsi_remove (&gsi
, true);
9989 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9990 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9992 /* Trip update code goes into TRIP_UPDATE_BB. */
9993 gsi
= gsi_start_bb (trip_update_bb
);
9995 t
= build_int_cst (itype
, 1);
9996 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9997 assign_stmt
= gimple_build_assign (trip_back
, t
);
9998 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10001 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10002 gsi
= gsi_last_bb (exit_bb
);
10003 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
10005 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
10006 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
10008 gsi_remove (&gsi
, true);
10010 /* Connect the new blocks. */
10011 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
10012 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
10016 se
= find_edge (cont_bb
, body_bb
);
10019 se
= BRANCH_EDGE (cont_bb
);
10020 gcc_assert (single_succ (se
->dest
) == body_bb
);
10022 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10027 else if (fd
->collapse
> 1)
10030 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10033 se
->flags
= EDGE_TRUE_VALUE
;
10034 find_edge (cont_bb
, trip_update_bb
)->flags
10035 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10037 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
10040 if (gimple_in_ssa_p (cfun
))
10048 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
10050 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10051 remove arguments of the phi nodes in fin_bb. We need to create
10052 appropriate phi nodes in iter_part_bb instead. */
10053 se
= find_edge (iter_part_bb
, fin_bb
);
10054 re
= single_succ_edge (trip_update_bb
);
10055 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
10056 ene
= single_succ_edge (entry_bb
);
10058 psi
= gsi_start_phis (fin_bb
);
10059 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
10060 gsi_next (&psi
), ++i
)
10063 source_location locus
;
10066 t
= gimple_phi_result (phi
);
10067 gcc_assert (t
== redirect_edge_var_map_result (vm
));
10069 if (!single_pred_p (fin_bb
))
10070 t
= copy_ssa_name (t
, phi
);
10072 nphi
= create_phi_node (t
, iter_part_bb
);
10074 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
10075 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
10077 /* A special case -- fd->loop.v is not yet computed in
10078 iter_part_bb, we need to use vextra instead. */
10079 if (t
== fd
->loop
.v
)
10081 add_phi_arg (nphi
, t
, ene
, locus
);
10082 locus
= redirect_edge_var_map_location (vm
);
10083 tree back_arg
= redirect_edge_var_map_def (vm
);
10084 add_phi_arg (nphi
, back_arg
, re
, locus
);
10085 edge ce
= find_edge (cont_bb
, body_bb
);
10088 ce
= BRANCH_EDGE (cont_bb
);
10089 gcc_assert (single_succ (ce
->dest
) == body_bb
);
10090 ce
= single_succ_edge (ce
->dest
);
10092 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
10093 gcc_assert (inner_loop_phi
!= NULL
);
10094 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
10095 find_edge (seq_start_bb
, body_bb
), locus
);
10097 if (!single_pred_p (fin_bb
))
10098 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
10100 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
10101 redirect_edge_var_map_clear (re
);
10102 if (single_pred_p (fin_bb
))
10105 psi
= gsi_start_phis (fin_bb
);
10106 if (gsi_end_p (psi
))
10108 remove_phi_node (&psi
, false);
10111 /* Make phi node for trip. */
10112 phi
= create_phi_node (trip_main
, iter_part_bb
);
10113 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
10115 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
10120 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
10121 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
10122 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
10123 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10124 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10125 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
10126 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
10127 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10128 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10132 struct loop
*loop
= body_bb
->loop_father
;
10133 struct loop
*trip_loop
= alloc_loop ();
10134 trip_loop
->header
= iter_part_bb
;
10135 trip_loop
->latch
= trip_update_bb
;
10136 add_loop (trip_loop
, iter_part_bb
->loop_father
);
10138 if (loop
!= entry_bb
->loop_father
)
10140 gcc_assert (loop
->header
== body_bb
);
10141 gcc_assert (loop
->latch
== region
->cont
10142 || single_pred (loop
->latch
) == region
->cont
);
10143 trip_loop
->inner
= loop
;
10147 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10149 loop
= alloc_loop ();
10150 loop
->header
= body_bb
;
10151 if (collapse_bb
== NULL
)
10152 loop
->latch
= cont_bb
;
10153 add_loop (loop
, trip_loop
);
10158 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10160 for (V = N1; V cond N2; V += STEP) BODY;
10162 where COND is "<" or ">" or "!=", we generate pseudocode
10164 for (ind_var = low; ind_var < high; ind_var++)
10166 V = n1 + (ind_var * STEP)
10171 In the above pseudocode, low and high are function parameters of the
10172 child function. In the function below, we are inserting a temp.
10173 variable that will be making a call to two OMP functions that will not be
10174 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10175 with _Cilk_for). These functions are replaced with low and high
10176 by the function that handles taskreg. */
10180 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10182 bool broken_loop
= region
->cont
== NULL
;
10183 basic_block entry_bb
= region
->entry
;
10184 basic_block cont_bb
= region
->cont
;
10186 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10187 gcc_assert (broken_loop
10188 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10189 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10190 basic_block l1_bb
, l2_bb
;
10194 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10195 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10196 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10197 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10201 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10202 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10203 l2_bb
= single_succ (l1_bb
);
10205 basic_block exit_bb
= region
->exit
;
10206 basic_block l2_dom_bb
= NULL
;
10208 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10210 /* Below statements until the "tree high_val = ..." are pseudo statements
10211 used to pass information to be used by expand_omp_taskreg.
10212 low_val and high_val will be replaced by the __low and __high
10213 parameter from the child function.
10215 The call_exprs part is a place-holder, it is mainly used
10216 to distinctly identify to the top-level part that this is
10217 where we should put low and high (reasoning given in header
10221 = gimple_omp_parallel_child_fn (
10222 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10223 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10224 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10226 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10228 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10231 gcc_assert (low_val
&& high_val
);
10233 tree type
= TREE_TYPE (low_val
);
10234 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10235 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10237 /* Not needed in SSA form right now. */
10238 gcc_assert (!gimple_in_ssa_p (cfun
));
10239 if (l2_dom_bb
== NULL
)
10243 tree n2
= high_val
;
10245 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10247 /* Replace the GIMPLE_OMP_FOR statement. */
10248 gsi_replace (&gsi
, stmt
, true);
10252 /* Code to control the increment goes in the CONT_BB. */
10253 gsi
= gsi_last_bb (cont_bb
);
10254 stmt
= gsi_stmt (gsi
);
10255 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10256 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10257 build_one_cst (type
));
10259 /* Replace GIMPLE_OMP_CONTINUE. */
10260 gsi_replace (&gsi
, stmt
, true);
10263 /* Emit the condition in L1_BB. */
10264 gsi
= gsi_after_labels (l1_bb
);
10265 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10266 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10268 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10269 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10270 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10272 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10273 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10274 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10275 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10277 /* The condition is always '<' since the runtime will fill in the low
10278 and high values. */
10279 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10280 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10282 /* Remove GIMPLE_OMP_RETURN. */
10283 gsi
= gsi_last_bb (exit_bb
);
10284 gsi_remove (&gsi
, true);
10286 /* Connect the new blocks. */
10287 remove_edge (FALLTHRU_EDGE (entry_bb
));
10292 remove_edge (BRANCH_EDGE (entry_bb
));
10293 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10295 e
= BRANCH_EDGE (l1_bb
);
10296 ne
= FALLTHRU_EDGE (l1_bb
);
10297 e
->flags
= EDGE_TRUE_VALUE
;
10301 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10303 ne
= single_succ_edge (l1_bb
);
10304 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10307 ne
->flags
= EDGE_FALSE_VALUE
;
10308 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10309 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10311 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10312 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10313 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10317 struct loop
*loop
= alloc_loop ();
10318 loop
->header
= l1_bb
;
10319 loop
->latch
= cont_bb
;
10320 add_loop (loop
, l1_bb
->loop_father
);
10321 loop
->safelen
= INT_MAX
;
10324 /* Pick the correct library function based on the precision of the
10325 induction variable type. */
10326 tree lib_fun
= NULL_TREE
;
10327 if (TYPE_PRECISION (type
) == 32)
10328 lib_fun
= cilk_for_32_fndecl
;
10329 else if (TYPE_PRECISION (type
) == 64)
10330 lib_fun
= cilk_for_64_fndecl
;
10332 gcc_unreachable ();
10334 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10336 /* WS_ARGS contains the library function flavor to call:
10337 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10338 user-defined grain value. If the user does not define one, then zero
10339 is passed in by the parser. */
10340 vec_alloc (region
->ws_args
, 2);
10341 region
->ws_args
->quick_push (lib_fun
);
10342 region
->ws_args
->quick_push (fd
->chunk_size
);
10345 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10346 loop. Given parameters:
10348 for (V = N1; V cond N2; V += STEP) BODY;
10350 where COND is "<" or ">", we generate pseudocode
10358 if (V cond N2) goto L0; else goto L2;
10361 For collapsed loops, given parameters:
10363 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10364 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10365 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10368 we generate pseudocode
10374 count3 = (adj + N32 - N31) / STEP3;
10379 count2 = (adj + N22 - N21) / STEP2;
10384 count1 = (adj + N12 - N11) / STEP1;
10385 count = count1 * count2 * count3;
10395 V2 += (V3 cond3 N32) ? 0 : STEP2;
10396 V3 = (V3 cond3 N32) ? V3 : N31;
10397 V1 += (V2 cond2 N22) ? 0 : STEP1;
10398 V2 = (V2 cond2 N22) ? V2 : N21;
10400 if (V < count) goto L0; else goto L2;
10406 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10409 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10410 gimple_stmt_iterator gsi
;
10413 bool broken_loop
= region
->cont
== NULL
;
10415 tree
*counts
= NULL
;
10417 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10418 OMP_CLAUSE_SAFELEN
);
10419 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10420 OMP_CLAUSE__SIMDUID_
);
10423 type
= TREE_TYPE (fd
->loop
.v
);
10424 entry_bb
= region
->entry
;
10425 cont_bb
= region
->cont
;
10426 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10427 gcc_assert (broken_loop
10428 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10429 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10432 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10433 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10434 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10435 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10439 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10440 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10441 l2_bb
= single_succ (l1_bb
);
10443 exit_bb
= region
->exit
;
10446 gsi
= gsi_last_bb (entry_bb
);
10448 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10449 /* Not needed in SSA form right now. */
10450 gcc_assert (!gimple_in_ssa_p (cfun
));
10451 if (fd
->collapse
> 1)
10453 int first_zero_iter
= -1, dummy
= -1;
10454 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10456 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10457 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10458 zero_iter_bb
, first_zero_iter
,
10459 dummy_bb
, dummy
, l2_dom_bb
);
10461 if (l2_dom_bb
== NULL
)
10466 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10468 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10469 OMP_CLAUSE__LOOPTEMP_
);
10470 gcc_assert (innerc
);
10471 n1
= OMP_CLAUSE_DECL (innerc
);
10472 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10473 OMP_CLAUSE__LOOPTEMP_
);
10474 gcc_assert (innerc
);
10475 n2
= OMP_CLAUSE_DECL (innerc
);
10476 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10477 fold_convert (type
, n1
));
10478 if (fd
->collapse
> 1)
10481 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10487 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10488 fold_convert (type
, fd
->loop
.n1
));
10489 if (fd
->collapse
> 1)
10490 for (i
= 0; i
< fd
->collapse
; i
++)
10492 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10493 if (POINTER_TYPE_P (itype
))
10494 itype
= signed_type_for (itype
);
10495 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10496 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10500 /* Remove the GIMPLE_OMP_FOR statement. */
10501 gsi_remove (&gsi
, true);
10505 /* Code to control the increment goes in the CONT_BB. */
10506 gsi
= gsi_last_bb (cont_bb
);
10507 stmt
= gsi_stmt (gsi
);
10508 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10510 if (POINTER_TYPE_P (type
))
10511 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10513 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10514 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10516 if (fd
->collapse
> 1)
10518 i
= fd
->collapse
- 1;
10519 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10521 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10522 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10526 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10527 fd
->loops
[i
].step
);
10528 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10529 fd
->loops
[i
].v
, t
);
10531 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10533 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10535 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10536 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10537 if (POINTER_TYPE_P (itype2
))
10538 itype2
= signed_type_for (itype2
);
10539 t
= build3 (COND_EXPR
, itype2
,
10540 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10542 fold_convert (itype
, fd
->loops
[i
].n2
)),
10543 build_int_cst (itype2
, 0),
10544 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10545 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10546 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10548 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10549 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10551 t
= build3 (COND_EXPR
, itype
,
10552 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10554 fold_convert (itype
, fd
->loops
[i
].n2
)),
10556 fold_convert (itype
, fd
->loops
[i
].n1
));
10557 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10561 /* Remove GIMPLE_OMP_CONTINUE. */
10562 gsi_remove (&gsi
, true);
10565 /* Emit the condition in L1_BB. */
10566 gsi
= gsi_start_bb (l1_bb
);
10568 t
= fold_convert (type
, n2
);
10569 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10570 false, GSI_CONTINUE_LINKING
);
10571 tree v
= fd
->loop
.v
;
10572 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10573 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10574 false, GSI_CONTINUE_LINKING
);
10575 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10576 cond_stmt
= gimple_build_cond_empty (t
);
10577 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10578 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10580 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10583 gsi
= gsi_for_stmt (cond_stmt
);
10584 gimple_regimplify_operands (cond_stmt
, &gsi
);
10587 /* Remove GIMPLE_OMP_RETURN. */
10588 gsi
= gsi_last_bb (exit_bb
);
10589 gsi_remove (&gsi
, true);
10591 /* Connect the new blocks. */
10592 remove_edge (FALLTHRU_EDGE (entry_bb
));
10596 remove_edge (BRANCH_EDGE (entry_bb
));
10597 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10599 e
= BRANCH_EDGE (l1_bb
);
10600 ne
= FALLTHRU_EDGE (l1_bb
);
10601 e
->flags
= EDGE_TRUE_VALUE
;
10605 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10607 ne
= single_succ_edge (l1_bb
);
10608 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10611 ne
->flags
= EDGE_FALSE_VALUE
;
10612 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10613 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10615 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10616 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10617 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10621 struct loop
*loop
= alloc_loop ();
10622 loop
->header
= l1_bb
;
10623 loop
->latch
= cont_bb
;
10624 add_loop (loop
, l1_bb
->loop_father
);
10625 if (safelen
== NULL_TREE
)
10626 loop
->safelen
= INT_MAX
;
10629 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10630 if (TREE_CODE (safelen
) != INTEGER_CST
)
10632 else if (!tree_fits_uhwi_p (safelen
)
10633 || tree_to_uhwi (safelen
) > INT_MAX
)
10634 loop
->safelen
= INT_MAX
;
10636 loop
->safelen
= tree_to_uhwi (safelen
);
10637 if (loop
->safelen
== 1)
10642 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10643 cfun
->has_simduid_loops
= true;
10645 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10647 if ((flag_tree_loop_vectorize
10648 || (!global_options_set
.x_flag_tree_loop_vectorize
10649 && !global_options_set
.x_flag_tree_vectorize
))
10650 && flag_tree_loop_optimize
10651 && loop
->safelen
> 1)
10653 loop
->force_vectorize
= true;
10654 cfun
->has_force_vectorize_loops
= true;
10658 cfun
->has_simduid_loops
= true;
10661 /* Taskloop construct is represented after gimplification with
10662 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10663 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10664 which should just compute all the needed loop temporaries
10665 for GIMPLE_OMP_TASK. */
10668 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10669 struct omp_for_data
*fd
,
10670 gimple
*inner_stmt
)
10672 tree type
, bias
= NULL_TREE
;
10673 basic_block entry_bb
, cont_bb
, exit_bb
;
10674 gimple_stmt_iterator gsi
;
10675 gassign
*assign_stmt
;
10676 tree
*counts
= NULL
;
10679 gcc_assert (inner_stmt
);
10680 gcc_assert (region
->cont
);
10681 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10682 && gimple_omp_task_taskloop_p (inner_stmt
));
10683 type
= TREE_TYPE (fd
->loop
.v
);
10685 /* See if we need to bias by LLONG_MIN. */
10686 if (fd
->iter_type
== long_long_unsigned_type_node
10687 && TREE_CODE (type
) == INTEGER_TYPE
10688 && !TYPE_UNSIGNED (type
))
10692 if (fd
->loop
.cond_code
== LT_EXPR
)
10695 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10699 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10702 if (TREE_CODE (n1
) != INTEGER_CST
10703 || TREE_CODE (n2
) != INTEGER_CST
10704 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10705 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10708 entry_bb
= region
->entry
;
10709 cont_bb
= region
->cont
;
10710 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10711 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10712 exit_bb
= region
->exit
;
10714 gsi
= gsi_last_bb (entry_bb
);
10715 gimple
*for_stmt
= gsi_stmt (gsi
);
10716 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10717 if (fd
->collapse
> 1)
10719 int first_zero_iter
= -1, dummy
= -1;
10720 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10722 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10723 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10724 zero_iter_bb
, first_zero_iter
,
10725 dummy_bb
, dummy
, l2_dom_bb
);
10729 /* Some counts[i] vars might be uninitialized if
10730 some loop has zero iterations. But the body shouldn't
10731 be executed in that case, so just avoid uninit warnings. */
10732 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10733 if (SSA_VAR_P (counts
[i
]))
10734 TREE_NO_WARNING (counts
[i
]) = 1;
10736 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10737 entry_bb
= e
->dest
;
10738 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10739 gsi
= gsi_last_bb (entry_bb
);
10740 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10741 get_immediate_dominator (CDI_DOMINATORS
,
10749 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10750 && TYPE_PRECISION (TREE_TYPE (t0
))
10751 != TYPE_PRECISION (fd
->iter_type
))
10753 /* Avoid casting pointers to integer of a different size. */
10754 tree itype
= signed_type_for (type
);
10755 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10756 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10760 t1
= fold_convert (fd
->iter_type
, t1
);
10761 t0
= fold_convert (fd
->iter_type
, t0
);
10765 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10766 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10769 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10770 OMP_CLAUSE__LOOPTEMP_
);
10771 gcc_assert (innerc
);
10772 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10773 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10774 gcc_assert (innerc
);
10775 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10776 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10778 gcc_assert (innerc
);
10779 for (i
= 1; i
< fd
->collapse
; i
++)
10781 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10782 OMP_CLAUSE__LOOPTEMP_
);
10783 gcc_assert (innerc
);
10785 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10786 OMP_CLAUSE__LOOPTEMP_
);
10789 /* If needed (inner taskloop has lastprivate clause), propagate
10790 down the total number of iterations. */
10791 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10793 GSI_CONTINUE_LINKING
);
10794 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10795 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10799 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10800 GSI_CONTINUE_LINKING
);
10801 assign_stmt
= gimple_build_assign (startvar
, t0
);
10802 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10804 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10805 GSI_CONTINUE_LINKING
);
10806 assign_stmt
= gimple_build_assign (endvar
, t1
);
10807 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10808 if (fd
->collapse
> 1)
10809 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10811 /* Remove the GIMPLE_OMP_FOR statement. */
10812 gsi
= gsi_for_stmt (for_stmt
);
10813 gsi_remove (&gsi
, true);
10815 gsi
= gsi_last_bb (cont_bb
);
10816 gsi_remove (&gsi
, true);
10818 gsi
= gsi_last_bb (exit_bb
);
10819 gsi_remove (&gsi
, true);
10821 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10822 remove_edge (BRANCH_EDGE (entry_bb
));
10823 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10824 remove_edge (BRANCH_EDGE (cont_bb
));
10825 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10826 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10827 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10830 /* Taskloop construct is represented after gimplification with
10831 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10832 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10833 GOMP_taskloop{,_ull} function arranges for each task to be given just
10834 a single range of iterations. */
10837 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10838 struct omp_for_data
*fd
,
10839 gimple
*inner_stmt
)
10841 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10842 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10843 basic_block fin_bb
;
10844 gimple_stmt_iterator gsi
;
10846 bool broken_loop
= region
->cont
== NULL
;
10847 tree
*counts
= NULL
;
10850 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10851 if (POINTER_TYPE_P (type
))
10852 itype
= signed_type_for (type
);
10854 /* See if we need to bias by LLONG_MIN. */
10855 if (fd
->iter_type
== long_long_unsigned_type_node
10856 && TREE_CODE (type
) == INTEGER_TYPE
10857 && !TYPE_UNSIGNED (type
))
10861 if (fd
->loop
.cond_code
== LT_EXPR
)
10864 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10868 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10871 if (TREE_CODE (n1
) != INTEGER_CST
10872 || TREE_CODE (n2
) != INTEGER_CST
10873 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10874 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10877 entry_bb
= region
->entry
;
10878 cont_bb
= region
->cont
;
10879 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10880 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10881 gcc_assert (broken_loop
10882 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10883 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10886 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10887 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10889 exit_bb
= region
->exit
;
10891 /* Iteration space partitioning goes in ENTRY_BB. */
10892 gsi
= gsi_last_bb (entry_bb
);
10893 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10895 if (fd
->collapse
> 1)
10897 int first_zero_iter
= -1, dummy
= -1;
10898 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10900 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10901 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10902 fin_bb
, first_zero_iter
,
10903 dummy_bb
, dummy
, l2_dom_bb
);
10907 t
= integer_one_node
;
10909 step
= fd
->loop
.step
;
10910 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10911 OMP_CLAUSE__LOOPTEMP_
);
10912 gcc_assert (innerc
);
10913 n1
= OMP_CLAUSE_DECL (innerc
);
10914 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10915 gcc_assert (innerc
);
10916 n2
= OMP_CLAUSE_DECL (innerc
);
10919 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10920 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10922 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10923 true, NULL_TREE
, true, GSI_SAME_STMT
);
10924 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10925 true, NULL_TREE
, true, GSI_SAME_STMT
);
10926 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10927 true, NULL_TREE
, true, GSI_SAME_STMT
);
10929 tree startvar
= fd
->loop
.v
;
10930 tree endvar
= NULL_TREE
;
10932 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10934 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10935 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10936 gcc_assert (innerc
);
10937 startvar
= OMP_CLAUSE_DECL (innerc
);
10938 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10939 OMP_CLAUSE__LOOPTEMP_
);
10940 gcc_assert (innerc
);
10941 endvar
= OMP_CLAUSE_DECL (innerc
);
10943 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10944 t
= force_gimple_operand_gsi (&gsi
, t
,
10946 && TREE_ADDRESSABLE (startvar
),
10947 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10948 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10949 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10951 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10952 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10953 false, GSI_CONTINUE_LINKING
);
10956 assign_stmt
= gimple_build_assign (endvar
, e
);
10957 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10958 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10959 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10961 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10962 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10964 if (fd
->collapse
> 1)
10965 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10969 /* The code controlling the sequential loop replaces the
10970 GIMPLE_OMP_CONTINUE. */
10971 gsi
= gsi_last_bb (cont_bb
);
10972 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10973 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10974 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10975 vback
= gimple_omp_continue_control_def (cont_stmt
);
10977 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10979 if (POINTER_TYPE_P (type
))
10980 t
= fold_build_pointer_plus (vmain
, step
);
10982 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10983 t
= force_gimple_operand_gsi (&gsi
, t
,
10985 && TREE_ADDRESSABLE (vback
),
10986 NULL_TREE
, true, GSI_SAME_STMT
);
10987 assign_stmt
= gimple_build_assign (vback
, t
);
10988 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10990 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10991 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10993 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10996 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10997 gsi_remove (&gsi
, true);
10999 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
11000 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
11003 /* Remove the GIMPLE_OMP_FOR statement. */
11004 gsi
= gsi_for_stmt (fd
->for_stmt
);
11005 gsi_remove (&gsi
, true);
11007 /* Remove the GIMPLE_OMP_RETURN statement. */
11008 gsi
= gsi_last_bb (exit_bb
);
11009 gsi_remove (&gsi
, true);
11011 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
11013 remove_edge (BRANCH_EDGE (entry_bb
));
11016 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
11017 region
->outer
->cont
= NULL
;
11020 /* Connect all the blocks. */
11023 ep
= find_edge (cont_bb
, body_bb
);
11024 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11029 else if (fd
->collapse
> 1)
11032 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
11035 ep
->flags
= EDGE_TRUE_VALUE
;
11036 find_edge (cont_bb
, fin_bb
)->flags
11037 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
11040 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
11041 recompute_dominator (CDI_DOMINATORS
, body_bb
));
11043 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
11044 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
11046 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
11048 struct loop
*loop
= alloc_loop ();
11049 loop
->header
= body_bb
;
11050 if (collapse_bb
== NULL
)
11051 loop
->latch
= cont_bb
;
11052 add_loop (loop
, body_bb
->loop_father
);
11056 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11057 partitioned loop. The lowering here is abstracted, in that the
11058 loop parameters are passed through internal functions, which are
11059 further lowered by oacc_device_lower, once we get to the target
11060 compiler. The loop is of the form:
11062 for (V = B; V LTGT E; V += S) {BODY}
11064 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11065 (constant 0 for no chunking) and we will have a GWV partitioning
11066 mask, specifying dimensions over which the loop is to be
11067 partitioned (see note below). We generate code that looks like:
11069 <entry_bb> [incoming FALL->body, BRANCH->exit]
11070 typedef signedintify (typeof (V)) T; // underlying signed integral type
11073 T DIR = LTGT == '<' ? +1 : -1;
11074 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11075 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11077 <head_bb> [created by splitting end of entry_bb]
11078 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11079 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11080 if (!(offset LTGT bound)) goto bottom_bb;
11082 <body_bb> [incoming]
11086 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11088 if (offset LTGT bound) goto body_bb; [*]
11090 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11092 if (chunk < chunk_max) goto head_bb;
11094 <exit_bb> [incoming]
11095 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11097 [*] Needed if V live at end of loop
11099 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11100 transition, and will be specified by a more general mechanism shortly.
11104 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
11106 tree v
= fd
->loop
.v
;
11107 enum tree_code cond_code
= fd
->loop
.cond_code
;
11108 enum tree_code plus_code
= PLUS_EXPR
;
11110 tree chunk_size
= integer_minus_one_node
;
11111 tree gwv
= integer_zero_node
;
11112 tree iter_type
= TREE_TYPE (v
);
11113 tree diff_type
= iter_type
;
11114 tree plus_type
= iter_type
;
11115 struct oacc_collapse
*counts
= NULL
;
11117 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
11118 == GF_OMP_FOR_KIND_OACC_LOOP
);
11119 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
11120 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
11122 if (POINTER_TYPE_P (iter_type
))
11124 plus_code
= POINTER_PLUS_EXPR
;
11125 plus_type
= sizetype
;
11127 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
11128 diff_type
= signed_type_for (diff_type
);
11130 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
11131 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
11132 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
11133 basic_block bottom_bb
= NULL
;
11135 /* entry_bb has two sucessors; the branch edge is to the exit
11136 block, fallthrough edge to body. */
11137 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
11138 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
11140 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11141 body_bb, or to a block whose only successor is the body_bb. Its
11142 fallthrough successor is the final block (same as the branch
11143 successor of the entry_bb). */
11146 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
11147 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
11149 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
11150 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
11153 gcc_assert (!gimple_in_ssa_p (cfun
));
11155 /* The exit block only has entry_bb and cont_bb as predecessors. */
11156 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
11159 tree chunk_max
= NULL_TREE
;
11160 tree bound
, offset
;
11161 tree step
= create_tmp_var (diff_type
, ".step");
11162 bool up
= cond_code
== LT_EXPR
;
11163 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
11164 bool chunking
= !gimple_in_ssa_p (cfun
);;
11167 /* SSA instances. */
11168 tree offset_incr
= NULL_TREE
;
11169 tree offset_init
= NULL_TREE
;
11171 gimple_stmt_iterator gsi
;
11177 edge split
, be
, fte
;
11179 /* Split the end of entry_bb to create head_bb. */
11180 split
= split_block (entry_bb
, last_stmt (entry_bb
));
11181 basic_block head_bb
= split
->dest
;
11182 entry_bb
= split
->src
;
11184 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11185 gsi
= gsi_last_bb (entry_bb
);
11186 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
11187 loc
= gimple_location (for_stmt
);
11189 if (gimple_in_ssa_p (cfun
))
11191 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11192 gcc_assert (integer_zerop (fd
->loop
.n1
));
11193 /* The SSA parallelizer does gang parallelism. */
11194 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11197 if (fd
->collapse
> 1)
11199 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11200 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11201 TREE_TYPE (fd
->loop
.n2
));
11203 if (SSA_VAR_P (fd
->loop
.n2
))
11205 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11206 true, GSI_SAME_STMT
);
11207 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11208 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11213 tree b
= fd
->loop
.n1
;
11214 tree e
= fd
->loop
.n2
;
11215 tree s
= fd
->loop
.step
;
11217 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11218 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11220 /* Convert the step, avoiding possible unsigned->signed overflow. */
11221 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11223 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11224 s
= fold_convert (diff_type
, s
);
11226 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11227 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11230 chunk_size
= integer_zero_node
;
11231 expr
= fold_convert (diff_type
, chunk_size
);
11232 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11233 NULL_TREE
, true, GSI_SAME_STMT
);
11234 /* Determine the range, avoiding possible unsigned->signed overflow. */
11235 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11236 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11237 fold_convert (plus_type
, negating
? b
: e
),
11238 fold_convert (plus_type
, negating
? e
: b
));
11239 expr
= fold_convert (diff_type
, expr
);
11241 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11242 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11243 NULL_TREE
, true, GSI_SAME_STMT
);
11245 chunk_no
= build_int_cst (diff_type
, 0);
11248 gcc_assert (!gimple_in_ssa_p (cfun
));
11251 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11252 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11254 ass
= gimple_build_assign (chunk_no
, expr
);
11255 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11257 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11258 build_int_cst (integer_type_node
,
11259 IFN_GOACC_LOOP_CHUNKS
),
11260 dir
, range
, s
, chunk_size
, gwv
);
11261 gimple_call_set_lhs (call
, chunk_max
);
11262 gimple_set_location (call
, loc
);
11263 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11266 chunk_size
= chunk_no
;
11268 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11269 build_int_cst (integer_type_node
,
11270 IFN_GOACC_LOOP_STEP
),
11271 dir
, range
, s
, chunk_size
, gwv
);
11272 gimple_call_set_lhs (call
, step
);
11273 gimple_set_location (call
, loc
);
11274 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11276 /* Remove the GIMPLE_OMP_FOR. */
11277 gsi_remove (&gsi
, true);
11279 /* Fixup edges from head_bb */
11280 be
= BRANCH_EDGE (head_bb
);
11281 fte
= FALLTHRU_EDGE (head_bb
);
11282 be
->flags
|= EDGE_FALSE_VALUE
;
11283 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11285 basic_block body_bb
= fte
->dest
;
11287 if (gimple_in_ssa_p (cfun
))
11289 gsi
= gsi_last_bb (cont_bb
);
11290 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11292 offset
= gimple_omp_continue_control_use (cont_stmt
);
11293 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11297 offset
= create_tmp_var (diff_type
, ".offset");
11298 offset_init
= offset_incr
= offset
;
11300 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11302 /* Loop offset & bound go into head_bb. */
11303 gsi
= gsi_start_bb (head_bb
);
11305 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11306 build_int_cst (integer_type_node
,
11307 IFN_GOACC_LOOP_OFFSET
),
11309 chunk_size
, gwv
, chunk_no
);
11310 gimple_call_set_lhs (call
, offset_init
);
11311 gimple_set_location (call
, loc
);
11312 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11314 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11315 build_int_cst (integer_type_node
,
11316 IFN_GOACC_LOOP_BOUND
),
11318 chunk_size
, gwv
, offset_init
);
11319 gimple_call_set_lhs (call
, bound
);
11320 gimple_set_location (call
, loc
);
11321 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11323 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11324 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11325 GSI_CONTINUE_LINKING
);
11327 /* V assignment goes into body_bb. */
11328 if (!gimple_in_ssa_p (cfun
))
11330 gsi
= gsi_start_bb (body_bb
);
11332 expr
= build2 (plus_code
, iter_type
, b
,
11333 fold_convert (plus_type
, offset
));
11334 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11335 true, GSI_SAME_STMT
);
11336 ass
= gimple_build_assign (v
, expr
);
11337 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11338 if (fd
->collapse
> 1)
11339 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11342 /* Loop increment goes into cont_bb. If this is not a loop, we
11343 will have spawned threads as if it was, and each one will
11344 execute one iteration. The specification is not explicit about
11345 whether such constructs are ill-formed or not, and they can
11346 occur, especially when noreturn routines are involved. */
11349 gsi
= gsi_last_bb (cont_bb
);
11350 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11351 loc
= gimple_location (cont_stmt
);
11353 /* Increment offset. */
11354 if (gimple_in_ssa_p (cfun
))
11355 expr
= build2 (plus_code
, iter_type
, offset
,
11356 fold_convert (plus_type
, step
));
11358 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11359 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11360 true, GSI_SAME_STMT
);
11361 ass
= gimple_build_assign (offset_incr
, expr
);
11362 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11363 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11364 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11366 /* Remove the GIMPLE_OMP_CONTINUE. */
11367 gsi_remove (&gsi
, true);
11369 /* Fixup edges from cont_bb */
11370 be
= BRANCH_EDGE (cont_bb
);
11371 fte
= FALLTHRU_EDGE (cont_bb
);
11372 be
->flags
|= EDGE_TRUE_VALUE
;
11373 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11377 /* Split the beginning of exit_bb to make bottom_bb. We
11378 need to insert a nop at the start, because splitting is
11379 after a stmt, not before. */
11380 gsi
= gsi_start_bb (exit_bb
);
11381 stmt
= gimple_build_nop ();
11382 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11383 split
= split_block (exit_bb
, stmt
);
11384 bottom_bb
= split
->src
;
11385 exit_bb
= split
->dest
;
11386 gsi
= gsi_last_bb (bottom_bb
);
11388 /* Chunk increment and test goes into bottom_bb. */
11389 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11390 build_int_cst (diff_type
, 1));
11391 ass
= gimple_build_assign (chunk_no
, expr
);
11392 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11394 /* Chunk test at end of bottom_bb. */
11395 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11396 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11397 GSI_CONTINUE_LINKING
);
11399 /* Fixup edges from bottom_bb. */
11400 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11401 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11405 gsi
= gsi_last_bb (exit_bb
);
11406 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11407 loc
= gimple_location (gsi_stmt (gsi
));
11409 if (!gimple_in_ssa_p (cfun
))
11411 /* Insert the final value of V, in case it is live. This is the
11412 value for the only thread that survives past the join. */
11413 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11414 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11415 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11416 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11417 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11418 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11419 true, GSI_SAME_STMT
);
11420 ass
= gimple_build_assign (v
, expr
);
11421 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11424 /* Remove the OMP_RETURN. */
11425 gsi_remove (&gsi
, true);
11429 /* We now have one or two nested loops. Update the loop
11431 struct loop
*parent
= entry_bb
->loop_father
;
11432 struct loop
*body
= body_bb
->loop_father
;
11436 struct loop
*chunk_loop
= alloc_loop ();
11437 chunk_loop
->header
= head_bb
;
11438 chunk_loop
->latch
= bottom_bb
;
11439 add_loop (chunk_loop
, parent
);
11440 parent
= chunk_loop
;
11442 else if (parent
!= body
)
11444 gcc_assert (body
->header
== body_bb
);
11445 gcc_assert (body
->latch
== cont_bb
11446 || single_pred (body
->latch
) == cont_bb
);
11452 struct loop
*body_loop
= alloc_loop ();
11453 body_loop
->header
= body_bb
;
11454 body_loop
->latch
= cont_bb
;
11455 add_loop (body_loop
, parent
);
11460 /* Expand the OMP loop defined by REGION. */
11463 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11465 struct omp_for_data fd
;
11466 struct omp_for_data_loop
*loops
;
11469 = (struct omp_for_data_loop
*)
11470 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11471 * sizeof (struct omp_for_data_loop
));
11472 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11474 region
->sched_kind
= fd
.sched_kind
;
11475 region
->sched_modifiers
= fd
.sched_modifiers
;
11477 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11478 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11479 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11482 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11483 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11484 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11487 /* If there isn't a continue then this is a degerate case where
11488 the introduction of abnormal edges during lowering will prevent
11489 original loops from being detected. Fix that up. */
11490 loops_state_set (LOOPS_NEED_FIXUP
);
11492 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11493 expand_omp_simd (region
, &fd
);
11494 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11495 expand_cilk_for (region
, &fd
);
11496 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11498 gcc_assert (!inner_stmt
);
11499 expand_oacc_for (region
, &fd
);
11501 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11503 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11504 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11506 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11508 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11509 && !fd
.have_ordered
)
11511 if (fd
.chunk_size
== NULL
)
11512 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11514 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11518 int fn_index
, start_ix
, next_ix
;
11520 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11521 == GF_OMP_FOR_KIND_FOR
);
11522 if (fd
.chunk_size
== NULL
11523 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11524 fd
.chunk_size
= integer_zero_node
;
11525 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11526 switch (fd
.sched_kind
)
11528 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11531 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11532 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11533 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11535 && !fd
.have_ordered
)
11537 fn_index
= 3 + fd
.sched_kind
;
11542 fn_index
= fd
.sched_kind
;
11546 fn_index
+= fd
.have_ordered
* 6;
11548 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11550 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11551 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11552 if (fd
.iter_type
== long_long_unsigned_type_node
)
11554 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11555 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11556 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11557 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11559 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11560 (enum built_in_function
) next_ix
, inner_stmt
);
11563 if (gimple_in_ssa_p (cfun
))
11564 update_ssa (TODO_update_ssa_only_virtuals
);
11568 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11570 v = GOMP_sections_start (n);
11587 v = GOMP_sections_next ();
11592 If this is a combined parallel sections, replace the call to
11593 GOMP_sections_start with call to GOMP_sections_next. */
11596 expand_omp_sections (struct omp_region
*region
)
11598 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11600 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11601 gimple_stmt_iterator si
, switch_si
;
11602 gomp_sections
*sections_stmt
;
11604 gomp_continue
*cont
;
11607 struct omp_region
*inner
;
11609 bool exit_reachable
= region
->cont
!= NULL
;
11611 gcc_assert (region
->exit
!= NULL
);
11612 entry_bb
= region
->entry
;
11613 l0_bb
= single_succ (entry_bb
);
11614 l1_bb
= region
->cont
;
11615 l2_bb
= region
->exit
;
11616 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11617 l2
= gimple_block_label (l2_bb
);
11620 /* This can happen if there are reductions. */
11621 len
= EDGE_COUNT (l0_bb
->succs
);
11622 gcc_assert (len
> 0);
11623 e
= EDGE_SUCC (l0_bb
, len
- 1);
11624 si
= gsi_last_bb (e
->dest
);
11627 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11628 l2
= gimple_block_label (e
->dest
);
11630 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11632 si
= gsi_last_bb (e
->dest
);
11634 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11636 l2
= gimple_block_label (e
->dest
);
11641 if (exit_reachable
)
11642 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11644 default_bb
= create_empty_bb (l0_bb
);
11646 /* We will build a switch() with enough cases for all the
11647 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11648 and a default case to abort if something goes wrong. */
11649 len
= EDGE_COUNT (l0_bb
->succs
);
11651 /* Use vec::quick_push on label_vec throughout, since we know the size
11653 auto_vec
<tree
> label_vec (len
);
11655 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11656 GIMPLE_OMP_SECTIONS statement. */
11657 si
= gsi_last_bb (entry_bb
);
11658 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11659 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11660 vin
= gimple_omp_sections_control (sections_stmt
);
11661 if (!is_combined_parallel (region
))
11663 /* If we are not inside a combined parallel+sections region,
11664 call GOMP_sections_start. */
11665 t
= build_int_cst (unsigned_type_node
, len
- 1);
11666 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11667 stmt
= gimple_build_call (u
, 1, t
);
11671 /* Otherwise, call GOMP_sections_next. */
11672 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11673 stmt
= gimple_build_call (u
, 0);
11675 gimple_call_set_lhs (stmt
, vin
);
11676 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11677 gsi_remove (&si
, true);
11679 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11681 switch_si
= gsi_last_bb (l0_bb
);
11682 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11683 if (exit_reachable
)
11685 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11686 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11687 vmain
= gimple_omp_continue_control_use (cont
);
11688 vnext
= gimple_omp_continue_control_def (cont
);
11696 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11697 label_vec
.quick_push (t
);
11700 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11701 for (inner
= region
->inner
, casei
= 1;
11703 inner
= inner
->next
, i
++, casei
++)
11705 basic_block s_entry_bb
, s_exit_bb
;
11707 /* Skip optional reduction region. */
11708 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11715 s_entry_bb
= inner
->entry
;
11716 s_exit_bb
= inner
->exit
;
11718 t
= gimple_block_label (s_entry_bb
);
11719 u
= build_int_cst (unsigned_type_node
, casei
);
11720 u
= build_case_label (u
, NULL
, t
);
11721 label_vec
.quick_push (u
);
11723 si
= gsi_last_bb (s_entry_bb
);
11724 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11725 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11726 gsi_remove (&si
, true);
11727 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11729 if (s_exit_bb
== NULL
)
11732 si
= gsi_last_bb (s_exit_bb
);
11733 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11734 gsi_remove (&si
, true);
11736 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11739 /* Error handling code goes in DEFAULT_BB. */
11740 t
= gimple_block_label (default_bb
);
11741 u
= build_case_label (NULL
, NULL
, t
);
11742 make_edge (l0_bb
, default_bb
, 0);
11743 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11745 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11746 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11747 gsi_remove (&switch_si
, true);
11749 si
= gsi_start_bb (default_bb
);
11750 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11751 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11753 if (exit_reachable
)
11757 /* Code to get the next section goes in L1_BB. */
11758 si
= gsi_last_bb (l1_bb
);
11759 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11761 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11762 stmt
= gimple_build_call (bfn_decl
, 0);
11763 gimple_call_set_lhs (stmt
, vnext
);
11764 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11765 gsi_remove (&si
, true);
11767 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11770 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11771 si
= gsi_last_bb (l2_bb
);
11772 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11773 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11774 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11775 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11777 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11778 stmt
= gimple_build_call (t
, 0);
11779 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11780 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11781 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11782 gsi_remove (&si
, true);
11784 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11788 /* Expand code for an OpenMP single directive. We've already expanded
11789 much of the code, here we simply place the GOMP_barrier call. */
11792 expand_omp_single (struct omp_region
*region
)
11794 basic_block entry_bb
, exit_bb
;
11795 gimple_stmt_iterator si
;
11797 entry_bb
= region
->entry
;
11798 exit_bb
= region
->exit
;
11800 si
= gsi_last_bb (entry_bb
);
11801 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11802 gsi_remove (&si
, true);
11803 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11805 si
= gsi_last_bb (exit_bb
);
11806 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11808 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11809 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11811 gsi_remove (&si
, true);
11812 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11816 /* Generic expansion for OpenMP synchronization directives: master,
11817 ordered and critical. All we need to do here is remove the entry
11818 and exit markers for REGION. */
11821 expand_omp_synch (struct omp_region
*region
)
11823 basic_block entry_bb
, exit_bb
;
11824 gimple_stmt_iterator si
;
11826 entry_bb
= region
->entry
;
11827 exit_bb
= region
->exit
;
11829 si
= gsi_last_bb (entry_bb
);
11830 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11831 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11832 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11833 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11834 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11835 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11836 gsi_remove (&si
, true);
11837 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11841 si
= gsi_last_bb (exit_bb
);
11842 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11843 gsi_remove (&si
, true);
11844 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11848 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11849 operation as a normal volatile load. */
11852 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11853 tree loaded_val
, int index
)
11855 enum built_in_function tmpbase
;
11856 gimple_stmt_iterator gsi
;
11857 basic_block store_bb
;
11860 tree decl
, call
, type
, itype
;
11862 gsi
= gsi_last_bb (load_bb
);
11863 stmt
= gsi_stmt (gsi
);
11864 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11865 loc
= gimple_location (stmt
);
11867 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11868 is smaller than word size, then expand_atomic_load assumes that the load
11869 is atomic. We could avoid the builtin entirely in this case. */
11871 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11872 decl
= builtin_decl_explicit (tmpbase
);
11873 if (decl
== NULL_TREE
)
11876 type
= TREE_TYPE (loaded_val
);
11877 itype
= TREE_TYPE (TREE_TYPE (decl
));
11879 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11880 build_int_cst (NULL
,
11881 gimple_omp_atomic_seq_cst_p (stmt
)
11883 : MEMMODEL_RELAXED
));
11884 if (!useless_type_conversion_p (type
, itype
))
11885 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11886 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11888 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11889 gsi_remove (&gsi
, true);
11891 store_bb
= single_succ (load_bb
);
11892 gsi
= gsi_last_bb (store_bb
);
11893 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11894 gsi_remove (&gsi
, true);
11896 if (gimple_in_ssa_p (cfun
))
11897 update_ssa (TODO_update_ssa_no_phi
);
11902 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11903 operation as a normal volatile store. */
11906 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11907 tree loaded_val
, tree stored_val
, int index
)
11909 enum built_in_function tmpbase
;
11910 gimple_stmt_iterator gsi
;
11911 basic_block store_bb
= single_succ (load_bb
);
11914 tree decl
, call
, type
, itype
;
11915 machine_mode imode
;
11918 gsi
= gsi_last_bb (load_bb
);
11919 stmt
= gsi_stmt (gsi
);
11920 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11922 /* If the load value is needed, then this isn't a store but an exchange. */
11923 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11925 gsi
= gsi_last_bb (store_bb
);
11926 stmt
= gsi_stmt (gsi
);
11927 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11928 loc
= gimple_location (stmt
);
11930 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11931 is smaller than word size, then expand_atomic_store assumes that the store
11932 is atomic. We could avoid the builtin entirely in this case. */
11934 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11935 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
11936 decl
= builtin_decl_explicit (tmpbase
);
11937 if (decl
== NULL_TREE
)
11940 type
= TREE_TYPE (stored_val
);
11942 /* Dig out the type of the function's second argument. */
11943 itype
= TREE_TYPE (decl
);
11944 itype
= TYPE_ARG_TYPES (itype
);
11945 itype
= TREE_CHAIN (itype
);
11946 itype
= TREE_VALUE (itype
);
11947 imode
= TYPE_MODE (itype
);
11949 if (exchange
&& !can_atomic_exchange_p (imode
, true))
11952 if (!useless_type_conversion_p (itype
, type
))
11953 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
11954 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
11955 build_int_cst (NULL
,
11956 gimple_omp_atomic_seq_cst_p (stmt
)
11958 : MEMMODEL_RELAXED
));
11961 if (!useless_type_conversion_p (type
, itype
))
11962 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11963 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11966 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11967 gsi_remove (&gsi
, true);
11969 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
11970 gsi
= gsi_last_bb (load_bb
);
11971 gsi_remove (&gsi
, true);
11973 if (gimple_in_ssa_p (cfun
))
11974 update_ssa (TODO_update_ssa_no_phi
);
11979 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11980 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
11981 size of the data type, and thus usable to find the index of the builtin
11982 decl. Returns false if the expression is not of the proper form. */
11985 expand_omp_atomic_fetch_op (basic_block load_bb
,
11986 tree addr
, tree loaded_val
,
11987 tree stored_val
, int index
)
11989 enum built_in_function oldbase
, newbase
, tmpbase
;
11990 tree decl
, itype
, call
;
11992 basic_block store_bb
= single_succ (load_bb
);
11993 gimple_stmt_iterator gsi
;
11996 enum tree_code code
;
11997 bool need_old
, need_new
;
11998 machine_mode imode
;
12001 /* We expect to find the following sequences:
12004 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12007 val = tmp OP something; (or: something OP tmp)
12008 GIMPLE_OMP_STORE (val)
12010 ???FIXME: Allow a more flexible sequence.
12011 Perhaps use data flow to pick the statements.
12015 gsi
= gsi_after_labels (store_bb
);
12016 stmt
= gsi_stmt (gsi
);
12017 loc
= gimple_location (stmt
);
12018 if (!is_gimple_assign (stmt
))
12021 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
12023 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
12024 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
12025 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
12026 gcc_checking_assert (!need_old
|| !need_new
);
12028 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
12031 /* Check for one of the supported fetch-op operations. */
12032 code
= gimple_assign_rhs_code (stmt
);
12036 case POINTER_PLUS_EXPR
:
12037 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
12038 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
12041 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
12042 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
12045 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
12046 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
12049 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
12050 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
12053 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
12054 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
12060 /* Make sure the expression is of the proper form. */
12061 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
12062 rhs
= gimple_assign_rhs2 (stmt
);
12063 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
12064 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
12065 rhs
= gimple_assign_rhs1 (stmt
);
12069 tmpbase
= ((enum built_in_function
)
12070 ((need_new
? newbase
: oldbase
) + index
+ 1));
12071 decl
= builtin_decl_explicit (tmpbase
);
12072 if (decl
== NULL_TREE
)
12074 itype
= TREE_TYPE (TREE_TYPE (decl
));
12075 imode
= TYPE_MODE (itype
);
12077 /* We could test all of the various optabs involved, but the fact of the
12078 matter is that (with the exception of i486 vs i586 and xadd) all targets
12079 that support any atomic operaton optab also implements compare-and-swap.
12080 Let optabs.c take care of expanding any compare-and-swap loop. */
12081 if (!can_compare_and_swap_p (imode
, true))
12084 gsi
= gsi_last_bb (load_bb
);
12085 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12087 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12088 It only requires that the operation happen atomically. Thus we can
12089 use the RELAXED memory model. */
12090 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
12091 fold_convert_loc (loc
, itype
, rhs
),
12092 build_int_cst (NULL
,
12093 seq_cst
? MEMMODEL_SEQ_CST
12094 : MEMMODEL_RELAXED
));
12096 if (need_old
|| need_new
)
12098 lhs
= need_old
? loaded_val
: stored_val
;
12099 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
12100 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
12103 call
= fold_convert_loc (loc
, void_type_node
, call
);
12104 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12105 gsi_remove (&gsi
, true);
12107 gsi
= gsi_last_bb (store_bb
);
12108 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
12109 gsi_remove (&gsi
, true);
12110 gsi
= gsi_last_bb (store_bb
);
12111 stmt
= gsi_stmt (gsi
);
12112 gsi_remove (&gsi
, true);
12114 if (gimple_in_ssa_p (cfun
))
12116 release_defs (stmt
);
12117 update_ssa (TODO_update_ssa_no_phi
);
12123 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12127 newval = rhs; // with oldval replacing *addr in rhs
12128 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12129 if (oldval != newval)
12132 INDEX is log2 of the size of the data type, and thus usable to find the
12133 index of the builtin decl. */
12136 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
12137 tree addr
, tree loaded_val
, tree stored_val
,
12140 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
12141 tree type
, itype
, cmpxchg
, iaddr
;
12142 gimple_stmt_iterator si
;
12143 basic_block loop_header
= single_succ (load_bb
);
12144 gimple
*phi
, *stmt
;
12146 enum built_in_function fncode
;
12148 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12149 order to use the RELAXED memory model effectively. */
12150 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12152 cmpxchg
= builtin_decl_explicit (fncode
);
12153 if (cmpxchg
== NULL_TREE
)
12155 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12156 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
12158 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
12161 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12162 si
= gsi_last_bb (load_bb
);
12163 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12165 /* For floating-point values, we'll need to view-convert them to integers
12166 so that we can perform the atomic compare and swap. Simplify the
12167 following code by always setting up the "i"ntegral variables. */
12168 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
12172 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
12175 = force_gimple_operand_gsi (&si
,
12176 fold_convert (TREE_TYPE (iaddr
), addr
),
12177 false, NULL_TREE
, true, GSI_SAME_STMT
);
12178 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
12179 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12180 loadedi
= create_tmp_var (itype
);
12181 if (gimple_in_ssa_p (cfun
))
12182 loadedi
= make_ssa_name (loadedi
);
12187 loadedi
= loaded_val
;
12190 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
12191 tree loaddecl
= builtin_decl_explicit (fncode
);
12194 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12195 build_call_expr (loaddecl
, 2, iaddr
,
12196 build_int_cst (NULL_TREE
,
12197 MEMMODEL_RELAXED
)));
12199 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12200 build_int_cst (TREE_TYPE (iaddr
), 0));
12203 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12206 /* Move the value to the LOADEDI temporary. */
12207 if (gimple_in_ssa_p (cfun
))
12209 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12210 phi
= create_phi_node (loadedi
, loop_header
);
12211 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12215 gsi_insert_before (&si
,
12216 gimple_build_assign (loadedi
, initial
),
12218 if (loadedi
!= loaded_val
)
12220 gimple_stmt_iterator gsi2
;
12223 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12224 gsi2
= gsi_start_bb (loop_header
);
12225 if (gimple_in_ssa_p (cfun
))
12228 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12229 true, GSI_SAME_STMT
);
12230 stmt
= gimple_build_assign (loaded_val
, x
);
12231 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12235 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12236 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12237 true, GSI_SAME_STMT
);
12240 gsi_remove (&si
, true);
12242 si
= gsi_last_bb (store_bb
);
12243 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12246 storedi
= stored_val
;
12249 force_gimple_operand_gsi (&si
,
12250 build1 (VIEW_CONVERT_EXPR
, itype
,
12251 stored_val
), true, NULL_TREE
, true,
12254 /* Build the compare&swap statement. */
12255 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12256 new_storedi
= force_gimple_operand_gsi (&si
,
12257 fold_convert (TREE_TYPE (loadedi
),
12260 true, GSI_SAME_STMT
);
12262 if (gimple_in_ssa_p (cfun
))
12263 old_vali
= loadedi
;
12266 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12267 stmt
= gimple_build_assign (old_vali
, loadedi
);
12268 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12270 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12271 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12274 /* Note that we always perform the comparison as an integer, even for
12275 floating point. This allows the atomic operation to properly
12276 succeed even with NaNs and -0.0. */
12277 stmt
= gimple_build_cond_empty
12278 (build2 (NE_EXPR
, boolean_type_node
,
12279 new_storedi
, old_vali
));
12280 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12283 e
= single_succ_edge (store_bb
);
12284 e
->flags
&= ~EDGE_FALLTHRU
;
12285 e
->flags
|= EDGE_FALSE_VALUE
;
12287 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12289 /* Copy the new value to loadedi (we already did that before the condition
12290 if we are not in SSA). */
12291 if (gimple_in_ssa_p (cfun
))
12293 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12294 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12297 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12298 gsi_remove (&si
, true);
12300 struct loop
*loop
= alloc_loop ();
12301 loop
->header
= loop_header
;
12302 loop
->latch
= store_bb
;
12303 add_loop (loop
, loop_header
->loop_father
);
12305 if (gimple_in_ssa_p (cfun
))
12306 update_ssa (TODO_update_ssa_no_phi
);
12311 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12313 GOMP_atomic_start ();
12315 GOMP_atomic_end ();
12317 The result is not globally atomic, but works so long as all parallel
12318 references are within #pragma omp atomic directives. According to
12319 responses received from omp@openmp.org, appears to be within spec.
12320 Which makes sense, since that's how several other compilers handle
12321 this situation as well.
12322 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12323 expanding. STORED_VAL is the operand of the matching
12324 GIMPLE_OMP_ATOMIC_STORE.
12327 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12328 loaded_val = *addr;
12331 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12332 *addr = stored_val;
12336 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12337 tree addr
, tree loaded_val
, tree stored_val
)
12339 gimple_stmt_iterator si
;
12343 si
= gsi_last_bb (load_bb
);
12344 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12346 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12347 t
= build_call_expr (t
, 0);
12348 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12350 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12351 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12352 gsi_remove (&si
, true);
12354 si
= gsi_last_bb (store_bb
);
12355 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12357 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12359 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12361 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12362 t
= build_call_expr (t
, 0);
12363 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12364 gsi_remove (&si
, true);
12366 if (gimple_in_ssa_p (cfun
))
12367 update_ssa (TODO_update_ssa_no_phi
);
12371 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12372 using expand_omp_atomic_fetch_op. If it failed, we try to
12373 call expand_omp_atomic_pipeline, and if it fails too, the
12374 ultimate fallback is wrapping the operation in a mutex
12375 (expand_omp_atomic_mutex). REGION is the atomic region built
12376 by build_omp_regions_1(). */
12379 expand_omp_atomic (struct omp_region
*region
)
12381 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12382 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12383 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12384 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12385 tree addr
= gimple_omp_atomic_load_rhs (load
);
12386 tree stored_val
= gimple_omp_atomic_store_val (store
);
12387 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12388 HOST_WIDE_INT index
;
12390 /* Make sure the type is one of the supported sizes. */
12391 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12392 index
= exact_log2 (index
);
12393 if (index
>= 0 && index
<= 4)
12395 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12397 /* __sync builtins require strict data alignment. */
12398 if (exact_log2 (align
) >= index
)
12401 if (loaded_val
== stored_val
12402 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12403 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12404 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12405 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12408 /* Atomic store. */
12409 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12410 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12411 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12412 && store_bb
== single_succ (load_bb
)
12413 && first_stmt (store_bb
) == store
12414 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12415 stored_val
, index
))
12418 /* When possible, use specialized atomic update functions. */
12419 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12420 && store_bb
== single_succ (load_bb
)
12421 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12422 loaded_val
, stored_val
, index
))
12425 /* If we don't have specialized __sync builtins, try and implement
12426 as a compare and swap loop. */
12427 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12428 loaded_val
, stored_val
, index
))
12433 /* The ultimate fallback is wrapping the operation in a mutex. */
12434 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12438 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12439 macro on gomp-constants.h. We do not check for overflow. */
12442 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12446 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12449 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12450 device
, build_int_cst (unsigned_type_node
,
12451 GOMP_LAUNCH_DEVICE_SHIFT
));
12452 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12457 /* Look for compute grid dimension clauses and convert to an attribute
12458 attached to FN. This permits the target-side code to (a) massage
12459 the dimensions, (b) emit that data and (c) optimize. Non-constant
12460 dimensions are pushed onto ARGS.
12462 The attribute value is a TREE_LIST. A set of dimensions is
12463 represented as a list of INTEGER_CST. Those that are runtime
12464 exprs are represented as an INTEGER_CST of zero.
12466 TOOO. Normally the attribute will just contain a single such list. If
12467 however it contains a list of lists, this will represent the use of
12468 device_type. Each member of the outer list is an assoc list of
12469 dimensions, keyed by the device type. The first entry will be the
12470 default. Well, that's the plan. */
12472 #define OACC_FN_ATTRIB "oacc function"
12474 /* Replace any existing oacc fn attribute with updated dimensions. */
12477 replace_oacc_fn_attrib (tree fn
, tree dims
)
12479 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12480 tree attribs
= DECL_ATTRIBUTES (fn
);
12482 /* If we happen to be present as the first attrib, drop it. */
12483 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12484 attribs
= TREE_CHAIN (attribs
);
12485 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12488 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12489 function attribute. Push any that are non-constant onto the ARGS
12490 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
12493 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
12495 /* Must match GOMP_DIM ordering. */
12496 static const omp_clause_code ids
[]
12497 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12498 OMP_CLAUSE_VECTOR_LENGTH
};
12500 tree dims
[GOMP_DIM_MAX
];
12501 tree attr
= NULL_TREE
;
12502 unsigned non_const
= 0;
12504 for (ix
= GOMP_DIM_MAX
; ix
--;)
12506 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12507 tree dim
= NULL_TREE
;
12510 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12512 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12514 dim
= integer_zero_node
;
12515 non_const
|= GOMP_DIM_MASK (ix
);
12517 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12520 replace_oacc_fn_attrib (fn
, attr
);
12524 /* Push a dynamic argument set. */
12525 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12526 NULL_TREE
, non_const
));
12527 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12528 if (non_const
& GOMP_DIM_MASK (ix
))
12529 args
->safe_push (dims
[ix
]);
12533 /* Process the routine's dimension clauess to generate an attribute
12534 value. Issue diagnostics as appropriate. We default to SEQ
12535 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12536 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12537 can have a loop partitioned on it. non-zero indicates
12538 yes, zero indicates no. By construction once a non-zero has been
12539 reached, further inner dimensions must also be non-zero. We set
12540 TREE_VALUE to zero for the dimensions that may be partitioned and
12541 1 for the other ones -- if a loop is (erroneously) spawned at
12542 an outer level, we don't want to try and partition it. */
12545 build_oacc_routine_dims (tree clauses
)
12547 /* Must match GOMP_DIM ordering. */
12548 static const omp_clause_code ids
[] =
12549 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12553 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12554 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12555 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12558 error_at (OMP_CLAUSE_LOCATION (clauses
),
12559 "multiple loop axes specified for routine");
12564 /* Default to SEQ. */
12566 level
= GOMP_DIM_MAX
;
12568 tree dims
= NULL_TREE
;
12570 for (ix
= GOMP_DIM_MAX
; ix
--;)
12571 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12572 build_int_cst (integer_type_node
, ix
< level
), dims
);
12577 /* Retrieve the oacc function attrib and return it. Non-oacc
12578 functions will return NULL. */
12581 get_oacc_fn_attrib (tree fn
)
12583 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12586 /* Extract an oacc execution dimension from FN. FN must be an
12587 offloaded function or routine that has already had its execution
12588 dimensions lowered to the target-specific values. */
12591 get_oacc_fn_dim_size (tree fn
, int axis
)
12593 tree attrs
= get_oacc_fn_attrib (fn
);
12595 gcc_assert (axis
< GOMP_DIM_MAX
);
12597 tree dims
= TREE_VALUE (attrs
);
12599 dims
= TREE_CHAIN (dims
);
12601 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12606 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12607 IFN_GOACC_DIM_SIZE call. */
12610 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12612 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12613 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12614 tree arg
= gimple_call_arg (stmt
, 0);
12615 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12617 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12621 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12625 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
12626 basic_block region_exit
)
12628 struct loop
*outer
= region_entry
->loop_father
;
12629 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
12631 /* Don't parallelize the kernels region if it contains more than one outer
12633 unsigned int nr_outer_loops
= 0;
12634 struct loop
*single_outer
= NULL
;
12635 for (struct loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
12637 gcc_assert (loop_outer (loop
) == outer
);
12639 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
12642 if (region_exit
!= NULL
12643 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
12647 single_outer
= loop
;
12649 if (nr_outer_loops
!= 1)
12652 for (struct loop
*loop
= single_outer
->inner
; loop
!= NULL
; loop
= loop
->inner
)
12656 /* Mark the loops in the region. */
12657 for (struct loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
12658 loop
->in_oacc_kernels_region
= true;
12661 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12663 struct GTY(()) grid_launch_attributes_trees
12665 tree kernel_dim_array_type
;
12666 tree kernel_lattrs_dimnum_decl
;
12667 tree kernel_lattrs_grid_decl
;
12668 tree kernel_lattrs_group_decl
;
12669 tree kernel_launch_attributes_type
;
12672 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
12674 /* Create types used to pass kernel launch attributes to target. */
12677 grid_create_kernel_launch_attr_types (void)
12679 if (grid_attr_trees
)
12681 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
12683 tree dim_arr_index_type
12684 = build_index_type (build_int_cst (integer_type_node
, 2));
12685 grid_attr_trees
->kernel_dim_array_type
12686 = build_array_type (uint32_type_node
, dim_arr_index_type
);
12688 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
12689 grid_attr_trees
->kernel_lattrs_dimnum_decl
12690 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
12692 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
12694 grid_attr_trees
->kernel_lattrs_grid_decl
12695 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
12696 grid_attr_trees
->kernel_dim_array_type
);
12697 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
12698 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
12699 grid_attr_trees
->kernel_lattrs_group_decl
12700 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
12701 grid_attr_trees
->kernel_dim_array_type
);
12702 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
12703 = grid_attr_trees
->kernel_lattrs_grid_decl
;
12704 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
12705 "__gomp_kernel_launch_attributes",
12706 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
12709 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12710 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12711 of type uint32_type_node. */
12714 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
12715 tree fld_decl
, int index
, tree value
)
12717 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
12718 build3 (COMPONENT_REF
,
12719 grid_attr_trees
->kernel_dim_array_type
,
12720 range_var
, fld_decl
, NULL_TREE
),
12721 build_int_cst (integer_type_node
, index
),
12722 NULL_TREE
, NULL_TREE
);
12723 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
12726 /* Return a tree representation of a pointer to a structure with grid and
12727 work-group size information. Statements filling that information will be
12728 inserted before GSI, TGT_STMT is the target statement which has the
12729 necessary information in it. */
12732 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
12733 gomp_target
*tgt_stmt
)
12735 grid_create_kernel_launch_attr_types ();
12736 tree u32_one
= build_one_cst (uint32_type_node
);
12737 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
12738 "__kernel_launch_attrs");
12740 unsigned max_dim
= 0;
12741 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
12743 clause
= OMP_CLAUSE_CHAIN (clause
))
12745 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
12748 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
12749 max_dim
= MAX (dim
, max_dim
);
12751 grid_insert_store_range_dim (gsi
, lattrs
,
12752 grid_attr_trees
->kernel_lattrs_grid_decl
,
12753 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
12754 grid_insert_store_range_dim (gsi
, lattrs
,
12755 grid_attr_trees
->kernel_lattrs_group_decl
,
12756 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
12759 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
12760 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
12761 /* At this moment we cannot gridify a loop with a collapse clause. */
12762 /* TODO: Adjust when we support bigger collapse. */
12763 gcc_assert (max_dim
== 0);
12764 gsi_insert_before (gsi
, gimple_build_assign (dimref
, u32_one
), GSI_SAME_STMT
);
12765 TREE_ADDRESSABLE (lattrs
) = 1;
12766 return build_fold_addr_expr (lattrs
);
12769 /* Build target argument identifier from the DEVICE identifier, value
12770 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12773 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
12775 tree t
= build_int_cst (integer_type_node
, device
);
12776 if (subseqent_param
)
12777 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12778 build_int_cst (integer_type_node
,
12779 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
12780 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12781 build_int_cst (integer_type_node
, id
));
12785 /* Like above but return it in type that can be directly stored as an element
12786 of the argument array. */
12789 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
12791 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
12792 return fold_convert (ptr_type_node
, t
);
12795 /* Return a target argument consisting of DEVICE identifier, value identifier
12796 ID, and the actual VALUE. */
12799 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
12802 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
12803 fold_convert (integer_type_node
, value
),
12804 build_int_cst (unsigned_type_node
,
12805 GOMP_TARGET_ARG_VALUE_SHIFT
));
12806 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12807 get_target_argument_identifier_1 (device
, false, id
));
12808 t
= fold_convert (ptr_type_node
, t
);
12809 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
12812 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12813 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12814 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12818 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
12819 int id
, tree value
, vec
<tree
> *args
)
12821 if (tree_fits_shwi_p (value
)
12822 && tree_to_shwi (value
) > -(1 << 15)
12823 && tree_to_shwi (value
) < (1 << 15))
12824 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
12827 args
->quick_push (get_target_argument_identifier (device
, true, id
));
12828 value
= fold_convert (ptr_type_node
, value
);
12829 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
12831 args
->quick_push (value
);
12835 /* Create an array of arguments that is then passed to GOMP_target. */
12838 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
12840 auto_vec
<tree
, 6> args
;
12841 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
12842 tree t
, c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12844 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
12846 t
= integer_minus_one_node
;
12847 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12848 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
12850 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12852 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
12854 t
= integer_minus_one_node
;
12855 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12856 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
12859 /* Add HSA-specific grid sizes, if available. */
12860 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
12861 OMP_CLAUSE__GRIDDIM_
))
12863 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true,
12864 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
);
12865 args
.quick_push (t
);
12866 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
12869 /* Produce more, perhaps device specific, arguments here. */
12871 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
12872 args
.length () + 1),
12873 ".omp_target_args");
12874 for (unsigned i
= 0; i
< args
.length (); i
++)
12876 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12877 build_int_cst (integer_type_node
, i
),
12878 NULL_TREE
, NULL_TREE
);
12879 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
12882 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12883 build_int_cst (integer_type_node
, args
.length ()),
12884 NULL_TREE
, NULL_TREE
);
12885 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
12887 TREE_ADDRESSABLE (argarray
) = 1;
12888 return build_fold_addr_expr (argarray
);
12891 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12894 expand_omp_target (struct omp_region
*region
)
12896 basic_block entry_bb
, exit_bb
, new_bb
;
12897 struct function
*child_cfun
;
12898 tree child_fn
, block
, t
;
12899 gimple_stmt_iterator gsi
;
12900 gomp_target
*entry_stmt
;
12903 bool offloaded
, data_region
;
12905 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
12906 new_bb
= region
->entry
;
12908 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
12909 switch (gimple_omp_target_kind (entry_stmt
))
12911 case GF_OMP_TARGET_KIND_REGION
:
12912 case GF_OMP_TARGET_KIND_UPDATE
:
12913 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12914 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12915 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12916 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12917 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12918 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12919 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
12920 data_region
= false;
12922 case GF_OMP_TARGET_KIND_DATA
:
12923 case GF_OMP_TARGET_KIND_OACC_DATA
:
12924 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
12925 data_region
= true;
12928 gcc_unreachable ();
12931 child_fn
= NULL_TREE
;
12935 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
12936 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
12939 /* Supported by expand_omp_taskreg, but not here. */
12940 if (child_cfun
!= NULL
)
12941 gcc_checking_assert (!child_cfun
->cfg
);
12942 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
12944 entry_bb
= region
->entry
;
12945 exit_bb
= region
->exit
;
12947 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
12948 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
12952 unsigned srcidx
, dstidx
, num
;
12954 /* If the offloading region needs data sent from the parent
12955 function, then the very first statement (except possible
12956 tree profile counter updates) of the offloading body
12957 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
12958 &.OMP_DATA_O is passed as an argument to the child function,
12959 we need to replace it with the argument as seen by the child
12962 In most cases, this will end up being the identity assignment
12963 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
12964 a function call that has been inlined, the original PARM_DECL
12965 .OMP_DATA_I may have been converted into a different local
12966 variable. In which case, we need to keep the assignment. */
12967 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
12970 basic_block entry_succ_bb
= single_succ (entry_bb
);
12971 gimple_stmt_iterator gsi
;
12973 gimple
*tgtcopy_stmt
= NULL
;
12974 tree sender
= TREE_VEC_ELT (data_arg
, 0);
12976 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
12978 gcc_assert (!gsi_end_p (gsi
));
12979 stmt
= gsi_stmt (gsi
);
12980 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
12983 if (gimple_num_ops (stmt
) == 2)
12985 tree arg
= gimple_assign_rhs1 (stmt
);
12987 /* We're ignoring the subcode because we're
12988 effectively doing a STRIP_NOPS. */
12990 if (TREE_CODE (arg
) == ADDR_EXPR
12991 && TREE_OPERAND (arg
, 0) == sender
)
12993 tgtcopy_stmt
= stmt
;
12999 gcc_assert (tgtcopy_stmt
!= NULL
);
13000 arg
= DECL_ARGUMENTS (child_fn
);
13002 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
13003 gsi_remove (&gsi
, true);
13006 /* Declare local variables needed in CHILD_CFUN. */
13007 block
= DECL_INITIAL (child_fn
);
13008 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
13009 /* The gimplifier could record temporaries in the offloading block
13010 rather than in containing function's local_decls chain,
13011 which would mean cgraph missed finalizing them. Do it now. */
13012 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
13013 if (TREE_CODE (t
) == VAR_DECL
13015 && !DECL_EXTERNAL (t
))
13016 varpool_node::finalize_decl (t
);
13017 DECL_SAVED_TREE (child_fn
) = NULL
;
13018 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13019 gimple_set_body (child_fn
, NULL
);
13020 TREE_USED (block
) = 1;
13022 /* Reset DECL_CONTEXT on function arguments. */
13023 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13024 DECL_CONTEXT (t
) = child_fn
;
13026 /* Split ENTRY_BB at GIMPLE_*,
13027 so that it can be moved to the child function. */
13028 gsi
= gsi_last_bb (entry_bb
);
13029 stmt
= gsi_stmt (gsi
);
13031 && gimple_code (stmt
) == gimple_code (entry_stmt
));
13032 e
= split_block (entry_bb
, stmt
);
13033 gsi_remove (&gsi
, true);
13034 entry_bb
= e
->dest
;
13035 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
13037 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13040 gsi
= gsi_last_bb (exit_bb
);
13041 gcc_assert (!gsi_end_p (gsi
)
13042 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13043 stmt
= gimple_build_return (NULL
);
13044 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
13045 gsi_remove (&gsi
, true);
13048 /* Move the offloading region into CHILD_CFUN. */
13050 block
= gimple_block (entry_stmt
);
13052 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
13054 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
13055 /* When the OMP expansion process cannot guarantee an up-to-date
13056 loop tree arrange for the child function to fixup loops. */
13057 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13058 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
13060 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13061 num
= vec_safe_length (child_cfun
->local_decls
);
13062 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
13064 t
= (*child_cfun
->local_decls
)[srcidx
];
13065 if (DECL_CONTEXT (t
) == cfun
->decl
)
13067 if (srcidx
!= dstidx
)
13068 (*child_cfun
->local_decls
)[dstidx
] = t
;
13072 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
13074 /* Inform the callgraph about the new function. */
13075 child_cfun
->curr_properties
= cfun
->curr_properties
;
13076 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
13077 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
13078 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
13079 node
->parallelized_function
= 1;
13080 cgraph_node::add_new_function (child_fn
, true);
13082 /* Add the new function to the offload table. */
13083 if (ENABLE_OFFLOADING
)
13084 vec_safe_push (offload_funcs
, child_fn
);
13086 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
13087 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
13089 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13090 fixed in a following pass. */
13091 push_cfun (child_cfun
);
13093 assign_assembler_name_if_neeeded (child_fn
);
13094 cgraph_edge::rebuild_edges ();
13096 /* Some EH regions might become dead, see PR34608. If
13097 pass_cleanup_cfg isn't the first pass to happen with the
13098 new child, these dead EH edges might cause problems.
13099 Clean them up now. */
13100 if (flag_exceptions
)
13103 bool changed
= false;
13105 FOR_EACH_BB_FN (bb
, cfun
)
13106 changed
|= gimple_purge_dead_eh_edges (bb
);
13108 cleanup_tree_cfg ();
13110 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13111 verify_loop_structure ();
13114 if (dump_file
&& !gimple_in_ssa_p (cfun
))
13116 omp_any_child_fn_dumped
= true;
13117 dump_function_header (dump_file
, child_fn
, dump_flags
);
13118 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
13122 /* Emit a library call to launch the offloading region, or do data
13124 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
13125 enum built_in_function start_ix
;
13126 location_t clause_loc
;
13127 unsigned int flags_i
= 0;
13129 switch (gimple_omp_target_kind (entry_stmt
))
13131 case GF_OMP_TARGET_KIND_REGION
:
13132 start_ix
= BUILT_IN_GOMP_TARGET
;
13134 case GF_OMP_TARGET_KIND_DATA
:
13135 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
13137 case GF_OMP_TARGET_KIND_UPDATE
:
13138 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
13140 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13141 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13143 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13144 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13145 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
13147 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13148 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13149 start_ix
= BUILT_IN_GOACC_PARALLEL
;
13151 case GF_OMP_TARGET_KIND_OACC_DATA
:
13152 start_ix
= BUILT_IN_GOACC_DATA_START
;
13154 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13155 start_ix
= BUILT_IN_GOACC_UPDATE
;
13157 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13158 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
13160 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13161 start_ix
= BUILT_IN_GOACC_DECLARE
;
13163 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13164 start_ix
= BUILT_IN_GOACC_HOST_DATA
;
13167 gcc_unreachable ();
13170 clauses
= gimple_omp_target_clauses (entry_stmt
);
13172 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13173 library choose) and there is no conditional. */
13175 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
13177 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
13179 cond
= OMP_CLAUSE_IF_EXPR (c
);
13181 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
13184 /* Even if we pass it to all library function calls, it is currently only
13185 defined/used for the OpenMP target ones. */
13186 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
13187 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
13188 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
13189 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
13191 device
= OMP_CLAUSE_DEVICE_ID (c
);
13192 clause_loc
= OMP_CLAUSE_LOCATION (c
);
13195 clause_loc
= gimple_location (entry_stmt
);
13197 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
13199 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
13201 /* Ensure 'device' is of the correct type. */
13202 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
13204 /* If we found the clause 'if (cond)', build
13205 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13208 cond
= gimple_boolify (cond
);
13210 basic_block cond_bb
, then_bb
, else_bb
;
13214 tmp_var
= create_tmp_var (TREE_TYPE (device
));
13216 e
= split_block_after_labels (new_bb
);
13219 gsi
= gsi_last_bb (new_bb
);
13221 e
= split_block (new_bb
, gsi_stmt (gsi
));
13227 then_bb
= create_empty_bb (cond_bb
);
13228 else_bb
= create_empty_bb (then_bb
);
13229 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
13230 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
13232 stmt
= gimple_build_cond_empty (cond
);
13233 gsi
= gsi_last_bb (cond_bb
);
13234 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13236 gsi
= gsi_start_bb (then_bb
);
13237 stmt
= gimple_build_assign (tmp_var
, device
);
13238 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13240 gsi
= gsi_start_bb (else_bb
);
13241 stmt
= gimple_build_assign (tmp_var
,
13242 build_int_cst (integer_type_node
,
13243 GOMP_DEVICE_HOST_FALLBACK
));
13244 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13246 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
13247 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
13248 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
13249 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
13250 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
13251 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
13256 gsi
= gsi_last_bb (new_bb
);
13257 t
= gimple_omp_target_data_arg (entry_stmt
);
13260 t1
= size_zero_node
;
13261 t2
= build_zero_cst (ptr_type_node
);
13267 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
13268 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
13269 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
13270 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
13271 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
13275 bool tagging
= false;
13276 /* The maximum number used by any start_ix, without varargs. */
13277 auto_vec
<tree
, 11> args
;
13278 args
.quick_push (device
);
13280 args
.quick_push (build_fold_addr_expr (child_fn
));
13281 args
.quick_push (t1
);
13282 args
.quick_push (t2
);
13283 args
.quick_push (t3
);
13284 args
.quick_push (t4
);
13287 case BUILT_IN_GOACC_DATA_START
:
13288 case BUILT_IN_GOACC_DECLARE
:
13289 case BUILT_IN_GOMP_TARGET_DATA
:
13290 case BUILT_IN_GOACC_HOST_DATA
:
13292 case BUILT_IN_GOMP_TARGET
:
13293 case BUILT_IN_GOMP_TARGET_UPDATE
:
13294 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
13295 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
13296 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
13298 depend
= OMP_CLAUSE_DECL (c
);
13300 depend
= build_int_cst (ptr_type_node
, 0);
13301 args
.quick_push (depend
);
13302 if (start_ix
== BUILT_IN_GOMP_TARGET
)
13303 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
13305 case BUILT_IN_GOACC_PARALLEL
:
13307 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
13311 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
13312 case BUILT_IN_GOACC_UPDATE
:
13314 tree t_async
= NULL_TREE
;
13316 /* If present, use the value specified by the respective
13317 clause, making sure that is of the correct type. */
13318 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
13320 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13322 OMP_CLAUSE_ASYNC_EXPR (c
));
13324 /* Default values for t_async. */
13325 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
13327 build_int_cst (integer_type_node
,
13329 if (tagging
&& t_async
)
13331 unsigned HOST_WIDE_INT i_async
= GOMP_LAUNCH_OP_MAX
;
13333 if (TREE_CODE (t_async
) == INTEGER_CST
)
13335 /* See if we can pack the async arg in to the tag's
13337 i_async
= TREE_INT_CST_LOW (t_async
);
13338 if (i_async
< GOMP_LAUNCH_OP_MAX
)
13339 t_async
= NULL_TREE
;
13341 i_async
= GOMP_LAUNCH_OP_MAX
;
13343 args
.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC
, NULL_TREE
,
13347 args
.safe_push (t_async
);
13349 /* Save the argument index, and ... */
13350 unsigned t_wait_idx
= args
.length ();
13351 unsigned num_waits
= 0;
13352 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
13354 /* ... push a placeholder. */
13355 args
.safe_push (integer_zero_node
);
13357 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
13358 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
13360 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13362 OMP_CLAUSE_WAIT_EXPR (c
)));
13366 if (!tagging
|| num_waits
)
13370 /* Now that we know the number, update the placeholder. */
13372 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
13374 len
= build_int_cst (integer_type_node
, num_waits
);
13375 len
= fold_convert_loc (gimple_location (entry_stmt
),
13376 unsigned_type_node
, len
);
13377 args
[t_wait_idx
] = len
;
13382 gcc_unreachable ();
13385 /* Push terminal marker - zero. */
13386 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
13388 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
13389 gimple_set_location (g
, gimple_location (entry_stmt
));
13390 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13393 g
= gsi_stmt (gsi
);
13394 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
13395 gsi_remove (&gsi
, true);
13397 if (data_region
&& region
->exit
)
13399 gsi
= gsi_last_bb (region
->exit
);
13400 g
= gsi_stmt (gsi
);
13401 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
13402 gsi_remove (&gsi
, true);
13406 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13407 variable derived from the thread number. */
13410 grid_expand_omp_for_loop (struct omp_region
*kfor
)
13414 gimple_stmt_iterator gsi
;
13416 struct omp_for_data fd
;
13418 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
13419 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
13420 == GF_OMP_FOR_KIND_GRID_LOOP
);
13421 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
13423 gcc_assert (gimple_omp_for_collapse (for_stmt
) == 1);
13424 gcc_assert (kfor
->cont
);
13425 extract_omp_for_data (for_stmt
, &fd
, NULL
);
13427 itype
= type
= TREE_TYPE (fd
.loop
.v
);
13428 if (POINTER_TYPE_P (type
))
13429 itype
= signed_type_for (type
);
13431 gsi
= gsi_start_bb (body_bb
);
13434 step
= fd
.loop
.step
;
13435 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
13436 true, NULL_TREE
, true, GSI_SAME_STMT
);
13437 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
13438 true, NULL_TREE
, true, GSI_SAME_STMT
);
13439 threadid
= build_call_expr (builtin_decl_explicit
13440 (BUILT_IN_OMP_GET_THREAD_NUM
), 0);
13441 threadid
= fold_convert (itype
, threadid
);
13442 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
13443 true, GSI_SAME_STMT
);
13445 tree startvar
= fd
.loop
.v
;
13446 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
13447 if (POINTER_TYPE_P (type
))
13448 t
= fold_build_pointer_plus (n1
, t
);
13450 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
13451 t
= fold_convert (type
, t
);
13452 t
= force_gimple_operand_gsi (&gsi
, t
,
13454 && TREE_ADDRESSABLE (startvar
),
13455 NULL_TREE
, true, GSI_SAME_STMT
);
13456 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
13457 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
13459 /* Remove the omp for statement */
13460 gsi
= gsi_last_bb (kfor
->entry
);
13461 gsi_remove (&gsi
, true);
13463 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13464 gsi
= gsi_last_bb (kfor
->cont
);
13465 gcc_assert (!gsi_end_p (gsi
)
13466 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
13467 gsi_remove (&gsi
, true);
13469 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13470 gsi
= gsi_last_bb (kfor
->exit
);
13471 gcc_assert (!gsi_end_p (gsi
)
13472 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13473 gsi_remove (&gsi
, true);
13475 /* Fixup the much simpler CFG. */
13476 remove_edge (find_edge (kfor
->cont
, body_bb
));
13478 if (kfor
->cont
!= body_bb
)
13479 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
13480 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
13483 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13486 struct grid_arg_decl_map
13492 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13493 pertaining to kernel function. */
13496 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
13498 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13499 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
13502 if (t
== adm
->old_arg
)
13503 *tp
= adm
->new_arg
;
13504 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
13508 static void expand_omp (struct omp_region
*region
);
13510 /* If TARGET region contains a kernel body for loop, remove its region from the
13511 TARGET and expand it in GPGPU kernel fashion. */
13514 grid_expand_target_grid_body (struct omp_region
*target
)
13516 if (!hsa_gen_requested_p ())
13519 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
13520 struct omp_region
**pp
;
13522 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
13523 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
13526 struct omp_region
*gpukernel
= *pp
;
13528 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
13531 /* HSA cannot handle OACC stuff. */
13532 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
13534 gcc_checking_assert (orig_child_fndecl
);
13535 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13536 OMP_CLAUSE__GRIDDIM_
));
13537 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
13539 hsa_register_kernel (n
);
13543 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13544 OMP_CLAUSE__GRIDDIM_
));
13545 tree inside_block
= gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
13546 *pp
= gpukernel
->next
;
13547 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
13548 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
13551 struct omp_region
*kfor
= *pp
;
13553 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor
)->entry
))
13554 == GF_OMP_FOR_KIND_GRID_LOOP
);
13557 expand_omp (kfor
->inner
);
13558 if (gpukernel
->inner
)
13559 expand_omp (gpukernel
->inner
);
13561 tree kern_fndecl
= copy_node (orig_child_fndecl
);
13562 DECL_NAME (kern_fndecl
) = clone_function_name (kern_fndecl
, "kernel");
13563 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
13564 tree tgtblock
= gimple_block (tgt_stmt
);
13565 tree fniniblock
= make_node (BLOCK
);
13566 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = tgtblock
;
13567 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
13568 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
13569 DECL_INITIAL (kern_fndecl
) = fniniblock
;
13570 push_struct_function (kern_fndecl
);
13571 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
13574 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
13575 gcc_assert (!DECL_CHAIN (old_parm_decl
));
13576 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
13577 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
13578 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
13579 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
13580 kern_cfun
->curr_properties
= cfun
->curr_properties
;
13582 remove_edge (BRANCH_EDGE (kfor
->entry
));
13583 grid_expand_omp_for_loop (kfor
);
13585 /* Remove the omp for statement */
13586 gimple_stmt_iterator gsi
= gsi_last_bb (gpukernel
->entry
);
13587 gsi_remove (&gsi
, true);
13588 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13590 gsi
= gsi_last_bb (gpukernel
->exit
);
13591 gcc_assert (!gsi_end_p (gsi
)
13592 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13593 gimple
*ret_stmt
= gimple_build_return (NULL
);
13594 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
13595 gsi_remove (&gsi
, true);
13597 /* Statements in the first BB in the target construct have been produced by
13598 target lowering and must be copied inside the GPUKERNEL, with the two
13599 exceptions of the first OMP statement and the OMP_DATA assignment
13601 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
13602 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
13603 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
13604 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
13605 !gsi_end_p (tsi
); gsi_next (&tsi
))
13607 gimple
*stmt
= gsi_stmt (tsi
);
13608 if (is_gimple_omp (stmt
))
13611 && is_gimple_assign (stmt
)
13612 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
13613 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
13615 gimple
*copy
= gimple_copy (stmt
);
13616 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
13617 gimple_set_block (copy
, fniniblock
);
13620 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
13621 gpukernel
->exit
, inside_block
);
13623 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
13624 kcn
->mark_force_output ();
13625 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
13627 hsa_register_kernel (kcn
, orig_child
);
13629 cgraph_node::add_new_function (kern_fndecl
, true);
13630 push_cfun (kern_cfun
);
13631 cgraph_edge::rebuild_edges ();
13633 /* Re-map any mention of the PARM_DECL of the original function to the
13634 PARM_DECL of the new one.
13636 TODO: It would be great if lowering produced references into the GPU
13637 kernel decl straight away and we did not have to do this. */
13638 struct grid_arg_decl_map adm
;
13639 adm
.old_arg
= old_parm_decl
;
13640 adm
.new_arg
= new_parm_decl
;
13642 FOR_EACH_BB_FN (bb
, kern_cfun
)
13644 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13646 gimple
*stmt
= gsi_stmt (gsi
);
13647 struct walk_stmt_info wi
;
13648 memset (&wi
, 0, sizeof (wi
));
13650 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
13658 /* Expand the parallel region tree rooted at REGION. Expansion
13659 proceeds in depth-first order. Innermost regions are expanded
13660 first. This way, parallel regions that require a new function to
13661 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13662 internal dependencies in their body. */
13665 expand_omp (struct omp_region
*region
)
13667 omp_any_child_fn_dumped
= false;
13670 location_t saved_location
;
13671 gimple
*inner_stmt
= NULL
;
13673 /* First, determine whether this is a combined parallel+workshare
13675 if (region
->type
== GIMPLE_OMP_PARALLEL
)
13676 determine_parallel_type (region
);
13677 else if (region
->type
== GIMPLE_OMP_TARGET
)
13678 grid_expand_target_grid_body (region
);
13680 if (region
->type
== GIMPLE_OMP_FOR
13681 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
13682 inner_stmt
= last_stmt (region
->inner
->entry
);
13685 expand_omp (region
->inner
);
13687 saved_location
= input_location
;
13688 if (gimple_has_location (last_stmt (region
->entry
)))
13689 input_location
= gimple_location (last_stmt (region
->entry
));
13691 switch (region
->type
)
13693 case GIMPLE_OMP_PARALLEL
:
13694 case GIMPLE_OMP_TASK
:
13695 expand_omp_taskreg (region
);
13698 case GIMPLE_OMP_FOR
:
13699 expand_omp_for (region
, inner_stmt
);
13702 case GIMPLE_OMP_SECTIONS
:
13703 expand_omp_sections (region
);
13706 case GIMPLE_OMP_SECTION
:
13707 /* Individual omp sections are handled together with their
13708 parent GIMPLE_OMP_SECTIONS region. */
13711 case GIMPLE_OMP_SINGLE
:
13712 expand_omp_single (region
);
13715 case GIMPLE_OMP_ORDERED
:
13717 gomp_ordered
*ord_stmt
13718 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13719 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13720 OMP_CLAUSE_DEPEND
))
13722 /* We'll expand these when expanding corresponding
13723 worksharing region with ordered(n) clause. */
13724 gcc_assert (region
->outer
13725 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13726 region
->ord_stmt
= ord_stmt
;
13731 case GIMPLE_OMP_MASTER
:
13732 case GIMPLE_OMP_TASKGROUP
:
13733 case GIMPLE_OMP_CRITICAL
:
13734 case GIMPLE_OMP_TEAMS
:
13735 expand_omp_synch (region
);
13738 case GIMPLE_OMP_ATOMIC_LOAD
:
13739 expand_omp_atomic (region
);
13742 case GIMPLE_OMP_TARGET
:
13743 expand_omp_target (region
);
13747 gcc_unreachable ();
13750 input_location
= saved_location
;
13751 region
= region
->next
;
13753 if (omp_any_child_fn_dumped
)
13756 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13757 omp_any_child_fn_dumped
= false;
13762 /* Helper for build_omp_regions. Scan the dominator tree starting at
13763 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13764 true, the function ends once a single tree is built (otherwise, whole
13765 forest of OMP constructs may be built). */
13768 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13771 gimple_stmt_iterator gsi
;
13775 gsi
= gsi_last_bb (bb
);
13776 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13778 struct omp_region
*region
;
13779 enum gimple_code code
;
13781 stmt
= gsi_stmt (gsi
);
13782 code
= gimple_code (stmt
);
13783 if (code
== GIMPLE_OMP_RETURN
)
13785 /* STMT is the return point out of region PARENT. Mark it
13786 as the exit point and make PARENT the immediately
13787 enclosing region. */
13788 gcc_assert (parent
);
13791 parent
= parent
->outer
;
13793 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13795 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13796 GIMPLE_OMP_RETURN, but matches with
13797 GIMPLE_OMP_ATOMIC_LOAD. */
13798 gcc_assert (parent
);
13799 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13802 parent
= parent
->outer
;
13804 else if (code
== GIMPLE_OMP_CONTINUE
)
13806 gcc_assert (parent
);
13809 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13811 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13812 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13816 region
= new_omp_region (bb
, code
, parent
);
13818 if (code
== GIMPLE_OMP_TARGET
)
13820 switch (gimple_omp_target_kind (stmt
))
13822 case GF_OMP_TARGET_KIND_REGION
:
13823 case GF_OMP_TARGET_KIND_DATA
:
13824 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13825 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13826 case GF_OMP_TARGET_KIND_OACC_DATA
:
13827 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13829 case GF_OMP_TARGET_KIND_UPDATE
:
13830 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13831 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13832 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13833 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13834 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13835 /* ..., other than for those stand-alone directives... */
13839 gcc_unreachable ();
13842 else if (code
== GIMPLE_OMP_ORDERED
13843 && find_omp_clause (gimple_omp_ordered_clauses
13844 (as_a
<gomp_ordered
*> (stmt
)),
13845 OMP_CLAUSE_DEPEND
))
13846 /* #pragma omp ordered depend is also just a stand-alone
13849 /* ..., this directive becomes the parent for a new region. */
13855 if (single_tree
&& !parent
)
13858 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13860 son
= next_dom_son (CDI_DOMINATORS
, son
))
13861 build_omp_regions_1 (son
, parent
, single_tree
);
13864 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13865 root_omp_region. */
13868 build_omp_regions_root (basic_block root
)
13870 gcc_assert (root_omp_region
== NULL
);
13871 build_omp_regions_1 (root
, NULL
, true);
13872 gcc_assert (root_omp_region
!= NULL
);
13875 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13878 omp_expand_local (basic_block head
)
13880 build_omp_regions_root (head
);
13881 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13883 fprintf (dump_file
, "\nOMP region tree\n\n");
13884 dump_omp_region (dump_file
, root_omp_region
, 0);
13885 fprintf (dump_file
, "\n");
13888 remove_exit_barriers (root_omp_region
);
13889 expand_omp (root_omp_region
);
13891 free_omp_regions ();
13894 /* Scan the CFG and build a tree of OMP regions. Return the root of
13895 the OMP region tree. */
13898 build_omp_regions (void)
13900 gcc_assert (root_omp_region
== NULL
);
13901 calculate_dominance_info (CDI_DOMINATORS
);
13902 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
13905 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
13907 static unsigned int
13908 execute_expand_omp (void)
13910 build_omp_regions ();
13912 if (!root_omp_region
)
13917 fprintf (dump_file
, "\nOMP region tree\n\n");
13918 dump_omp_region (dump_file
, root_omp_region
, 0);
13919 fprintf (dump_file
, "\n");
13922 remove_exit_barriers (root_omp_region
);
13924 expand_omp (root_omp_region
);
13926 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13927 verify_loop_structure ();
13928 cleanup_tree_cfg ();
13930 free_omp_regions ();
13935 /* OMP expansion -- the default pass, run before creation of SSA form. */
13939 const pass_data pass_data_expand_omp
=
13941 GIMPLE_PASS
, /* type */
13942 "ompexp", /* name */
13943 OPTGROUP_NONE
, /* optinfo_flags */
13944 TV_NONE
, /* tv_id */
13945 PROP_gimple_any
, /* properties_required */
13946 PROP_gimple_eomp
, /* properties_provided */
13947 0, /* properties_destroyed */
13948 0, /* todo_flags_start */
13949 0, /* todo_flags_finish */
13952 class pass_expand_omp
: public gimple_opt_pass
13955 pass_expand_omp (gcc::context
*ctxt
)
13956 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
13959 /* opt_pass methods: */
13960 virtual unsigned int execute (function
*)
13962 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
13963 || flag_openmp_simd
!= 0)
13964 && !seen_error ());
13966 /* This pass always runs, to provide PROP_gimple_eomp.
13967 But often, there is nothing to do. */
13971 return execute_expand_omp ();
13974 }; // class pass_expand_omp
13976 } // anon namespace
13979 make_pass_expand_omp (gcc::context
*ctxt
)
13981 return new pass_expand_omp (ctxt
);
13986 const pass_data pass_data_expand_omp_ssa
=
13988 GIMPLE_PASS
, /* type */
13989 "ompexpssa", /* name */
13990 OPTGROUP_NONE
, /* optinfo_flags */
13991 TV_NONE
, /* tv_id */
13992 PROP_cfg
| PROP_ssa
, /* properties_required */
13993 PROP_gimple_eomp
, /* properties_provided */
13994 0, /* properties_destroyed */
13995 0, /* todo_flags_start */
13996 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
13999 class pass_expand_omp_ssa
: public gimple_opt_pass
14002 pass_expand_omp_ssa (gcc::context
*ctxt
)
14003 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
14006 /* opt_pass methods: */
14007 virtual bool gate (function
*fun
)
14009 return !(fun
->curr_properties
& PROP_gimple_eomp
);
14011 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
14012 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
14014 }; // class pass_expand_omp_ssa
14016 } // anon namespace
14019 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
14021 return new pass_expand_omp_ssa (ctxt
);
14024 /* Routines to lower OMP directives into OMP-GIMPLE. */
14026 /* If ctx is a worksharing context inside of a cancellable parallel
14027 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14028 and conditional branch to parallel's cancel_label to handle
14029 cancellation in the implicit barrier. */
14032 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
14034 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
14035 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
14036 if (gimple_omp_return_nowait_p (omp_return
))
14039 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
14040 && ctx
->outer
->cancellable
)
14042 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
14043 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
14044 tree lhs
= create_tmp_var (c_bool_type
);
14045 gimple_omp_return_set_lhs (omp_return
, lhs
);
14046 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
14047 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
14048 fold_convert (c_bool_type
,
14049 boolean_false_node
),
14050 ctx
->outer
->cancel_label
, fallthru_label
);
14051 gimple_seq_add_stmt (body
, g
);
14052 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
14056 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14057 CTX is the enclosing OMP context for the current statement. */
14060 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14062 tree block
, control
;
14063 gimple_stmt_iterator tgsi
;
14064 gomp_sections
*stmt
;
14066 gbind
*new_stmt
, *bind
;
14067 gimple_seq ilist
, dlist
, olist
, new_body
;
14069 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
14071 push_gimplify_context ();
14075 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
14076 &ilist
, &dlist
, ctx
, NULL
);
14078 new_body
= gimple_omp_body (stmt
);
14079 gimple_omp_set_body (stmt
, NULL
);
14080 tgsi
= gsi_start (new_body
);
14081 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
14086 sec_start
= gsi_stmt (tgsi
);
14087 sctx
= maybe_lookup_ctx (sec_start
);
14090 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
14091 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
14092 GSI_CONTINUE_LINKING
);
14093 gimple_omp_set_body (sec_start
, NULL
);
14095 if (gsi_one_before_end_p (tgsi
))
14097 gimple_seq l
= NULL
;
14098 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
14100 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
14101 gimple_omp_section_set_last (sec_start
);
14104 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
14105 GSI_CONTINUE_LINKING
);
14108 block
= make_node (BLOCK
);
14109 bind
= gimple_build_bind (NULL
, new_body
, block
);
14112 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
14114 block
= make_node (BLOCK
);
14115 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14116 gsi_replace (gsi_p
, new_stmt
, true);
14118 pop_gimplify_context (new_stmt
);
14119 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14120 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14121 if (BLOCK_VARS (block
))
14122 TREE_USED (block
) = 1;
14125 gimple_seq_add_seq (&new_body
, ilist
);
14126 gimple_seq_add_stmt (&new_body
, stmt
);
14127 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
14128 gimple_seq_add_stmt (&new_body
, bind
);
14130 control
= create_tmp_var (unsigned_type_node
, ".section");
14131 t
= gimple_build_omp_continue (control
, control
);
14132 gimple_omp_sections_set_control (stmt
, control
);
14133 gimple_seq_add_stmt (&new_body
, t
);
14135 gimple_seq_add_seq (&new_body
, olist
);
14136 if (ctx
->cancellable
)
14137 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14138 gimple_seq_add_seq (&new_body
, dlist
);
14140 new_body
= maybe_catch_exception (new_body
);
14142 t
= gimple_build_omp_return
14143 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
14144 OMP_CLAUSE_NOWAIT
));
14145 gimple_seq_add_stmt (&new_body
, t
);
14146 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
14148 gimple_bind_set_body (new_stmt
, new_body
);
14152 /* A subroutine of lower_omp_single. Expand the simple form of
14153 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14155 if (GOMP_single_start ())
14157 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14159 FIXME. It may be better to delay expanding the logic of this until
14160 pass_expand_omp. The expanded logic may make the job more difficult
14161 to a synchronization analysis pass. */
14164 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
14166 location_t loc
= gimple_location (single_stmt
);
14167 tree tlabel
= create_artificial_label (loc
);
14168 tree flabel
= create_artificial_label (loc
);
14169 gimple
*call
, *cond
;
14172 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
14173 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
14174 call
= gimple_build_call (decl
, 0);
14175 gimple_call_set_lhs (call
, lhs
);
14176 gimple_seq_add_stmt (pre_p
, call
);
14178 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
14179 fold_convert_loc (loc
, TREE_TYPE (lhs
),
14180 boolean_true_node
),
14182 gimple_seq_add_stmt (pre_p
, cond
);
14183 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
14184 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14185 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
14189 /* A subroutine of lower_omp_single. Expand the simple form of
14190 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14192 #pragma omp single copyprivate (a, b, c)
14194 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14197 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14203 GOMP_single_copy_end (©out);
14214 FIXME. It may be better to delay expanding the logic of this until
14215 pass_expand_omp. The expanded logic may make the job more difficult
14216 to a synchronization analysis pass. */
14219 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
14222 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
14223 gimple_seq copyin_seq
;
14224 location_t loc
= gimple_location (single_stmt
);
14226 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
14228 ptr_type
= build_pointer_type (ctx
->record_type
);
14229 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
14231 l0
= create_artificial_label (loc
);
14232 l1
= create_artificial_label (loc
);
14233 l2
= create_artificial_label (loc
);
14235 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
14236 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
14237 t
= fold_convert_loc (loc
, ptr_type
, t
);
14238 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
14240 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
14241 build_int_cst (ptr_type
, 0));
14242 t
= build3 (COND_EXPR
, void_type_node
, t
,
14243 build_and_jump (&l0
), build_and_jump (&l1
));
14244 gimplify_and_add (t
, pre_p
);
14246 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
14248 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14251 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
14254 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14255 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
14256 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
14257 gimplify_and_add (t
, pre_p
);
14259 t
= build_and_jump (&l2
);
14260 gimplify_and_add (t
, pre_p
);
14262 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
14264 gimple_seq_add_seq (pre_p
, copyin_seq
);
14266 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
14270 /* Expand code for an OpenMP single directive. */
14273 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14277 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
14279 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
14281 push_gimplify_context ();
14283 block
= make_node (BLOCK
);
14284 bind
= gimple_build_bind (NULL
, NULL
, block
);
14285 gsi_replace (gsi_p
, bind
, true);
14288 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
14289 &bind_body
, &dlist
, ctx
, NULL
);
14290 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
14292 gimple_seq_add_stmt (&bind_body
, single_stmt
);
14294 if (ctx
->record_type
)
14295 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
14297 lower_omp_single_simple (single_stmt
, &bind_body
);
14299 gimple_omp_set_body (single_stmt
, NULL
);
14301 gimple_seq_add_seq (&bind_body
, dlist
);
14303 bind_body
= maybe_catch_exception (bind_body
);
14305 t
= gimple_build_omp_return
14306 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
14307 OMP_CLAUSE_NOWAIT
));
14308 gimple_seq_add_stmt (&bind_body_tail
, t
);
14309 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
14310 if (ctx
->record_type
)
14312 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
14313 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14314 TREE_THIS_VOLATILE (clobber
) = 1;
14315 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
14316 clobber
), GSI_SAME_STMT
);
14318 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
14319 gimple_bind_set_body (bind
, bind_body
);
14321 pop_gimplify_context (bind
);
14323 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14324 BLOCK_VARS (block
) = ctx
->block_vars
;
14325 if (BLOCK_VARS (block
))
14326 TREE_USED (block
) = 1;
14330 /* Expand code for an OpenMP master directive. */
14333 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14335 tree block
, lab
= NULL
, x
, bfn_decl
;
14336 gimple
*stmt
= gsi_stmt (*gsi_p
);
14338 location_t loc
= gimple_location (stmt
);
14341 push_gimplify_context ();
14343 block
= make_node (BLOCK
);
14344 bind
= gimple_build_bind (NULL
, NULL
, block
);
14345 gsi_replace (gsi_p
, bind
, true);
14346 gimple_bind_add_stmt (bind
, stmt
);
14348 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
14349 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
14350 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
14351 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
14353 gimplify_and_add (x
, &tseq
);
14354 gimple_bind_add_seq (bind
, tseq
);
14356 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14357 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14358 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14359 gimple_omp_set_body (stmt
, NULL
);
14361 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
14363 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14365 pop_gimplify_context (bind
);
14367 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14368 BLOCK_VARS (block
) = ctx
->block_vars
;
14372 /* Expand code for an OpenMP taskgroup directive. */
14375 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14377 gimple
*stmt
= gsi_stmt (*gsi_p
);
14380 tree block
= make_node (BLOCK
);
14382 bind
= gimple_build_bind (NULL
, NULL
, block
);
14383 gsi_replace (gsi_p
, bind
, true);
14384 gimple_bind_add_stmt (bind
, stmt
);
14386 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
14388 gimple_bind_add_stmt (bind
, x
);
14390 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14391 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14392 gimple_omp_set_body (stmt
, NULL
);
14394 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14396 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14397 BLOCK_VARS (block
) = ctx
->block_vars
;
14401 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14404 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
14407 struct omp_for_data fd
;
14408 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
14411 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
14412 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
14413 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
14417 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14418 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
14419 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14420 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
14422 /* Merge depend clauses from multiple adjacent
14423 #pragma omp ordered depend(sink:...) constructs
14424 into one #pragma omp ordered depend(sink:...), so that
14425 we can optimize them together. */
14426 gimple_stmt_iterator gsi
= *gsi_p
;
14428 while (!gsi_end_p (gsi
))
14430 gimple
*stmt
= gsi_stmt (gsi
);
14431 if (is_gimple_debug (stmt
)
14432 || gimple_code (stmt
) == GIMPLE_NOP
)
14437 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
14439 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
14440 c
= gimple_omp_ordered_clauses (ord_stmt2
);
14442 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
14443 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14446 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
14448 gsi_remove (&gsi
, true);
14452 /* Canonicalize sink dependence clauses into one folded clause if
14455 The basic algorithm is to create a sink vector whose first
14456 element is the GCD of all the first elements, and whose remaining
14457 elements are the minimum of the subsequent columns.
14459 We ignore dependence vectors whose first element is zero because
14460 such dependencies are known to be executed by the same thread.
14462 We take into account the direction of the loop, so a minimum
14463 becomes a maximum if the loop is iterating forwards. We also
14464 ignore sink clauses where the loop direction is unknown, or where
14465 the offsets are clearly invalid because they are not a multiple
14466 of the loop increment.
14470 #pragma omp for ordered(2)
14471 for (i=0; i < N; ++i)
14472 for (j=0; j < M; ++j)
14474 #pragma omp ordered \
14475 depend(sink:i-8,j-2) \
14476 depend(sink:i,j-1) \ // Completely ignored because i+0.
14477 depend(sink:i-4,j-3) \
14478 depend(sink:i-6,j-4)
14479 #pragma omp ordered depend(source)
14484 depend(sink:-gcd(8,4,6),-min(2,3,4))
14489 /* FIXME: Computing GCD's where the first element is zero is
14490 non-trivial in the presence of collapsed loops. Do this later. */
14491 if (fd
.collapse
> 1)
14494 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
14495 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
14496 tree folded_dep
= NULL_TREE
;
14497 /* TRUE if the first dimension's offset is negative. */
14498 bool neg_offset_p
= false;
14500 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14502 while ((c
= *list_p
) != NULL
)
14504 bool remove
= false;
14506 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
14507 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14508 goto next_ordered_clause
;
14511 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
14512 vec
&& TREE_CODE (vec
) == TREE_LIST
;
14513 vec
= TREE_CHAIN (vec
), ++i
)
14515 gcc_assert (i
< len
);
14517 /* extract_omp_for_data has canonicalized the condition. */
14518 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
14519 || fd
.loops
[i
].cond_code
== GT_EXPR
);
14520 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
14521 bool maybe_lexically_later
= true;
14523 /* While the committee makes up its mind, bail if we have any
14524 non-constant steps. */
14525 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
14526 goto lower_omp_ordered_ret
;
14528 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
14529 if (POINTER_TYPE_P (itype
))
14531 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
14532 TYPE_PRECISION (itype
),
14533 TYPE_SIGN (itype
));
14535 /* Ignore invalid offsets that are not multiples of the step. */
14536 if (!wi::multiple_of_p
14537 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
14540 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
14541 "ignoring sink clause with offset that is not "
14542 "a multiple of the loop step");
14544 goto next_ordered_clause
;
14547 /* Calculate the first dimension. The first dimension of
14548 the folded dependency vector is the GCD of the first
14549 elements, while ignoring any first elements whose offset
14553 /* Ignore dependence vectors whose first dimension is 0. */
14557 goto next_ordered_clause
;
14561 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
14563 error_at (OMP_CLAUSE_LOCATION (c
),
14564 "first offset must be in opposite direction "
14565 "of loop iterations");
14566 goto lower_omp_ordered_ret
;
14570 neg_offset_p
= forward
;
14571 /* Initialize the first time around. */
14572 if (folded_dep
== NULL_TREE
)
14575 folded_deps
[0] = offset
;
14578 folded_deps
[0] = wi::gcd (folded_deps
[0],
14582 /* Calculate minimum for the remaining dimensions. */
14585 folded_deps
[len
+ i
- 1] = offset
;
14586 if (folded_dep
== c
)
14587 folded_deps
[i
] = offset
;
14588 else if (maybe_lexically_later
14589 && !wi::eq_p (folded_deps
[i
], offset
))
14591 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
14595 for (j
= 1; j
<= i
; j
++)
14596 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
14599 maybe_lexically_later
= false;
14603 gcc_assert (i
== len
);
14607 next_ordered_clause
:
14609 *list_p
= OMP_CLAUSE_CHAIN (c
);
14611 list_p
= &OMP_CLAUSE_CHAIN (c
);
14617 folded_deps
[0] = -folded_deps
[0];
14619 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
14620 if (POINTER_TYPE_P (itype
))
14623 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14624 = wide_int_to_tree (itype
, folded_deps
[0]);
14625 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14626 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14629 lower_omp_ordered_ret
:
14631 /* Ordered without clauses is #pragma omp threads, while we want
14632 a nop instead if we remove all clauses. */
14633 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14634 gsi_replace (gsi_p
, gimple_build_nop (), true);
14638 /* Expand code for an OpenMP ordered directive. */
14641 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14644 gimple
*stmt
= gsi_stmt (*gsi_p
);
14645 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14648 bool simd
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14650 bool threads
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14651 OMP_CLAUSE_THREADS
);
14653 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14654 OMP_CLAUSE_DEPEND
))
14656 /* FIXME: This is needs to be moved to the expansion to verify various
14657 conditions only testable on cfg with dominators computed, and also
14658 all the depend clauses to be merged still might need to be available
14659 for the runtime checks. */
14661 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14665 push_gimplify_context ();
14667 block
= make_node (BLOCK
);
14668 bind
= gimple_build_bind (NULL
, NULL
, block
);
14669 gsi_replace (gsi_p
, bind
, true);
14670 gimple_bind_add_stmt (bind
, stmt
);
14674 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
14675 build_int_cst (NULL_TREE
, threads
));
14676 cfun
->has_simduid_loops
= true;
14679 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14681 gimple_bind_add_stmt (bind
, x
);
14683 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14684 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14685 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14686 gimple_omp_set_body (stmt
, NULL
);
14689 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
14690 build_int_cst (NULL_TREE
, threads
));
14692 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14694 gimple_bind_add_stmt (bind
, x
);
14696 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14698 pop_gimplify_context (bind
);
14700 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14701 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14705 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14706 substitution of a couple of function calls. But in the NAMED case,
14707 requires that languages coordinate a symbol name. It is therefore
14708 best put here in common code. */
14710 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14713 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14716 tree name
, lock
, unlock
;
14717 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14719 location_t loc
= gimple_location (stmt
);
14722 name
= gimple_omp_critical_name (stmt
);
14727 if (!critical_name_mutexes
)
14728 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14730 tree
*n
= critical_name_mutexes
->get (name
);
14735 decl
= create_tmp_var_raw (ptr_type_node
);
14737 new_str
= ACONCAT ((".gomp_critical_user_",
14738 IDENTIFIER_POINTER (name
), NULL
));
14739 DECL_NAME (decl
) = get_identifier (new_str
);
14740 TREE_PUBLIC (decl
) = 1;
14741 TREE_STATIC (decl
) = 1;
14742 DECL_COMMON (decl
) = 1;
14743 DECL_ARTIFICIAL (decl
) = 1;
14744 DECL_IGNORED_P (decl
) = 1;
14746 varpool_node::finalize_decl (decl
);
14748 critical_name_mutexes
->put (name
, decl
);
14753 /* If '#pragma omp critical' is inside offloaded region or
14754 inside function marked as offloadable, the symbol must be
14755 marked as offloadable too. */
14757 if (cgraph_node::get (current_function_decl
)->offloadable
)
14758 varpool_node::get_create (decl
)->offloadable
= 1;
14760 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14761 if (is_gimple_omp_offloaded (octx
->stmt
))
14763 varpool_node::get_create (decl
)->offloadable
= 1;
14767 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14768 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14770 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14771 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14772 build_fold_addr_expr_loc (loc
, decl
));
14776 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14777 lock
= build_call_expr_loc (loc
, lock
, 0);
14779 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14780 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14783 push_gimplify_context ();
14785 block
= make_node (BLOCK
);
14786 bind
= gimple_build_bind (NULL
, NULL
, block
);
14787 gsi_replace (gsi_p
, bind
, true);
14788 gimple_bind_add_stmt (bind
, stmt
);
14790 tbody
= gimple_bind_body (bind
);
14791 gimplify_and_add (lock
, &tbody
);
14792 gimple_bind_set_body (bind
, tbody
);
14794 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14795 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14796 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14797 gimple_omp_set_body (stmt
, NULL
);
14799 tbody
= gimple_bind_body (bind
);
14800 gimplify_and_add (unlock
, &tbody
);
14801 gimple_bind_set_body (bind
, tbody
);
14803 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14805 pop_gimplify_context (bind
);
14806 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14807 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14811 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14812 for a lastprivate clause. Given a loop control predicate of (V
14813 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14814 is appended to *DLIST, iterator initialization is appended to
14818 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14819 gimple_seq
*dlist
, struct omp_context
*ctx
)
14821 tree clauses
, cond
, vinit
;
14822 enum tree_code cond_code
;
14825 cond_code
= fd
->loop
.cond_code
;
14826 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14828 /* When possible, use a strict equality expression. This can let VRP
14829 type optimizations deduce the value and remove a copy. */
14830 if (tree_fits_shwi_p (fd
->loop
.step
))
14832 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14833 if (step
== 1 || step
== -1)
14834 cond_code
= EQ_EXPR
;
14837 tree n2
= fd
->loop
.n2
;
14838 if (fd
->collapse
> 1
14839 && TREE_CODE (n2
) != INTEGER_CST
14840 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14842 struct omp_context
*taskreg_ctx
= NULL
;
14843 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14845 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14846 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14847 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14849 if (gimple_omp_for_combined_into_p (gfor
))
14851 gcc_assert (ctx
->outer
->outer
14852 && is_parallel_ctx (ctx
->outer
->outer
));
14853 taskreg_ctx
= ctx
->outer
->outer
;
14857 struct omp_for_data outer_fd
;
14858 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14859 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14862 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14863 taskreg_ctx
= ctx
->outer
->outer
;
14865 else if (is_taskreg_ctx (ctx
->outer
))
14866 taskreg_ctx
= ctx
->outer
;
14871 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14872 OMP_CLAUSE__LOOPTEMP_
);
14873 gcc_assert (innerc
);
14874 for (i
= 0; i
< fd
->collapse
; i
++)
14876 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14877 OMP_CLAUSE__LOOPTEMP_
);
14878 gcc_assert (innerc
);
14880 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14881 OMP_CLAUSE__LOOPTEMP_
);
14883 n2
= fold_convert (TREE_TYPE (n2
),
14884 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14888 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14890 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14892 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14893 if (!gimple_seq_empty_p (stmts
))
14895 gimple_seq_add_seq (&stmts
, *dlist
);
14898 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
14899 vinit
= fd
->loop
.n1
;
14900 if (cond_code
== EQ_EXPR
14901 && tree_fits_shwi_p (fd
->loop
.n2
)
14902 && ! integer_zerop (fd
->loop
.n2
))
14903 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
14905 vinit
= unshare_expr (vinit
);
14907 /* Initialize the iterator variable, so that threads that don't execute
14908 any iterations don't execute the lastprivate clauses by accident. */
14909 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
14914 /* Lower code for an OMP loop directive. */
14917 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14919 tree
*rhs_p
, block
;
14920 struct omp_for_data fd
, *fdp
= NULL
;
14921 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
14923 gimple_seq omp_for_body
, body
, dlist
;
14924 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
14927 push_gimplify_context ();
14929 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
14931 block
= make_node (BLOCK
);
14932 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14933 /* Replace at gsi right away, so that 'stmt' is no member
14934 of a sequence anymore as we're going to add to a different
14936 gsi_replace (gsi_p
, new_stmt
, true);
14938 /* Move declaration of temporaries in the loop body before we make
14940 omp_for_body
= gimple_omp_body (stmt
);
14941 if (!gimple_seq_empty_p (omp_for_body
)
14942 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
14945 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
14946 tree vars
= gimple_bind_vars (inner_bind
);
14947 gimple_bind_append_vars (new_stmt
, vars
);
14948 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
14949 keep them on the inner_bind and it's block. */
14950 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
14951 if (gimple_bind_block (inner_bind
))
14952 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
14955 if (gimple_omp_for_combined_into_p (stmt
))
14957 extract_omp_for_data (stmt
, &fd
, NULL
);
14960 /* We need two temporaries with fd.loop.v type (istart/iend)
14961 and then (fd.collapse - 1) temporaries with the same
14962 type for count2 ... countN-1 vars if not constant. */
14964 tree type
= fd
.iter_type
;
14965 if (fd
.collapse
> 1
14966 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
14967 count
+= fd
.collapse
- 1;
14969 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
14970 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
14971 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
14972 tree clauses
= *pc
;
14975 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
14976 OMP_CLAUSE__LOOPTEMP_
);
14977 for (i
= 0; i
< count
; i
++)
14982 gcc_assert (outerc
);
14983 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
14984 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
14985 OMP_CLAUSE__LOOPTEMP_
);
14989 temp
= create_tmp_var (type
);
14990 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
14992 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
14993 OMP_CLAUSE_DECL (*pc
) = temp
;
14994 pc
= &OMP_CLAUSE_CHAIN (*pc
);
14999 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15002 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
15004 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
15006 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
15008 /* Lower the header expressions. At this point, we can assume that
15009 the header is of the form:
15011 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15013 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15014 using the .omp_data_s mapping, if needed. */
15015 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
15017 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
15018 if (!is_gimple_min_invariant (*rhs_p
))
15019 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15021 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
15022 if (!is_gimple_min_invariant (*rhs_p
))
15023 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15025 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
15026 if (!is_gimple_min_invariant (*rhs_p
))
15027 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15030 /* Once lowered, extract the bounds and clauses. */
15031 extract_omp_for_data (stmt
, &fd
, NULL
);
15033 if (is_gimple_omp_oacc (ctx
->stmt
)
15034 && !ctx_in_oacc_kernels_region (ctx
))
15035 lower_oacc_head_tail (gimple_location (stmt
),
15036 gimple_omp_for_clauses (stmt
),
15037 &oacc_head
, &oacc_tail
, ctx
);
15039 /* Add OpenACC partitioning and reduction markers just before the loop */
15041 gimple_seq_add_seq (&body
, oacc_head
);
15043 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
15045 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
15046 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15047 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
15048 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
15050 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
15051 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
15052 OMP_CLAUSE_LINEAR_STEP (c
)
15053 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
15057 if (!gimple_omp_for_grid_phony (stmt
))
15058 gimple_seq_add_stmt (&body
, stmt
);
15059 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
15061 if (!gimple_omp_for_grid_phony (stmt
))
15062 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
15065 /* After the loop, add exit clauses. */
15066 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
15068 if (ctx
->cancellable
)
15069 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
15071 gimple_seq_add_seq (&body
, dlist
);
15073 body
= maybe_catch_exception (body
);
15075 if (!gimple_omp_for_grid_phony (stmt
))
15077 /* Region exit marker goes at the end of the loop body. */
15078 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
15079 maybe_add_implicit_barrier_cancel (ctx
, &body
);
15082 /* Add OpenACC joining and reduction markers just after the loop. */
15084 gimple_seq_add_seq (&body
, oacc_tail
);
15086 pop_gimplify_context (new_stmt
);
15088 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
15089 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
15090 if (BLOCK_VARS (block
))
15091 TREE_USED (block
) = 1;
15093 gimple_bind_set_body (new_stmt
, body
);
15094 gimple_omp_set_body (stmt
, NULL
);
15095 gimple_omp_for_set_pre_body (stmt
, NULL
);
15098 /* Callback for walk_stmts. Check if the current statement only contains
15099 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15102 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
15103 bool *handled_ops_p
,
15104 struct walk_stmt_info
*wi
)
15106 int *info
= (int *) wi
->info
;
15107 gimple
*stmt
= gsi_stmt (*gsi_p
);
15109 *handled_ops_p
= true;
15110 switch (gimple_code (stmt
))
15114 case GIMPLE_OMP_FOR
:
15115 case GIMPLE_OMP_SECTIONS
:
15116 *info
= *info
== 0 ? 1 : -1;
15125 struct omp_taskcopy_context
15127 /* This field must be at the beginning, as we do "inheritance": Some
15128 callback functions for tree-inline.c (e.g., omp_copy_decl)
15129 receive a copy_body_data pointer that is up-casted to an
15130 omp_context pointer. */
15136 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
15138 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
15140 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
15141 return create_tmp_var (TREE_TYPE (var
));
15147 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
15149 tree name
, new_fields
= NULL
, type
, f
;
15151 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
15152 name
= DECL_NAME (TYPE_NAME (orig_type
));
15153 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
15154 TYPE_DECL
, name
, type
);
15155 TYPE_NAME (type
) = name
;
15157 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
15159 tree new_f
= copy_node (f
);
15160 DECL_CONTEXT (new_f
) = type
;
15161 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
15162 TREE_CHAIN (new_f
) = new_fields
;
15163 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15164 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15165 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
15167 new_fields
= new_f
;
15168 tcctx
->cb
.decl_map
->put (f
, new_f
);
15170 TYPE_FIELDS (type
) = nreverse (new_fields
);
15171 layout_type (type
);
15175 /* Create task copyfn. */
15178 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
15180 struct function
*child_cfun
;
15181 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
15182 tree record_type
, srecord_type
, bind
, list
;
15183 bool record_needs_remap
= false, srecord_needs_remap
= false;
15185 struct omp_taskcopy_context tcctx
;
15186 location_t loc
= gimple_location (task_stmt
);
15188 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
15189 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
15190 gcc_assert (child_cfun
->cfg
== NULL
);
15191 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
15193 /* Reset DECL_CONTEXT on function arguments. */
15194 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
15195 DECL_CONTEXT (t
) = child_fn
;
15197 /* Populate the function. */
15198 push_gimplify_context ();
15199 push_cfun (child_cfun
);
15201 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
15202 TREE_SIDE_EFFECTS (bind
) = 1;
15204 DECL_SAVED_TREE (child_fn
) = bind
;
15205 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
15207 /* Remap src and dst argument types if needed. */
15208 record_type
= ctx
->record_type
;
15209 srecord_type
= ctx
->srecord_type
;
15210 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
15211 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15213 record_needs_remap
= true;
15216 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
15217 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15219 srecord_needs_remap
= true;
15223 if (record_needs_remap
|| srecord_needs_remap
)
15225 memset (&tcctx
, '\0', sizeof (tcctx
));
15226 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
15227 tcctx
.cb
.dst_fn
= child_fn
;
15228 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
15229 gcc_checking_assert (tcctx
.cb
.src_node
);
15230 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
15231 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
15232 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
15233 tcctx
.cb
.eh_lp_nr
= 0;
15234 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
15235 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
15238 if (record_needs_remap
)
15239 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
15240 if (srecord_needs_remap
)
15241 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
15244 tcctx
.cb
.decl_map
= NULL
;
15246 arg
= DECL_ARGUMENTS (child_fn
);
15247 TREE_TYPE (arg
) = build_pointer_type (record_type
);
15248 sarg
= DECL_CHAIN (arg
);
15249 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
15251 /* First pass: initialize temporaries used in record_type and srecord_type
15252 sizes and field offsets. */
15253 if (tcctx
.cb
.decl_map
)
15254 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15255 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15259 decl
= OMP_CLAUSE_DECL (c
);
15260 p
= tcctx
.cb
.decl_map
->get (decl
);
15263 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15264 sf
= (tree
) n
->value
;
15265 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15266 src
= build_simple_mem_ref_loc (loc
, sarg
);
15267 src
= omp_build_component_ref (src
, sf
);
15268 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
15269 append_to_statement_list (t
, &list
);
15272 /* Second pass: copy shared var pointers and copy construct non-VLA
15273 firstprivate vars. */
15274 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15275 switch (OMP_CLAUSE_CODE (c
))
15277 splay_tree_key key
;
15278 case OMP_CLAUSE_SHARED
:
15279 decl
= OMP_CLAUSE_DECL (c
);
15280 key
= (splay_tree_key
) decl
;
15281 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
15282 key
= (splay_tree_key
) &DECL_UID (decl
);
15283 n
= splay_tree_lookup (ctx
->field_map
, key
);
15286 f
= (tree
) n
->value
;
15287 if (tcctx
.cb
.decl_map
)
15288 f
= *tcctx
.cb
.decl_map
->get (f
);
15289 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
15290 sf
= (tree
) n
->value
;
15291 if (tcctx
.cb
.decl_map
)
15292 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15293 src
= build_simple_mem_ref_loc (loc
, sarg
);
15294 src
= omp_build_component_ref (src
, sf
);
15295 dst
= build_simple_mem_ref_loc (loc
, arg
);
15296 dst
= omp_build_component_ref (dst
, f
);
15297 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15298 append_to_statement_list (t
, &list
);
15300 case OMP_CLAUSE_FIRSTPRIVATE
:
15301 decl
= OMP_CLAUSE_DECL (c
);
15302 if (is_variable_sized (decl
))
15304 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15307 f
= (tree
) n
->value
;
15308 if (tcctx
.cb
.decl_map
)
15309 f
= *tcctx
.cb
.decl_map
->get (f
);
15310 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15313 sf
= (tree
) n
->value
;
15314 if (tcctx
.cb
.decl_map
)
15315 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15316 src
= build_simple_mem_ref_loc (loc
, sarg
);
15317 src
= omp_build_component_ref (src
, sf
);
15318 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
15319 src
= build_simple_mem_ref_loc (loc
, src
);
15323 dst
= build_simple_mem_ref_loc (loc
, arg
);
15324 dst
= omp_build_component_ref (dst
, f
);
15325 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15326 append_to_statement_list (t
, &list
);
15328 case OMP_CLAUSE_PRIVATE
:
15329 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
15331 decl
= OMP_CLAUSE_DECL (c
);
15332 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15333 f
= (tree
) n
->value
;
15334 if (tcctx
.cb
.decl_map
)
15335 f
= *tcctx
.cb
.decl_map
->get (f
);
15336 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15339 sf
= (tree
) n
->value
;
15340 if (tcctx
.cb
.decl_map
)
15341 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15342 src
= build_simple_mem_ref_loc (loc
, sarg
);
15343 src
= omp_build_component_ref (src
, sf
);
15344 if (use_pointer_for_field (decl
, NULL
))
15345 src
= build_simple_mem_ref_loc (loc
, src
);
15349 dst
= build_simple_mem_ref_loc (loc
, arg
);
15350 dst
= omp_build_component_ref (dst
, f
);
15351 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15352 append_to_statement_list (t
, &list
);
15358 /* Last pass: handle VLA firstprivates. */
15359 if (tcctx
.cb
.decl_map
)
15360 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15361 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15365 decl
= OMP_CLAUSE_DECL (c
);
15366 if (!is_variable_sized (decl
))
15368 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15371 f
= (tree
) n
->value
;
15372 f
= *tcctx
.cb
.decl_map
->get (f
);
15373 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
15374 ind
= DECL_VALUE_EXPR (decl
);
15375 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
15376 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
15377 n
= splay_tree_lookup (ctx
->sfield_map
,
15378 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15379 sf
= (tree
) n
->value
;
15380 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15381 src
= build_simple_mem_ref_loc (loc
, sarg
);
15382 src
= omp_build_component_ref (src
, sf
);
15383 src
= build_simple_mem_ref_loc (loc
, src
);
15384 dst
= build_simple_mem_ref_loc (loc
, arg
);
15385 dst
= omp_build_component_ref (dst
, f
);
15386 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15387 append_to_statement_list (t
, &list
);
15388 n
= splay_tree_lookup (ctx
->field_map
,
15389 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15390 df
= (tree
) n
->value
;
15391 df
= *tcctx
.cb
.decl_map
->get (df
);
15392 ptr
= build_simple_mem_ref_loc (loc
, arg
);
15393 ptr
= omp_build_component_ref (ptr
, df
);
15394 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
15395 build_fold_addr_expr_loc (loc
, dst
));
15396 append_to_statement_list (t
, &list
);
15399 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
15400 append_to_statement_list (t
, &list
);
15402 if (tcctx
.cb
.decl_map
)
15403 delete tcctx
.cb
.decl_map
;
15404 pop_gimplify_context (NULL
);
15405 BIND_EXPR_BODY (bind
) = list
;
15410 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
15414 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
15416 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
15417 gcc_assert (clauses
);
15418 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15419 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
15420 switch (OMP_CLAUSE_DEPEND_KIND (c
))
15422 case OMP_CLAUSE_DEPEND_IN
:
15425 case OMP_CLAUSE_DEPEND_OUT
:
15426 case OMP_CLAUSE_DEPEND_INOUT
:
15429 case OMP_CLAUSE_DEPEND_SOURCE
:
15430 case OMP_CLAUSE_DEPEND_SINK
:
15433 gcc_unreachable ();
15435 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
15436 tree array
= create_tmp_var (type
);
15437 TREE_ADDRESSABLE (array
) = 1;
15438 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
15440 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
15441 gimple_seq_add_stmt (iseq
, g
);
15442 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
15444 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
15445 gimple_seq_add_stmt (iseq
, g
);
15446 for (i
= 0; i
< 2; i
++)
15448 if ((i
? n_in
: n_out
) == 0)
15450 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15451 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
15452 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
15454 tree t
= OMP_CLAUSE_DECL (c
);
15455 t
= fold_convert (ptr_type_node
, t
);
15456 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
15457 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
15458 NULL_TREE
, NULL_TREE
);
15459 g
= gimple_build_assign (r
, t
);
15460 gimple_seq_add_stmt (iseq
, g
);
15463 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
15464 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
15465 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
15467 tree clobber
= build_constructor (type
, NULL
);
15468 TREE_THIS_VOLATILE (clobber
) = 1;
15469 g
= gimple_build_assign (array
, clobber
);
15470 gimple_seq_add_stmt (oseq
, g
);
15473 /* Lower the OpenMP parallel or task directive in the current statement
15474 in GSI_P. CTX holds context information for the directive. */
15477 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15481 gimple
*stmt
= gsi_stmt (*gsi_p
);
15482 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
15483 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
15484 location_t loc
= gimple_location (stmt
);
15486 clauses
= gimple_omp_taskreg_clauses (stmt
);
15488 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
15489 par_body
= gimple_bind_body (par_bind
);
15490 child_fn
= ctx
->cb
.dst_fn
;
15491 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15492 && !gimple_omp_parallel_combined_p (stmt
))
15494 struct walk_stmt_info wi
;
15497 memset (&wi
, 0, sizeof (wi
));
15499 wi
.val_only
= true;
15500 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
15502 gimple_omp_parallel_set_combined_p (stmt
, true);
15504 gimple_seq dep_ilist
= NULL
;
15505 gimple_seq dep_olist
= NULL
;
15506 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
15507 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15509 push_gimplify_context ();
15510 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15511 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
15512 &dep_ilist
, &dep_olist
);
15515 if (ctx
->srecord_type
)
15516 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
15518 push_gimplify_context ();
15523 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15524 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
15525 if (phony_construct
&& ctx
->record_type
)
15527 gcc_checking_assert (!ctx
->receiver_decl
);
15528 ctx
->receiver_decl
= create_tmp_var
15529 (build_reference_type (ctx
->record_type
), ".omp_rec");
15531 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
15532 lower_omp (&par_body
, ctx
);
15533 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
15534 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
15536 /* Declare all the variables created by mapping and the variables
15537 declared in the scope of the parallel body. */
15538 record_vars_into (ctx
->block_vars
, child_fn
);
15539 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
15541 if (ctx
->record_type
)
15544 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
15545 : ctx
->record_type
, ".omp_data_o");
15546 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15547 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15548 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
15553 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
15554 lower_send_shared_vars (&ilist
, &olist
, ctx
);
15556 if (ctx
->record_type
)
15558 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
15559 TREE_THIS_VOLATILE (clobber
) = 1;
15560 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15564 /* Once all the expansions are done, sequence all the different
15565 fragments inside gimple_omp_body. */
15569 if (ctx
->record_type
)
15571 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15572 /* fixup_child_record_type might have changed receiver_decl's type. */
15573 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15574 gimple_seq_add_stmt (&new_body
,
15575 gimple_build_assign (ctx
->receiver_decl
, t
));
15578 gimple_seq_add_seq (&new_body
, par_ilist
);
15579 gimple_seq_add_seq (&new_body
, par_body
);
15580 gimple_seq_add_seq (&new_body
, par_rlist
);
15581 if (ctx
->cancellable
)
15582 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
15583 gimple_seq_add_seq (&new_body
, par_olist
);
15584 new_body
= maybe_catch_exception (new_body
);
15585 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
15586 gimple_seq_add_stmt (&new_body
,
15587 gimple_build_omp_continue (integer_zero_node
,
15588 integer_zero_node
));
15589 if (!phony_construct
)
15591 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15592 gimple_omp_set_body (stmt
, new_body
);
15595 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
15596 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15597 gimple_bind_add_seq (bind
, ilist
);
15598 if (!phony_construct
)
15599 gimple_bind_add_stmt (bind
, stmt
);
15601 gimple_bind_add_seq (bind
, new_body
);
15602 gimple_bind_add_seq (bind
, olist
);
15604 pop_gimplify_context (NULL
);
15608 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15609 gimple_bind_add_stmt (dep_bind
, bind
);
15610 gimple_bind_add_seq (dep_bind
, dep_olist
);
15611 pop_gimplify_context (dep_bind
);
15615 /* Lower the GIMPLE_OMP_TARGET in the current statement
15616 in GSI_P. CTX holds context information for the directive. */
15619 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15622 tree child_fn
, t
, c
;
15623 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
15624 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
15625 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
15626 location_t loc
= gimple_location (stmt
);
15627 bool offloaded
, data_region
;
15628 unsigned int map_cnt
= 0;
15629 bool has_depend
= false;
15631 offloaded
= is_gimple_omp_offloaded (stmt
);
15632 switch (gimple_omp_target_kind (stmt
))
15634 case GF_OMP_TARGET_KIND_REGION
:
15635 case GF_OMP_TARGET_KIND_UPDATE
:
15636 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15637 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15638 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15639 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15640 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15641 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15642 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
15643 data_region
= false;
15645 case GF_OMP_TARGET_KIND_DATA
:
15646 case GF_OMP_TARGET_KIND_OACC_DATA
:
15647 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
15648 data_region
= true;
15651 gcc_unreachable ();
15654 clauses
= gimple_omp_target_clauses (stmt
);
15656 gimple_seq dep_ilist
= NULL
;
15657 gimple_seq dep_olist
= NULL
;
15658 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15660 push_gimplify_context ();
15661 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15662 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15663 &dep_ilist
, &dep_olist
);
15671 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15672 tgt_body
= gimple_bind_body (tgt_bind
);
15674 else if (data_region
)
15675 tgt_body
= gimple_omp_body (stmt
);
15676 child_fn
= ctx
->cb
.dst_fn
;
15678 push_gimplify_context ();
15681 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15682 switch (OMP_CLAUSE_CODE (c
))
15688 case OMP_CLAUSE_MAP
:
15690 /* First check what we're prepared to handle in the following. */
15691 switch (OMP_CLAUSE_MAP_KIND (c
))
15693 case GOMP_MAP_ALLOC
:
15695 case GOMP_MAP_FROM
:
15696 case GOMP_MAP_TOFROM
:
15697 case GOMP_MAP_POINTER
:
15698 case GOMP_MAP_TO_PSET
:
15699 case GOMP_MAP_FORCE_DEALLOC
:
15700 case GOMP_MAP_RELEASE
:
15701 case GOMP_MAP_ALWAYS_TO
:
15702 case GOMP_MAP_ALWAYS_FROM
:
15703 case GOMP_MAP_ALWAYS_TOFROM
:
15704 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15705 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
15706 case GOMP_MAP_STRUCT
:
15707 case GOMP_MAP_ALWAYS_POINTER
:
15709 case GOMP_MAP_FORCE_ALLOC
:
15710 case GOMP_MAP_FORCE_TO
:
15711 case GOMP_MAP_FORCE_FROM
:
15712 case GOMP_MAP_FORCE_TOFROM
:
15713 case GOMP_MAP_FORCE_PRESENT
:
15714 case GOMP_MAP_FORCE_DEVICEPTR
:
15715 case GOMP_MAP_DEVICE_RESIDENT
:
15716 case GOMP_MAP_LINK
:
15717 gcc_assert (is_gimple_omp_oacc (stmt
));
15720 gcc_unreachable ();
15724 case OMP_CLAUSE_TO
:
15725 case OMP_CLAUSE_FROM
:
15727 var
= OMP_CLAUSE_DECL (c
);
15730 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15731 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15732 && (OMP_CLAUSE_MAP_KIND (c
)
15733 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15738 if (DECL_SIZE (var
)
15739 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15741 tree var2
= DECL_VALUE_EXPR (var
);
15742 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15743 var2
= TREE_OPERAND (var2
, 0);
15744 gcc_assert (DECL_P (var2
));
15749 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15750 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15751 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15753 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15755 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15756 && varpool_node::get_create (var
)->offloadable
)
15759 tree type
= build_pointer_type (TREE_TYPE (var
));
15760 tree new_var
= lookup_decl (var
, ctx
);
15761 x
= create_tmp_var_raw (type
, get_name (new_var
));
15762 gimple_add_tmp_var (x
);
15763 x
= build_simple_mem_ref (x
);
15764 SET_DECL_VALUE_EXPR (new_var
, x
);
15765 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15770 if (!maybe_lookup_field (var
, ctx
))
15775 x
= build_receiver_ref (var
, true, ctx
);
15776 tree new_var
= lookup_decl (var
, ctx
);
15778 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15779 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15780 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15781 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15782 x
= build_simple_mem_ref (x
);
15783 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15785 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15786 if (is_reference (new_var
))
15788 /* Create a local object to hold the instance
15790 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15791 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15792 tree inst
= create_tmp_var (type
, id
);
15793 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15794 x
= build_fold_addr_expr (inst
);
15796 gimplify_assign (new_var
, x
, &fplist
);
15798 else if (DECL_P (new_var
))
15800 SET_DECL_VALUE_EXPR (new_var
, x
);
15801 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15804 gcc_unreachable ();
15809 case OMP_CLAUSE_FIRSTPRIVATE
:
15810 if (is_oacc_parallel (ctx
))
15811 goto oacc_firstprivate
;
15813 var
= OMP_CLAUSE_DECL (c
);
15814 if (!is_reference (var
)
15815 && !is_gimple_reg_type (TREE_TYPE (var
)))
15817 tree new_var
= lookup_decl (var
, ctx
);
15818 if (is_variable_sized (var
))
15820 tree pvar
= DECL_VALUE_EXPR (var
);
15821 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15822 pvar
= TREE_OPERAND (pvar
, 0);
15823 gcc_assert (DECL_P (pvar
));
15824 tree new_pvar
= lookup_decl (pvar
, ctx
);
15825 x
= build_fold_indirect_ref (new_pvar
);
15826 TREE_THIS_NOTRAP (x
) = 1;
15829 x
= build_receiver_ref (var
, true, ctx
);
15830 SET_DECL_VALUE_EXPR (new_var
, x
);
15831 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15835 case OMP_CLAUSE_PRIVATE
:
15836 if (is_gimple_omp_oacc (ctx
->stmt
))
15838 var
= OMP_CLAUSE_DECL (c
);
15839 if (is_variable_sized (var
))
15841 tree new_var
= lookup_decl (var
, ctx
);
15842 tree pvar
= DECL_VALUE_EXPR (var
);
15843 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15844 pvar
= TREE_OPERAND (pvar
, 0);
15845 gcc_assert (DECL_P (pvar
));
15846 tree new_pvar
= lookup_decl (pvar
, ctx
);
15847 x
= build_fold_indirect_ref (new_pvar
);
15848 TREE_THIS_NOTRAP (x
) = 1;
15849 SET_DECL_VALUE_EXPR (new_var
, x
);
15850 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15854 case OMP_CLAUSE_USE_DEVICE_PTR
:
15855 case OMP_CLAUSE_IS_DEVICE_PTR
:
15856 var
= OMP_CLAUSE_DECL (c
);
15858 if (is_variable_sized (var
))
15860 tree new_var
= lookup_decl (var
, ctx
);
15861 tree pvar
= DECL_VALUE_EXPR (var
);
15862 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15863 pvar
= TREE_OPERAND (pvar
, 0);
15864 gcc_assert (DECL_P (pvar
));
15865 tree new_pvar
= lookup_decl (pvar
, ctx
);
15866 x
= build_fold_indirect_ref (new_pvar
);
15867 TREE_THIS_NOTRAP (x
) = 1;
15868 SET_DECL_VALUE_EXPR (new_var
, x
);
15869 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15871 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15873 tree new_var
= lookup_decl (var
, ctx
);
15874 tree type
= build_pointer_type (TREE_TYPE (var
));
15875 x
= create_tmp_var_raw (type
, get_name (new_var
));
15876 gimple_add_tmp_var (x
);
15877 x
= build_simple_mem_ref (x
);
15878 SET_DECL_VALUE_EXPR (new_var
, x
);
15879 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15886 target_nesting_level
++;
15887 lower_omp (&tgt_body
, ctx
);
15888 target_nesting_level
--;
15890 else if (data_region
)
15891 lower_omp (&tgt_body
, ctx
);
15895 /* Declare all the variables created by mapping and the variables
15896 declared in the scope of the target body. */
15897 record_vars_into (ctx
->block_vars
, child_fn
);
15898 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
15903 if (ctx
->record_type
)
15906 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
15907 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15908 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15909 t
= make_tree_vec (3);
15910 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
15911 TREE_VEC_ELT (t
, 1)
15912 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
15913 ".omp_data_sizes");
15914 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
15915 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
15916 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
15917 tree tkind_type
= short_unsigned_type_node
;
15918 int talign_shift
= 8;
15919 TREE_VEC_ELT (t
, 2)
15920 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
15921 ".omp_data_kinds");
15922 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
15923 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
15924 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
15925 gimple_omp_target_set_data_arg (stmt
, t
);
15927 vec
<constructor_elt
, va_gc
> *vsize
;
15928 vec
<constructor_elt
, va_gc
> *vkind
;
15929 vec_alloc (vsize
, map_cnt
);
15930 vec_alloc (vkind
, map_cnt
);
15931 unsigned int map_idx
= 0;
15933 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15934 switch (OMP_CLAUSE_CODE (c
))
15936 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
15937 unsigned int talign
;
15942 case OMP_CLAUSE_MAP
:
15943 case OMP_CLAUSE_TO
:
15944 case OMP_CLAUSE_FROM
:
15945 oacc_firstprivate_map
:
15947 ovar
= OMP_CLAUSE_DECL (c
);
15948 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15949 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15950 || (OMP_CLAUSE_MAP_KIND (c
)
15951 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15953 if (!DECL_P (ovar
))
15955 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15956 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
15958 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
15959 == get_base_address (ovar
));
15960 nc
= OMP_CLAUSE_CHAIN (c
);
15961 ovar
= OMP_CLAUSE_DECL (nc
);
15965 tree x
= build_sender_ref (ovar
, ctx
);
15967 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
15968 gimplify_assign (x
, v
, &ilist
);
15974 if (DECL_SIZE (ovar
)
15975 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
15977 tree ovar2
= DECL_VALUE_EXPR (ovar
);
15978 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
15979 ovar2
= TREE_OPERAND (ovar2
, 0);
15980 gcc_assert (DECL_P (ovar2
));
15983 if (!maybe_lookup_field (ovar
, ctx
))
15987 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
15988 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
15989 talign
= DECL_ALIGN_UNIT (ovar
);
15992 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15993 x
= build_sender_ref (ovar
, ctx
);
15995 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15996 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15997 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15998 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
16000 gcc_assert (offloaded
);
16002 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
16003 mark_addressable (avar
);
16004 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
16005 talign
= DECL_ALIGN_UNIT (avar
);
16006 avar
= build_fold_addr_expr (avar
);
16007 gimplify_assign (x
, avar
, &ilist
);
16009 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16011 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
16012 if (!is_reference (var
))
16013 var
= build_fold_addr_expr (var
);
16015 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16016 gimplify_assign (x
, var
, &ilist
);
16018 else if (is_gimple_reg (var
))
16020 gcc_assert (offloaded
);
16021 tree avar
= create_tmp_var (TREE_TYPE (var
));
16022 mark_addressable (avar
);
16023 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
16024 if (GOMP_MAP_COPY_TO_P (map_kind
)
16025 || map_kind
== GOMP_MAP_POINTER
16026 || map_kind
== GOMP_MAP_TO_PSET
16027 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16028 gimplify_assign (avar
, var
, &ilist
);
16029 avar
= build_fold_addr_expr (avar
);
16030 gimplify_assign (x
, avar
, &ilist
);
16031 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
16032 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16033 && !TYPE_READONLY (TREE_TYPE (var
)))
16035 x
= unshare_expr (x
);
16036 x
= build_simple_mem_ref (x
);
16037 gimplify_assign (var
, x
, &olist
);
16042 var
= build_fold_addr_expr (var
);
16043 gimplify_assign (x
, var
, &ilist
);
16047 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16049 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16050 s
= TREE_TYPE (ovar
);
16051 if (TREE_CODE (s
) == REFERENCE_TYPE
)
16053 s
= TYPE_SIZE_UNIT (s
);
16056 s
= OMP_CLAUSE_SIZE (c
);
16057 if (s
== NULL_TREE
)
16058 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16059 s
= fold_convert (size_type_node
, s
);
16060 purpose
= size_int (map_idx
++);
16061 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16062 if (TREE_CODE (s
) != INTEGER_CST
)
16063 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16065 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
16066 switch (OMP_CLAUSE_CODE (c
))
16068 case OMP_CLAUSE_MAP
:
16069 tkind
= OMP_CLAUSE_MAP_KIND (c
);
16070 tkind_zero
= tkind
;
16071 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
16074 case GOMP_MAP_ALLOC
:
16076 case GOMP_MAP_FROM
:
16077 case GOMP_MAP_TOFROM
:
16078 case GOMP_MAP_ALWAYS_TO
:
16079 case GOMP_MAP_ALWAYS_FROM
:
16080 case GOMP_MAP_ALWAYS_TOFROM
:
16081 case GOMP_MAP_RELEASE
:
16082 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
16084 case GOMP_MAP_DELETE
:
16085 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
16089 if (tkind_zero
!= tkind
)
16091 if (integer_zerop (s
))
16092 tkind
= tkind_zero
;
16093 else if (integer_nonzerop (s
))
16094 tkind_zero
= tkind
;
16097 case OMP_CLAUSE_FIRSTPRIVATE
:
16098 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16099 tkind
= GOMP_MAP_TO
;
16100 tkind_zero
= tkind
;
16102 case OMP_CLAUSE_TO
:
16103 tkind
= GOMP_MAP_TO
;
16104 tkind_zero
= tkind
;
16106 case OMP_CLAUSE_FROM
:
16107 tkind
= GOMP_MAP_FROM
;
16108 tkind_zero
= tkind
;
16111 gcc_unreachable ();
16113 gcc_checking_assert (tkind
16114 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16115 gcc_checking_assert (tkind_zero
16116 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16117 talign
= ceil_log2 (talign
);
16118 tkind
|= talign
<< talign_shift
;
16119 tkind_zero
|= talign
<< talign_shift
;
16120 gcc_checking_assert (tkind
16121 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16122 gcc_checking_assert (tkind_zero
16123 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16124 if (tkind
== tkind_zero
)
16125 x
= build_int_cstu (tkind_type
, tkind
);
16128 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
16129 x
= build3 (COND_EXPR
, tkind_type
,
16130 fold_build2 (EQ_EXPR
, boolean_type_node
,
16131 unshare_expr (s
), size_zero_node
),
16132 build_int_cstu (tkind_type
, tkind_zero
),
16133 build_int_cstu (tkind_type
, tkind
));
16135 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
16140 case OMP_CLAUSE_FIRSTPRIVATE
:
16141 if (is_oacc_parallel (ctx
))
16142 goto oacc_firstprivate_map
;
16143 ovar
= OMP_CLAUSE_DECL (c
);
16144 if (is_reference (ovar
))
16145 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16147 talign
= DECL_ALIGN_UNIT (ovar
);
16148 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16149 x
= build_sender_ref (ovar
, ctx
);
16150 tkind
= GOMP_MAP_FIRSTPRIVATE
;
16151 type
= TREE_TYPE (ovar
);
16152 if (is_reference (ovar
))
16153 type
= TREE_TYPE (type
);
16154 bool use_firstprivate_int
, force_addr
;
16155 use_firstprivate_int
= false;
16156 force_addr
= false;
16157 if ((INTEGRAL_TYPE_P (type
)
16158 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16159 || TREE_CODE (type
) == POINTER_TYPE
)
16160 use_firstprivate_int
= true;
16163 if (is_reference (var
))
16164 use_firstprivate_int
= false;
16165 else if (is_gimple_reg (var
))
16167 if (DECL_HAS_VALUE_EXPR_P (var
))
16169 tree v
= get_base_address (var
);
16170 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
16172 use_firstprivate_int
= false;
16176 switch (TREE_CODE (v
))
16180 use_firstprivate_int
= false;
16189 use_firstprivate_int
= false;
16191 if (use_firstprivate_int
)
16193 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16195 if (is_reference (var
))
16196 t
= build_simple_mem_ref (var
);
16197 if (TREE_CODE (type
) != POINTER_TYPE
)
16198 t
= fold_convert (pointer_sized_int_node
, t
);
16199 t
= fold_convert (TREE_TYPE (x
), t
);
16200 gimplify_assign (x
, t
, &ilist
);
16202 else if (is_reference (var
))
16203 gimplify_assign (x
, var
, &ilist
);
16204 else if (!force_addr
&& is_gimple_reg (var
))
16206 tree avar
= create_tmp_var (TREE_TYPE (var
));
16207 mark_addressable (avar
);
16208 gimplify_assign (avar
, var
, &ilist
);
16209 avar
= build_fold_addr_expr (avar
);
16210 gimplify_assign (x
, avar
, &ilist
);
16214 var
= build_fold_addr_expr (var
);
16215 gimplify_assign (x
, var
, &ilist
);
16217 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
16219 else if (is_reference (var
))
16220 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16222 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16223 s
= fold_convert (size_type_node
, s
);
16224 purpose
= size_int (map_idx
++);
16225 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16226 if (TREE_CODE (s
) != INTEGER_CST
)
16227 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16229 gcc_checking_assert (tkind
16230 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16231 talign
= ceil_log2 (talign
);
16232 tkind
|= talign
<< talign_shift
;
16233 gcc_checking_assert (tkind
16234 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16235 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16236 build_int_cstu (tkind_type
, tkind
));
16239 case OMP_CLAUSE_USE_DEVICE_PTR
:
16240 case OMP_CLAUSE_IS_DEVICE_PTR
:
16241 ovar
= OMP_CLAUSE_DECL (c
);
16242 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16243 x
= build_sender_ref (ovar
, ctx
);
16244 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16245 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
16247 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16248 type
= TREE_TYPE (ovar
);
16249 if (TREE_CODE (type
) == ARRAY_TYPE
)
16250 var
= build_fold_addr_expr (var
);
16253 if (is_reference (ovar
))
16255 type
= TREE_TYPE (type
);
16256 if (TREE_CODE (type
) != ARRAY_TYPE
)
16257 var
= build_simple_mem_ref (var
);
16258 var
= fold_convert (TREE_TYPE (x
), var
);
16261 gimplify_assign (x
, var
, &ilist
);
16263 purpose
= size_int (map_idx
++);
16264 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16265 gcc_checking_assert (tkind
16266 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16267 gcc_checking_assert (tkind
16268 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16269 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16270 build_int_cstu (tkind_type
, tkind
));
16274 gcc_assert (map_idx
== map_cnt
);
16276 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
16277 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
16278 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
16279 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
16280 for (int i
= 1; i
<= 2; i
++)
16281 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
16283 gimple_seq initlist
= NULL
;
16284 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
16285 TREE_VEC_ELT (t
, i
)),
16286 &initlist
, true, NULL_TREE
);
16287 gimple_seq_add_seq (&ilist
, initlist
);
16289 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
16291 TREE_THIS_VOLATILE (clobber
) = 1;
16292 gimple_seq_add_stmt (&olist
,
16293 gimple_build_assign (TREE_VEC_ELT (t
, i
),
16297 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
16298 TREE_THIS_VOLATILE (clobber
) = 1;
16299 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
16303 /* Once all the expansions are done, sequence all the different
16304 fragments inside gimple_omp_body. */
16309 && ctx
->record_type
)
16311 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
16312 /* fixup_child_record_type might have changed receiver_decl's type. */
16313 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
16314 gimple_seq_add_stmt (&new_body
,
16315 gimple_build_assign (ctx
->receiver_decl
, t
));
16317 gimple_seq_add_seq (&new_body
, fplist
);
16319 if (offloaded
|| data_region
)
16321 tree prev
= NULL_TREE
;
16322 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16323 switch (OMP_CLAUSE_CODE (c
))
16328 case OMP_CLAUSE_FIRSTPRIVATE
:
16329 if (is_gimple_omp_oacc (ctx
->stmt
))
16331 var
= OMP_CLAUSE_DECL (c
);
16332 if (is_reference (var
)
16333 || is_gimple_reg_type (TREE_TYPE (var
)))
16335 tree new_var
= lookup_decl (var
, ctx
);
16337 type
= TREE_TYPE (var
);
16338 if (is_reference (var
))
16339 type
= TREE_TYPE (type
);
16340 bool use_firstprivate_int
;
16341 use_firstprivate_int
= false;
16342 if ((INTEGRAL_TYPE_P (type
)
16343 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16344 || TREE_CODE (type
) == POINTER_TYPE
)
16345 use_firstprivate_int
= true;
16348 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
16349 if (is_reference (v
))
16350 use_firstprivate_int
= false;
16351 else if (is_gimple_reg (v
))
16353 if (DECL_HAS_VALUE_EXPR_P (v
))
16355 v
= get_base_address (v
);
16356 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
16357 use_firstprivate_int
= false;
16359 switch (TREE_CODE (v
))
16363 use_firstprivate_int
= false;
16371 use_firstprivate_int
= false;
16373 if (use_firstprivate_int
)
16375 x
= build_receiver_ref (var
, false, ctx
);
16376 if (TREE_CODE (type
) != POINTER_TYPE
)
16377 x
= fold_convert (pointer_sized_int_node
, x
);
16378 x
= fold_convert (type
, x
);
16379 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16381 if (is_reference (var
))
16383 tree v
= create_tmp_var_raw (type
, get_name (var
));
16384 gimple_add_tmp_var (v
);
16385 TREE_ADDRESSABLE (v
) = 1;
16386 gimple_seq_add_stmt (&new_body
,
16387 gimple_build_assign (v
, x
));
16388 x
= build_fold_addr_expr (v
);
16390 gimple_seq_add_stmt (&new_body
,
16391 gimple_build_assign (new_var
, x
));
16395 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
16396 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16398 gimple_seq_add_stmt (&new_body
,
16399 gimple_build_assign (new_var
, x
));
16402 else if (is_variable_sized (var
))
16404 tree pvar
= DECL_VALUE_EXPR (var
);
16405 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16406 pvar
= TREE_OPERAND (pvar
, 0);
16407 gcc_assert (DECL_P (pvar
));
16408 tree new_var
= lookup_decl (pvar
, ctx
);
16409 x
= build_receiver_ref (var
, false, ctx
);
16410 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16411 gimple_seq_add_stmt (&new_body
,
16412 gimple_build_assign (new_var
, x
));
16415 case OMP_CLAUSE_PRIVATE
:
16416 if (is_gimple_omp_oacc (ctx
->stmt
))
16418 var
= OMP_CLAUSE_DECL (c
);
16419 if (is_reference (var
))
16421 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16422 tree new_var
= lookup_decl (var
, ctx
);
16423 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16424 if (TREE_CONSTANT (x
))
16426 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
16428 gimple_add_tmp_var (x
);
16429 TREE_ADDRESSABLE (x
) = 1;
16430 x
= build_fold_addr_expr_loc (clause_loc
, x
);
16435 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16436 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
16437 tree al
= size_int (TYPE_ALIGN (rtype
));
16438 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16441 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16442 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16443 gimple_seq_add_stmt (&new_body
,
16444 gimple_build_assign (new_var
, x
));
16447 case OMP_CLAUSE_USE_DEVICE_PTR
:
16448 case OMP_CLAUSE_IS_DEVICE_PTR
:
16449 var
= OMP_CLAUSE_DECL (c
);
16450 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16451 x
= build_sender_ref (var
, ctx
);
16453 x
= build_receiver_ref (var
, false, ctx
);
16454 if (is_variable_sized (var
))
16456 tree pvar
= DECL_VALUE_EXPR (var
);
16457 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16458 pvar
= TREE_OPERAND (pvar
, 0);
16459 gcc_assert (DECL_P (pvar
));
16460 tree new_var
= lookup_decl (pvar
, ctx
);
16461 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16462 gimple_seq_add_stmt (&new_body
,
16463 gimple_build_assign (new_var
, x
));
16465 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16467 tree new_var
= lookup_decl (var
, ctx
);
16468 new_var
= DECL_VALUE_EXPR (new_var
);
16469 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
16470 new_var
= TREE_OPERAND (new_var
, 0);
16471 gcc_assert (DECL_P (new_var
));
16472 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16473 gimple_seq_add_stmt (&new_body
,
16474 gimple_build_assign (new_var
, x
));
16478 tree type
= TREE_TYPE (var
);
16479 tree new_var
= lookup_decl (var
, ctx
);
16480 if (is_reference (var
))
16482 type
= TREE_TYPE (type
);
16483 if (TREE_CODE (type
) != ARRAY_TYPE
)
16485 tree v
= create_tmp_var_raw (type
, get_name (var
));
16486 gimple_add_tmp_var (v
);
16487 TREE_ADDRESSABLE (v
) = 1;
16488 x
= fold_convert (type
, x
);
16489 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16491 gimple_seq_add_stmt (&new_body
,
16492 gimple_build_assign (v
, x
));
16493 x
= build_fold_addr_expr (v
);
16496 x
= fold_convert (TREE_TYPE (new_var
), x
);
16497 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16498 gimple_seq_add_stmt (&new_body
,
16499 gimple_build_assign (new_var
, x
));
16503 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16504 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16505 are already handled. */
16506 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16507 switch (OMP_CLAUSE_CODE (c
))
16512 case OMP_CLAUSE_MAP
:
16513 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16514 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16516 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16517 HOST_WIDE_INT offset
= 0;
16519 var
= OMP_CLAUSE_DECL (c
);
16521 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
16522 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
16524 && varpool_node::get_create (var
)->offloadable
)
16526 if (TREE_CODE (var
) == INDIRECT_REF
16527 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
16528 var
= TREE_OPERAND (var
, 0);
16529 if (TREE_CODE (var
) == COMPONENT_REF
)
16531 var
= get_addr_base_and_unit_offset (var
, &offset
);
16532 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
16534 else if (DECL_SIZE (var
)
16535 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
16537 tree var2
= DECL_VALUE_EXPR (var
);
16538 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
16539 var2
= TREE_OPERAND (var2
, 0);
16540 gcc_assert (DECL_P (var2
));
16543 tree new_var
= lookup_decl (var
, ctx
), x
;
16544 tree type
= TREE_TYPE (new_var
);
16546 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
16547 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
16550 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
16552 new_var
= build2 (MEM_REF
, type
,
16553 build_fold_addr_expr (new_var
),
16554 build_int_cst (build_pointer_type (type
),
16557 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
16559 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
16560 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
16561 new_var
= build2 (MEM_REF
, type
,
16562 build_fold_addr_expr (new_var
),
16563 build_int_cst (build_pointer_type (type
),
16567 is_ref
= is_reference (var
);
16568 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16570 bool ref_to_array
= false;
16573 type
= TREE_TYPE (type
);
16574 if (TREE_CODE (type
) == ARRAY_TYPE
)
16576 type
= build_pointer_type (type
);
16577 ref_to_array
= true;
16580 else if (TREE_CODE (type
) == ARRAY_TYPE
)
16582 tree decl2
= DECL_VALUE_EXPR (new_var
);
16583 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
16584 decl2
= TREE_OPERAND (decl2
, 0);
16585 gcc_assert (DECL_P (decl2
));
16587 type
= TREE_TYPE (new_var
);
16589 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
16590 x
= fold_convert_loc (clause_loc
, type
, x
);
16591 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
16593 tree bias
= OMP_CLAUSE_SIZE (c
);
16595 bias
= lookup_decl (bias
, ctx
);
16596 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
16597 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
16599 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
16600 TREE_TYPE (x
), x
, bias
);
16603 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16604 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16605 if (is_ref
&& !ref_to_array
)
16607 tree t
= create_tmp_var_raw (type
, get_name (var
));
16608 gimple_add_tmp_var (t
);
16609 TREE_ADDRESSABLE (t
) = 1;
16610 gimple_seq_add_stmt (&new_body
,
16611 gimple_build_assign (t
, x
));
16612 x
= build_fold_addr_expr_loc (clause_loc
, t
);
16614 gimple_seq_add_stmt (&new_body
,
16615 gimple_build_assign (new_var
, x
));
16618 else if (OMP_CLAUSE_CHAIN (c
)
16619 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
16621 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16622 == GOMP_MAP_FIRSTPRIVATE_POINTER
16623 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16624 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16627 case OMP_CLAUSE_PRIVATE
:
16628 var
= OMP_CLAUSE_DECL (c
);
16629 if (is_variable_sized (var
))
16631 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16632 tree new_var
= lookup_decl (var
, ctx
);
16633 tree pvar
= DECL_VALUE_EXPR (var
);
16634 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16635 pvar
= TREE_OPERAND (pvar
, 0);
16636 gcc_assert (DECL_P (pvar
));
16637 tree new_pvar
= lookup_decl (pvar
, ctx
);
16638 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16639 tree al
= size_int (DECL_ALIGN (var
));
16640 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
16641 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16642 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
16643 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16644 gimple_seq_add_stmt (&new_body
,
16645 gimple_build_assign (new_pvar
, x
));
16650 gimple_seq fork_seq
= NULL
;
16651 gimple_seq join_seq
= NULL
;
16653 if (is_oacc_parallel (ctx
))
16655 /* If there are reductions on the offloaded region itself, treat
16656 them as a dummy GANG loop. */
16657 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
16659 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
16660 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
16663 gimple_seq_add_seq (&new_body
, fork_seq
);
16664 gimple_seq_add_seq (&new_body
, tgt_body
);
16665 gimple_seq_add_seq (&new_body
, join_seq
);
16668 new_body
= maybe_catch_exception (new_body
);
16670 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
16671 gimple_omp_set_body (stmt
, new_body
);
16674 bind
= gimple_build_bind (NULL
, NULL
,
16675 tgt_bind
? gimple_bind_block (tgt_bind
)
16677 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
16678 gimple_bind_add_seq (bind
, ilist
);
16679 gimple_bind_add_stmt (bind
, stmt
);
16680 gimple_bind_add_seq (bind
, olist
);
16682 pop_gimplify_context (NULL
);
16686 gimple_bind_add_seq (dep_bind
, dep_ilist
);
16687 gimple_bind_add_stmt (dep_bind
, bind
);
16688 gimple_bind_add_seq (dep_bind
, dep_olist
);
16689 pop_gimplify_context (dep_bind
);
16693 /* Expand code for an OpenMP teams directive. */
16696 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16698 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
16699 push_gimplify_context ();
16701 tree block
= make_node (BLOCK
);
16702 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
16703 gsi_replace (gsi_p
, bind
, true);
16704 gimple_seq bind_body
= NULL
;
16705 gimple_seq dlist
= NULL
;
16706 gimple_seq olist
= NULL
;
16708 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16709 OMP_CLAUSE_NUM_TEAMS
);
16710 if (num_teams
== NULL_TREE
)
16711 num_teams
= build_int_cst (unsigned_type_node
, 0);
16714 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16715 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16716 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16718 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16719 OMP_CLAUSE_THREAD_LIMIT
);
16720 if (thread_limit
== NULL_TREE
)
16721 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16724 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16725 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16726 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16730 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16731 &bind_body
, &dlist
, ctx
, NULL
);
16732 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16733 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16734 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16736 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16737 location_t loc
= gimple_location (teams_stmt
);
16738 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16739 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16740 gimple_set_location (call
, loc
);
16741 gimple_seq_add_stmt (&bind_body
, call
);
16744 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16745 gimple_omp_set_body (teams_stmt
, NULL
);
16746 gimple_seq_add_seq (&bind_body
, olist
);
16747 gimple_seq_add_seq (&bind_body
, dlist
);
16748 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16749 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16750 gimple_bind_set_body (bind
, bind_body
);
16752 pop_gimplify_context (bind
);
16754 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16755 BLOCK_VARS (block
) = ctx
->block_vars
;
16756 if (BLOCK_VARS (block
))
16757 TREE_USED (block
) = 1;
16760 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16763 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16765 gimple
*stmt
= gsi_stmt (*gsi_p
);
16766 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
16767 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
16768 gimple_build_omp_return (false));
16772 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16773 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16774 of OMP context, but with task_shared_vars set. */
16777 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16782 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16783 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16786 if (task_shared_vars
16788 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16791 /* If a global variable has been privatized, TREE_CONSTANT on
16792 ADDR_EXPR might be wrong. */
16793 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16794 recompute_tree_invariant_for_addr_expr (t
);
16796 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16800 /* Data to be communicated between lower_omp_regimplify_operands and
16801 lower_omp_regimplify_operands_p. */
16803 struct lower_omp_regimplify_operands_data
16809 /* Helper function for lower_omp_regimplify_operands. Find
16810 omp_member_access_dummy_var vars and adjust temporarily their
16811 DECL_VALUE_EXPRs if needed. */
16814 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16817 tree t
= omp_member_access_dummy_var (*tp
);
16820 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16821 lower_omp_regimplify_operands_data
*ldata
16822 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16823 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16826 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16827 ldata
->decls
->safe_push (*tp
);
16828 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16829 SET_DECL_VALUE_EXPR (*tp
, v
);
16832 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16836 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16837 of omp_member_access_dummy_var vars during regimplification. */
16840 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16841 gimple_stmt_iterator
*gsi_p
)
16843 auto_vec
<tree
, 10> decls
;
16846 struct walk_stmt_info wi
;
16847 memset (&wi
, '\0', sizeof (wi
));
16848 struct lower_omp_regimplify_operands_data data
;
16850 data
.decls
= &decls
;
16852 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16854 gimple_regimplify_operands (stmt
, gsi_p
);
16855 while (!decls
.is_empty ())
16857 tree t
= decls
.pop ();
16858 tree v
= decls
.pop ();
16859 SET_DECL_VALUE_EXPR (t
, v
);
16864 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16866 gimple
*stmt
= gsi_stmt (*gsi_p
);
16867 struct walk_stmt_info wi
;
16870 if (gimple_has_location (stmt
))
16871 input_location
= gimple_location (stmt
);
16873 if (task_shared_vars
)
16874 memset (&wi
, '\0', sizeof (wi
));
16876 /* If we have issued syntax errors, avoid doing any heavy lifting.
16877 Just replace the OMP directives with a NOP to avoid
16878 confusing RTL expansion. */
16879 if (seen_error () && is_gimple_omp (stmt
))
16881 gsi_replace (gsi_p
, gimple_build_nop (), true);
16885 switch (gimple_code (stmt
))
16889 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16890 if ((ctx
|| task_shared_vars
)
16891 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16892 lower_omp_regimplify_p
,
16893 ctx
? NULL
: &wi
, NULL
)
16894 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16895 lower_omp_regimplify_p
,
16896 ctx
? NULL
: &wi
, NULL
)))
16897 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16901 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16903 case GIMPLE_EH_FILTER
:
16904 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16907 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16908 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16910 case GIMPLE_TRANSACTION
:
16911 lower_omp (gimple_transaction_body_ptr (
16912 as_a
<gtransaction
*> (stmt
)),
16916 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16918 case GIMPLE_OMP_PARALLEL
:
16919 case GIMPLE_OMP_TASK
:
16920 ctx
= maybe_lookup_ctx (stmt
);
16922 if (ctx
->cancellable
)
16923 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16924 lower_omp_taskreg (gsi_p
, ctx
);
16926 case GIMPLE_OMP_FOR
:
16927 ctx
= maybe_lookup_ctx (stmt
);
16929 if (ctx
->cancellable
)
16930 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16931 lower_omp_for (gsi_p
, ctx
);
16933 case GIMPLE_OMP_SECTIONS
:
16934 ctx
= maybe_lookup_ctx (stmt
);
16936 if (ctx
->cancellable
)
16937 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16938 lower_omp_sections (gsi_p
, ctx
);
16940 case GIMPLE_OMP_SINGLE
:
16941 ctx
= maybe_lookup_ctx (stmt
);
16943 lower_omp_single (gsi_p
, ctx
);
16945 case GIMPLE_OMP_MASTER
:
16946 ctx
= maybe_lookup_ctx (stmt
);
16948 lower_omp_master (gsi_p
, ctx
);
16950 case GIMPLE_OMP_TASKGROUP
:
16951 ctx
= maybe_lookup_ctx (stmt
);
16953 lower_omp_taskgroup (gsi_p
, ctx
);
16955 case GIMPLE_OMP_ORDERED
:
16956 ctx
= maybe_lookup_ctx (stmt
);
16958 lower_omp_ordered (gsi_p
, ctx
);
16960 case GIMPLE_OMP_CRITICAL
:
16961 ctx
= maybe_lookup_ctx (stmt
);
16963 lower_omp_critical (gsi_p
, ctx
);
16965 case GIMPLE_OMP_ATOMIC_LOAD
:
16966 if ((ctx
|| task_shared_vars
)
16967 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
16968 as_a
<gomp_atomic_load
*> (stmt
)),
16969 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
16970 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16972 case GIMPLE_OMP_TARGET
:
16973 ctx
= maybe_lookup_ctx (stmt
);
16975 lower_omp_target (gsi_p
, ctx
);
16977 case GIMPLE_OMP_TEAMS
:
16978 ctx
= maybe_lookup_ctx (stmt
);
16980 lower_omp_teams (gsi_p
, ctx
);
16982 case GIMPLE_OMP_GRID_BODY
:
16983 ctx
= maybe_lookup_ctx (stmt
);
16985 lower_omp_grid_body (gsi_p
, ctx
);
16989 call_stmt
= as_a
<gcall
*> (stmt
);
16990 fndecl
= gimple_call_fndecl (call_stmt
);
16992 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
16993 switch (DECL_FUNCTION_CODE (fndecl
))
16995 case BUILT_IN_GOMP_BARRIER
:
16999 case BUILT_IN_GOMP_CANCEL
:
17000 case BUILT_IN_GOMP_CANCELLATION_POINT
:
17003 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
17004 cctx
= cctx
->outer
;
17005 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
17006 if (!cctx
->cancellable
)
17008 if (DECL_FUNCTION_CODE (fndecl
)
17009 == BUILT_IN_GOMP_CANCELLATION_POINT
)
17011 stmt
= gimple_build_nop ();
17012 gsi_replace (gsi_p
, stmt
, false);
17016 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
17018 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
17019 gimple_call_set_fndecl (call_stmt
, fndecl
);
17020 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
17023 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
17024 gimple_call_set_lhs (call_stmt
, lhs
);
17025 tree fallthru_label
;
17026 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
17028 g
= gimple_build_label (fallthru_label
);
17029 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17030 g
= gimple_build_cond (NE_EXPR
, lhs
,
17031 fold_convert (TREE_TYPE (lhs
),
17032 boolean_false_node
),
17033 cctx
->cancel_label
, fallthru_label
);
17034 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17041 if ((ctx
|| task_shared_vars
)
17042 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
17045 /* Just remove clobbers, this should happen only if we have
17046 "privatized" local addressable variables in SIMD regions,
17047 the clobber isn't needed in that case and gimplifying address
17048 of the ARRAY_REF into a pointer and creating MEM_REF based
17049 clobber would create worse code than we get with the clobber
17051 if (gimple_clobber_p (stmt
))
17053 gsi_replace (gsi_p
, gimple_build_nop (), true);
17056 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17063 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
17065 location_t saved_location
= input_location
;
17066 gimple_stmt_iterator gsi
;
17067 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17068 lower_omp_1 (&gsi
, ctx
);
17069 /* During gimplification, we haven't folded statments inside offloading
17070 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17071 if (target_nesting_level
|| taskreg_nesting_level
)
17072 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17074 input_location
= saved_location
;
17077 /* Returen true if STMT is an assignment of a register-type into a local
17081 grid_reg_assignment_to_local_var_p (gimple
*stmt
)
17083 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
17086 tree lhs
= gimple_assign_lhs (assign
);
17087 if (TREE_CODE (lhs
) != VAR_DECL
17088 || !is_gimple_reg_type (TREE_TYPE (lhs
))
17089 || is_global_var (lhs
))
17094 /* Return true if all statements in SEQ are assignments to local register-type
17098 grid_seq_only_contains_local_assignments (gimple_seq seq
)
17103 gimple_stmt_iterator gsi
;
17104 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17105 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi
)))
17110 /* Scan statements in SEQ and call itself recursively on any bind. If during
17111 whole search only assignments to register-type local variables and one
17112 single OMP statement is encountered, return true, otherwise return false.
17113 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17114 are used for dumping a note about a failure. */
17117 grid_find_single_omp_among_assignments_1 (gimple_seq seq
, location_t target_loc
,
17118 const char *name
, gimple
**ret
)
17120 gimple_stmt_iterator gsi
;
17121 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17123 gimple
*stmt
= gsi_stmt (gsi
);
17125 if (grid_reg_assignment_to_local_var_p (stmt
))
17127 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17129 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind
),
17130 target_loc
, name
, ret
))
17133 else if (is_gimple_omp (stmt
))
17137 if (dump_enabled_p ())
17138 dump_printf_loc (MSG_NOTE
, target_loc
,
17139 "Will not turn target construct into a simple "
17140 "GPGPU kernel because %s construct contains "
17141 "multiple OpenMP constructs\n", name
);
17148 if (dump_enabled_p ())
17149 dump_printf_loc (MSG_NOTE
, target_loc
,
17150 "Will not turn target construct into a simple "
17151 "GPGPU kernel because %s construct contains "
17152 "a complex statement\n", name
);
17159 /* Scan statements in SEQ and make sure that it and any binds in it contain
17160 only assignments to local register-type variables and one OMP construct. If
17161 so, return that construct, otherwise return NULL. If dumping is enabled and
17162 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17166 grid_find_single_omp_among_assignments (gimple_seq seq
, location_t target_loc
,
17171 if (dump_enabled_p ())
17172 dump_printf_loc (MSG_NOTE
, target_loc
,
17173 "Will not turn target construct into a simple "
17174 "GPGPU kernel because %s construct has empty "
17180 gimple
*ret
= NULL
;
17181 if (grid_find_single_omp_among_assignments_1 (seq
, target_loc
, name
, &ret
))
17183 if (!ret
&& dump_enabled_p ())
17184 dump_printf_loc (MSG_NOTE
, target_loc
,
17185 "Will not turn target construct into a simple "
17186 "GPGPU kernel because %s construct does not contain"
17187 "any other OpenMP construct\n", name
);
17194 /* Walker function looking for statements there is no point gridifying (and for
17195 noreturn function calls which we cannot do). Return non-NULL if such a
17196 function is found. */
17199 grid_find_ungridifiable_statement (gimple_stmt_iterator
*gsi
,
17200 bool *handled_ops_p
,
17201 struct walk_stmt_info
*)
17203 *handled_ops_p
= false;
17204 gimple
*stmt
= gsi_stmt (*gsi
);
17205 switch (gimple_code (stmt
))
17208 if (gimple_call_noreturn_p (as_a
<gcall
*> (stmt
)))
17210 *handled_ops_p
= true;
17211 return error_mark_node
;
17215 /* We may reduce the following list if we find a way to implement the
17216 clauses, but now there is no point trying further. */
17217 case GIMPLE_OMP_CRITICAL
:
17218 case GIMPLE_OMP_TASKGROUP
:
17219 case GIMPLE_OMP_TASK
:
17220 case GIMPLE_OMP_SECTION
:
17221 case GIMPLE_OMP_SECTIONS
:
17222 case GIMPLE_OMP_SECTIONS_SWITCH
:
17223 case GIMPLE_OMP_TARGET
:
17224 case GIMPLE_OMP_ORDERED
:
17225 *handled_ops_p
= true;
17226 return error_mark_node
;
17235 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17236 kernel, return true, otherwise return false. In the case of success, also
17237 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17241 grid_target_follows_gridifiable_pattern (gomp_target
*target
, tree
*group_size_p
)
17243 if (gimple_omp_target_kind (target
) != GF_OMP_TARGET_KIND_REGION
)
17246 location_t tloc
= gimple_location (target
);
17248 = grid_find_single_omp_among_assignments (gimple_omp_body (target
),
17252 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17253 tree group_size
= NULL
;
17256 dump_printf_loc (MSG_NOTE
, tloc
,
17257 "Will not turn target construct into a simple "
17258 "GPGPU kernel because it does not have a sole teams "
17259 "construct in it.\n");
17263 tree clauses
= gimple_omp_teams_clauses (teams
);
17266 switch (OMP_CLAUSE_CODE (clauses
))
17268 case OMP_CLAUSE_NUM_TEAMS
:
17269 if (dump_enabled_p ())
17270 dump_printf_loc (MSG_NOTE
, tloc
,
17271 "Will not turn target construct into a "
17272 "gridified GPGPU kernel because we cannot "
17273 "handle num_teams clause of teams "
17277 case OMP_CLAUSE_REDUCTION
:
17278 if (dump_enabled_p ())
17279 dump_printf_loc (MSG_NOTE
, tloc
,
17280 "Will not turn target construct into a "
17281 "gridified GPGPU kernel because a reduction "
17282 "clause is present\n ");
17285 case OMP_CLAUSE_LASTPRIVATE
:
17286 if (dump_enabled_p ())
17287 dump_printf_loc (MSG_NOTE
, tloc
,
17288 "Will not turn target construct into a "
17289 "gridified GPGPU kernel because a lastprivate "
17290 "clause is present\n ");
17293 case OMP_CLAUSE_THREAD_LIMIT
:
17294 group_size
= OMP_CLAUSE_OPERAND (clauses
, 0);
17300 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17303 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (teams
), tloc
,
17307 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17310 dump_printf_loc (MSG_NOTE
, tloc
,
17311 "Will not turn target construct into a simple "
17312 "GPGPU kernel because the teams construct does not have "
17313 "a sole distribute construct in it.\n");
17317 gcc_assert (gimple_omp_for_kind (dist
) == GF_OMP_FOR_KIND_DISTRIBUTE
);
17318 if (!gimple_omp_for_combined_p (dist
))
17320 if (dump_enabled_p ())
17321 dump_printf_loc (MSG_NOTE
, tloc
,
17322 "Will not turn target construct into a gridified GPGPU "
17323 "kernel because we cannot handle a standalone "
17324 "distribute construct\n ");
17327 if (dist
->collapse
> 1)
17329 if (dump_enabled_p ())
17330 dump_printf_loc (MSG_NOTE
, tloc
,
17331 "Will not turn target construct into a gridified GPGPU "
17332 "kernel because the distribute construct contains "
17333 "collapse clause\n");
17336 struct omp_for_data fd
;
17337 extract_omp_for_data (dist
, &fd
, NULL
);
17340 if (group_size
&& !operand_equal_p (group_size
, fd
.chunk_size
, 0))
17342 if (dump_enabled_p ())
17343 dump_printf_loc (MSG_NOTE
, tloc
,
17344 "Will not turn target construct into a "
17345 "gridified GPGPU kernel because the teams "
17346 "thread limit is different from distribute "
17347 "schedule chunk\n");
17350 group_size
= fd
.chunk_size
;
17352 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (dist
), tloc
,
17354 gomp_parallel
*par
;
17355 if (!stmt
|| !(par
= dyn_cast
<gomp_parallel
*> (stmt
)))
17358 clauses
= gimple_omp_parallel_clauses (par
);
17361 switch (OMP_CLAUSE_CODE (clauses
))
17363 case OMP_CLAUSE_NUM_THREADS
:
17364 if (dump_enabled_p ())
17365 dump_printf_loc (MSG_NOTE
, tloc
,
17366 "Will not turn target construct into a gridified"
17367 "GPGPU kernel because there is a num_threads "
17368 "clause of the parallel construct\n");
17371 case OMP_CLAUSE_REDUCTION
:
17372 if (dump_enabled_p ())
17373 dump_printf_loc (MSG_NOTE
, tloc
,
17374 "Will not turn target construct into a "
17375 "gridified GPGPU kernel because a reduction "
17376 "clause is present\n ");
17379 case OMP_CLAUSE_LASTPRIVATE
:
17380 if (dump_enabled_p ())
17381 dump_printf_loc (MSG_NOTE
, tloc
,
17382 "Will not turn target construct into a "
17383 "gridified GPGPU kernel because a lastprivate "
17384 "clause is present\n ");
17390 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17393 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (par
), tloc
,
17396 if (!stmt
|| !(gfor
= dyn_cast
<gomp_for
*> (stmt
)))
17399 if (gimple_omp_for_kind (gfor
) != GF_OMP_FOR_KIND_FOR
)
17401 if (dump_enabled_p ())
17402 dump_printf_loc (MSG_NOTE
, tloc
,
17403 "Will not turn target construct into a gridified GPGPU "
17404 "kernel because the inner loop is not a simple for "
17408 if (gfor
->collapse
> 1)
17410 if (dump_enabled_p ())
17411 dump_printf_loc (MSG_NOTE
, tloc
,
17412 "Will not turn target construct into a gridified GPGPU "
17413 "kernel because the inner loop contains collapse "
17418 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor
)))
17420 if (dump_enabled_p ())
17421 dump_printf_loc (MSG_NOTE
, tloc
,
17422 "Will not turn target construct into a gridified GPGPU "
17423 "kernel because the inner loop pre_body contains"
17424 "a complex instruction\n");
17428 clauses
= gimple_omp_for_clauses (gfor
);
17431 switch (OMP_CLAUSE_CODE (clauses
))
17433 case OMP_CLAUSE_SCHEDULE
:
17434 if (OMP_CLAUSE_SCHEDULE_KIND (clauses
) != OMP_CLAUSE_SCHEDULE_AUTO
)
17436 if (dump_enabled_p ())
17437 dump_printf_loc (MSG_NOTE
, tloc
,
17438 "Will not turn target construct into a "
17439 "gridified GPGPU kernel because the inner "
17440 "loop has a non-automatic scheduling clause\n");
17445 case OMP_CLAUSE_REDUCTION
:
17446 if (dump_enabled_p ())
17447 dump_printf_loc (MSG_NOTE
, tloc
,
17448 "Will not turn target construct into a "
17449 "gridified GPGPU kernel because a reduction "
17450 "clause is present\n ");
17453 case OMP_CLAUSE_LASTPRIVATE
:
17454 if (dump_enabled_p ())
17455 dump_printf_loc (MSG_NOTE
, tloc
,
17456 "Will not turn target construct into a "
17457 "gridified GPGPU kernel because a lastprivate "
17458 "clause is present\n ");
17464 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17467 struct walk_stmt_info wi
;
17468 memset (&wi
, 0, sizeof (wi
));
17469 if (gimple
*bad
= walk_gimple_seq (gimple_omp_body (gfor
),
17470 grid_find_ungridifiable_statement
,
17473 if (dump_enabled_p ())
17475 if (is_gimple_call (bad
))
17476 dump_printf_loc (MSG_NOTE
, tloc
,
17477 "Will not turn target construct into a gridified "
17478 " GPGPU kernel because the inner loop contains "
17479 "call to a noreturn function\n");
17481 dump_printf_loc (MSG_NOTE
, tloc
,
17482 "Will not turn target construct into a gridified "
17483 "GPGPU kernel because the inner loop contains "
17484 "statement %s which cannot be transformed\n",
17485 gimple_code_name
[(int) gimple_code (bad
)]);
17490 *group_size_p
= group_size
;
17494 /* Operand walker, used to remap pre-body declarations according to a hash map
17495 provided in DATA. */
17498 grid_remap_prebody_decls (tree
*tp
, int *walk_subtrees
, void *data
)
17502 if (DECL_P (t
) || TYPE_P (t
))
17503 *walk_subtrees
= 0;
17505 *walk_subtrees
= 1;
17507 if (TREE_CODE (t
) == VAR_DECL
)
17509 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17510 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17511 tree
*repl
= declmap
->get (t
);
17518 /* Copy leading register-type assignments to local variables in SRC to just
17519 before DST, Creating temporaries, adjusting mapping of operands in WI and
17520 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17521 Return the first statement that does not conform to
17522 grid_reg_assignment_to_local_var_p or NULL. */
17525 grid_copy_leading_local_assignments (gimple_seq src
, gimple_stmt_iterator
*dst
,
17526 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17528 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17529 gimple_stmt_iterator gsi
;
17530 for (gsi
= gsi_start (src
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17532 gimple
*stmt
= gsi_stmt (gsi
);
17533 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17535 gimple
*r
= grid_copy_leading_local_assignments
17536 (gimple_bind_body (bind
), dst
, tgt_bind
, wi
);
17542 if (!grid_reg_assignment_to_local_var_p (stmt
))
17544 tree lhs
= gimple_assign_lhs (as_a
<gassign
*> (stmt
));
17545 tree repl
= copy_var_decl (lhs
, create_tmp_var_name (NULL
),
17547 DECL_CONTEXT (repl
) = current_function_decl
;
17548 gimple_bind_append_vars (tgt_bind
, repl
);
17550 declmap
->put (lhs
, repl
);
17551 gassign
*copy
= as_a
<gassign
*> (gimple_copy (stmt
));
17552 walk_gimple_op (copy
, grid_remap_prebody_decls
, wi
);
17553 gsi_insert_before (dst
, copy
, GSI_SAME_STMT
);
17558 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17559 components, mark them as part of kernel and return the inner loop, and copy
17560 assignment leading to them just before DST, remapping them using WI and
17561 adding new temporaries to TGT_BIND. */
17564 grid_process_kernel_body_copy (gimple_seq seq
, gimple_stmt_iterator
*dst
,
17565 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17567 gimple
*stmt
= grid_copy_leading_local_assignments (seq
, dst
, tgt_bind
, wi
);
17568 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17569 gcc_assert (teams
);
17570 gimple_omp_teams_set_grid_phony (teams
, true);
17571 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (teams
), dst
,
17573 gcc_checking_assert (stmt
);
17574 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17576 gimple_seq prebody
= gimple_omp_for_pre_body (dist
);
17578 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17579 gimple_omp_for_set_grid_phony (dist
, true);
17580 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (dist
), dst
,
17582 gcc_checking_assert (stmt
);
17584 gomp_parallel
*parallel
= as_a
<gomp_parallel
*> (stmt
);
17585 gimple_omp_parallel_set_grid_phony (parallel
, true);
17586 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (parallel
), dst
,
17588 gomp_for
*inner_loop
= as_a
<gomp_for
*> (stmt
);
17589 gimple_omp_for_set_kind (inner_loop
, GF_OMP_FOR_KIND_GRID_LOOP
);
17590 prebody
= gimple_omp_for_pre_body (inner_loop
);
17592 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17597 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17598 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17599 is the bind into which temporaries inserted before TARGET should be
17603 grid_attempt_target_gridification (gomp_target
*target
,
17604 gimple_stmt_iterator
*gsi
,
17608 if (!target
|| !grid_target_follows_gridifiable_pattern (target
, &group_size
))
17611 location_t loc
= gimple_location (target
);
17612 if (dump_enabled_p ())
17613 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
17614 "Target construct will be turned into a gridified GPGPU "
17617 /* Copy target body to a GPUKERNEL construct: */
17618 gimple_seq kernel_seq
= copy_gimple_seq_and_replace_locals
17619 (gimple_omp_body (target
));
17621 hash_map
<tree
, tree
> *declmap
= new hash_map
<tree
, tree
>;
17622 struct walk_stmt_info wi
;
17623 memset (&wi
, 0, sizeof (struct walk_stmt_info
));
17626 /* Copy assignments in between OMP statements before target, mark OMP
17627 statements within copy appropriatly. */
17628 gomp_for
*inner_loop
= grid_process_kernel_body_copy (kernel_seq
, gsi
,
17631 gbind
*old_bind
= as_a
<gbind
*> (gimple_seq_first (gimple_omp_body (target
)));
17632 gbind
*new_bind
= as_a
<gbind
*> (gimple_seq_first (kernel_seq
));
17633 tree new_block
= gimple_bind_block (new_bind
);
17634 tree enc_block
= BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind
));
17635 BLOCK_CHAIN (new_block
) = BLOCK_SUBBLOCKS (enc_block
);
17636 BLOCK_SUBBLOCKS (enc_block
) = new_block
;
17637 BLOCK_SUPERCONTEXT (new_block
) = enc_block
;
17638 gimple
*gpukernel
= gimple_build_omp_grid_body (kernel_seq
);
17639 gimple_seq_add_stmt
17640 (gimple_bind_body_ptr (as_a
<gbind
*> (gimple_omp_body (target
))),
17643 walk_tree (&group_size
, grid_remap_prebody_decls
, &wi
, NULL
);
17644 push_gimplify_context ();
17645 size_t collapse
= gimple_omp_for_collapse (inner_loop
);
17646 for (size_t i
= 0; i
< collapse
; i
++)
17648 tree itype
, type
= TREE_TYPE (gimple_omp_for_index (inner_loop
, i
));
17649 if (POINTER_TYPE_P (type
))
17650 itype
= signed_type_for (type
);
17654 enum tree_code cond_code
= gimple_omp_for_cond (inner_loop
, i
);
17655 tree n1
= unshare_expr (gimple_omp_for_initial (inner_loop
, i
));
17656 walk_tree (&n1
, grid_remap_prebody_decls
, &wi
, NULL
);
17657 tree n2
= unshare_expr (gimple_omp_for_final (inner_loop
, i
));
17658 walk_tree (&n2
, grid_remap_prebody_decls
, &wi
, NULL
);
17659 adjust_for_condition (loc
, &cond_code
, &n2
);
17661 step
= get_omp_for_step_from_incr (loc
,
17662 gimple_omp_for_incr (inner_loop
, i
));
17663 gimple_seq tmpseq
= NULL
;
17664 n1
= fold_convert (itype
, n1
);
17665 n2
= fold_convert (itype
, n2
);
17666 tree t
= build_int_cst (itype
, (cond_code
== LT_EXPR
? -1 : 1));
17667 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
17668 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
17669 t
= fold_build2 (MINUS_EXPR
, itype
, t
, n1
);
17670 if (TYPE_UNSIGNED (itype
) && cond_code
== GT_EXPR
)
17671 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
17672 fold_build1 (NEGATE_EXPR
, itype
, t
),
17673 fold_build1 (NEGATE_EXPR
, itype
, step
));
17675 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
17676 tree gs
= fold_convert (uint32_type_node
, t
);
17677 gimplify_expr (&gs
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17678 if (!gimple_seq_empty_p (tmpseq
))
17679 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17682 if (i
== 0 && group_size
)
17684 ws
= fold_convert (uint32_type_node
, group_size
);
17686 gimplify_expr (&ws
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17687 if (!gimple_seq_empty_p (tmpseq
))
17688 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17691 ws
= build_zero_cst (uint32_type_node
);
17693 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__GRIDDIM_
);
17694 OMP_CLAUSE__GRIDDIM__DIMENSION (c
) = i
;
17695 OMP_CLAUSE__GRIDDIM__SIZE (c
) = gs
;
17696 OMP_CLAUSE__GRIDDIM__GROUP (c
) = ws
;
17697 OMP_CLAUSE_CHAIN (c
) = gimple_omp_target_clauses (target
);
17698 gimple_omp_target_set_clauses (target
, c
);
17700 pop_gimplify_context (tgt_bind
);
17705 /* Walker function doing all the work for create_target_kernels. */
17708 grid_gridify_all_targets_stmt (gimple_stmt_iterator
*gsi
,
17709 bool *handled_ops_p
,
17710 struct walk_stmt_info
*incoming
)
17712 *handled_ops_p
= false;
17714 gimple
*stmt
= gsi_stmt (*gsi
);
17715 gomp_target
*target
= dyn_cast
<gomp_target
*> (stmt
);
17718 gbind
*tgt_bind
= (gbind
*) incoming
->info
;
17719 gcc_checking_assert (tgt_bind
);
17720 grid_attempt_target_gridification (target
, gsi
, tgt_bind
);
17723 gbind
*bind
= dyn_cast
<gbind
*> (stmt
);
17726 *handled_ops_p
= true;
17727 struct walk_stmt_info wi
;
17728 memset (&wi
, 0, sizeof (wi
));
17730 walk_gimple_seq_mod (gimple_bind_body_ptr (bind
),
17731 grid_gridify_all_targets_stmt
, NULL
, &wi
);
17736 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17737 have their bodies duplicated, with the new copy being put into a
17738 gimple_omp_grid_body statement. All kernel-related construct within the
17739 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17740 re-structuring is often needed, such as copying pre-bodies before the target
17741 construct so that kernel grid sizes can be computed. */
17744 grid_gridify_all_targets (gimple_seq
*body_p
)
17746 struct walk_stmt_info wi
;
17747 memset (&wi
, 0, sizeof (wi
));
17748 walk_gimple_seq_mod (body_p
, grid_gridify_all_targets_stmt
, NULL
, &wi
);
17752 /* Main entry point. */
17754 static unsigned int
17755 execute_lower_omp (void)
17761 /* This pass always runs, to provide PROP_gimple_lomp.
17762 But often, there is nothing to do. */
17763 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
17764 && flag_openmp_simd
== 0)
17767 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
17768 delete_omp_context
);
17770 body
= gimple_body (current_function_decl
);
17772 if (hsa_gen_requested_p ())
17773 grid_gridify_all_targets (&body
);
17775 scan_omp (&body
, NULL
);
17776 gcc_assert (taskreg_nesting_level
== 0);
17777 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
17778 finish_taskreg_scan (ctx
);
17779 taskreg_contexts
.release ();
17781 if (all_contexts
->root
)
17783 if (task_shared_vars
)
17784 push_gimplify_context ();
17785 lower_omp (&body
, NULL
);
17786 if (task_shared_vars
)
17787 pop_gimplify_context (NULL
);
17792 splay_tree_delete (all_contexts
);
17793 all_contexts
= NULL
;
17795 BITMAP_FREE (task_shared_vars
);
17801 const pass_data pass_data_lower_omp
=
17803 GIMPLE_PASS
, /* type */
17804 "omplower", /* name */
17805 OPTGROUP_NONE
, /* optinfo_flags */
17806 TV_NONE
, /* tv_id */
17807 PROP_gimple_any
, /* properties_required */
17808 PROP_gimple_lomp
, /* properties_provided */
17809 0, /* properties_destroyed */
17810 0, /* todo_flags_start */
17811 0, /* todo_flags_finish */
17814 class pass_lower_omp
: public gimple_opt_pass
17817 pass_lower_omp (gcc::context
*ctxt
)
17818 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
17821 /* opt_pass methods: */
17822 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
17824 }; // class pass_lower_omp
17826 } // anon namespace
17829 make_pass_lower_omp (gcc::context
*ctxt
)
17831 return new pass_lower_omp (ctxt
);
17834 /* The following is a utility to diagnose structured block violations.
17835 It is not part of the "omplower" pass, as that's invoked too late. It
17836 should be invoked by the respective front ends after gimplification. */
17838 static splay_tree all_labels
;
17840 /* Check for mismatched contexts and generate an error if needed. Return
17841 true if an error is detected. */
17844 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
17845 gimple
*branch_ctx
, gimple
*label_ctx
)
17847 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
17848 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
17850 if (label_ctx
== branch_ctx
)
17853 const char* kind
= NULL
;
17858 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
17859 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
17861 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
17862 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
17863 kind
= "Cilk Plus";
17867 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
17868 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
17870 gcc_checking_assert (kind
== NULL
);
17876 gcc_checking_assert (flag_openmp
);
17881 Previously we kept track of the label's entire context in diagnose_sb_[12]
17882 so we could traverse it and issue a correct "exit" or "enter" error
17883 message upon a structured block violation.
17885 We built the context by building a list with tree_cons'ing, but there is
17886 no easy counterpart in gimple tuples. It seems like far too much work
17887 for issuing exit/enter error messages. If someone really misses the
17888 distinct error message... patches welcome.
17892 /* Try to avoid confusing the user by producing and error message
17893 with correct "exit" or "enter" verbiage. We prefer "exit"
17894 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
17895 if (branch_ctx
== NULL
)
17901 if (TREE_VALUE (label_ctx
) == branch_ctx
)
17906 label_ctx
= TREE_CHAIN (label_ctx
);
17911 error ("invalid exit from %s structured block", kind
);
17913 error ("invalid entry to %s structured block", kind
);
17916 /* If it's obvious we have an invalid entry, be specific about the error. */
17917 if (branch_ctx
== NULL
)
17918 error ("invalid entry to %s structured block", kind
);
17921 /* Otherwise, be vague and lazy, but efficient. */
17922 error ("invalid branch to/from %s structured block", kind
);
17925 gsi_replace (gsi_p
, gimple_build_nop (), false);
17929 /* Pass 1: Create a minimal tree of structured blocks, and record
17930 where each label is found. */
17933 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
17934 struct walk_stmt_info
*wi
)
17936 gimple
*context
= (gimple
*) wi
->info
;
17937 gimple
*inner_context
;
17938 gimple
*stmt
= gsi_stmt (*gsi_p
);
17940 *handled_ops_p
= true;
17942 switch (gimple_code (stmt
))
17946 case GIMPLE_OMP_PARALLEL
:
17947 case GIMPLE_OMP_TASK
:
17948 case GIMPLE_OMP_SECTIONS
:
17949 case GIMPLE_OMP_SINGLE
:
17950 case GIMPLE_OMP_SECTION
:
17951 case GIMPLE_OMP_MASTER
:
17952 case GIMPLE_OMP_ORDERED
:
17953 case GIMPLE_OMP_CRITICAL
:
17954 case GIMPLE_OMP_TARGET
:
17955 case GIMPLE_OMP_TEAMS
:
17956 case GIMPLE_OMP_TASKGROUP
:
17957 /* The minimal context here is just the current OMP construct. */
17958 inner_context
= stmt
;
17959 wi
->info
= inner_context
;
17960 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
17961 wi
->info
= context
;
17964 case GIMPLE_OMP_FOR
:
17965 inner_context
= stmt
;
17966 wi
->info
= inner_context
;
17967 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
17969 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
17970 diagnose_sb_1
, NULL
, wi
);
17971 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
17972 wi
->info
= context
;
17976 splay_tree_insert (all_labels
,
17977 (splay_tree_key
) gimple_label_label (
17978 as_a
<glabel
*> (stmt
)),
17979 (splay_tree_value
) context
);
17989 /* Pass 2: Check each branch and see if its context differs from that of
17990 the destination label's context. */
17993 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
17994 struct walk_stmt_info
*wi
)
17996 gimple
*context
= (gimple
*) wi
->info
;
17998 gimple
*stmt
= gsi_stmt (*gsi_p
);
18000 *handled_ops_p
= true;
18002 switch (gimple_code (stmt
))
18006 case GIMPLE_OMP_PARALLEL
:
18007 case GIMPLE_OMP_TASK
:
18008 case GIMPLE_OMP_SECTIONS
:
18009 case GIMPLE_OMP_SINGLE
:
18010 case GIMPLE_OMP_SECTION
:
18011 case GIMPLE_OMP_MASTER
:
18012 case GIMPLE_OMP_ORDERED
:
18013 case GIMPLE_OMP_CRITICAL
:
18014 case GIMPLE_OMP_TARGET
:
18015 case GIMPLE_OMP_TEAMS
:
18016 case GIMPLE_OMP_TASKGROUP
:
18018 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18019 wi
->info
= context
;
18022 case GIMPLE_OMP_FOR
:
18024 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18026 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
18027 diagnose_sb_2
, NULL
, wi
);
18028 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18029 wi
->info
= context
;
18034 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
18035 tree lab
= gimple_cond_true_label (cond_stmt
);
18038 n
= splay_tree_lookup (all_labels
,
18039 (splay_tree_key
) lab
);
18040 diagnose_sb_0 (gsi_p
, context
,
18041 n
? (gimple
*) n
->value
: NULL
);
18043 lab
= gimple_cond_false_label (cond_stmt
);
18046 n
= splay_tree_lookup (all_labels
,
18047 (splay_tree_key
) lab
);
18048 diagnose_sb_0 (gsi_p
, context
,
18049 n
? (gimple
*) n
->value
: NULL
);
18056 tree lab
= gimple_goto_dest (stmt
);
18057 if (TREE_CODE (lab
) != LABEL_DECL
)
18060 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18061 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
18065 case GIMPLE_SWITCH
:
18067 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
18069 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
18071 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
18072 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18073 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
18079 case GIMPLE_RETURN
:
18080 diagnose_sb_0 (gsi_p
, context
, NULL
);
18090 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18093 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
18096 gimple
*last
= last_stmt (bb
);
18097 enum gimple_code code
= gimple_code (last
);
18098 struct omp_region
*cur_region
= *region
;
18099 bool fallthru
= false;
18103 case GIMPLE_OMP_PARALLEL
:
18104 case GIMPLE_OMP_TASK
:
18105 case GIMPLE_OMP_FOR
:
18106 case GIMPLE_OMP_SINGLE
:
18107 case GIMPLE_OMP_TEAMS
:
18108 case GIMPLE_OMP_MASTER
:
18109 case GIMPLE_OMP_TASKGROUP
:
18110 case GIMPLE_OMP_CRITICAL
:
18111 case GIMPLE_OMP_SECTION
:
18112 case GIMPLE_OMP_GRID_BODY
:
18113 cur_region
= new_omp_region (bb
, code
, cur_region
);
18117 case GIMPLE_OMP_ORDERED
:
18118 cur_region
= new_omp_region (bb
, code
, cur_region
);
18120 if (find_omp_clause (gimple_omp_ordered_clauses
18121 (as_a
<gomp_ordered
*> (last
)),
18122 OMP_CLAUSE_DEPEND
))
18123 cur_region
= cur_region
->outer
;
18126 case GIMPLE_OMP_TARGET
:
18127 cur_region
= new_omp_region (bb
, code
, cur_region
);
18129 switch (gimple_omp_target_kind (last
))
18131 case GF_OMP_TARGET_KIND_REGION
:
18132 case GF_OMP_TARGET_KIND_DATA
:
18133 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
18134 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
18135 case GF_OMP_TARGET_KIND_OACC_DATA
:
18136 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
18138 case GF_OMP_TARGET_KIND_UPDATE
:
18139 case GF_OMP_TARGET_KIND_ENTER_DATA
:
18140 case GF_OMP_TARGET_KIND_EXIT_DATA
:
18141 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
18142 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
18143 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
18144 cur_region
= cur_region
->outer
;
18147 gcc_unreachable ();
18151 case GIMPLE_OMP_SECTIONS
:
18152 cur_region
= new_omp_region (bb
, code
, cur_region
);
18156 case GIMPLE_OMP_SECTIONS_SWITCH
:
18160 case GIMPLE_OMP_ATOMIC_LOAD
:
18161 case GIMPLE_OMP_ATOMIC_STORE
:
18165 case GIMPLE_OMP_RETURN
:
18166 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18167 somewhere other than the next block. This will be
18169 cur_region
->exit
= bb
;
18170 if (cur_region
->type
== GIMPLE_OMP_TASK
)
18171 /* Add an edge corresponding to not scheduling the task
18173 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
18174 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
18175 cur_region
= cur_region
->outer
;
18178 case GIMPLE_OMP_CONTINUE
:
18179 cur_region
->cont
= bb
;
18180 switch (cur_region
->type
)
18182 case GIMPLE_OMP_FOR
:
18183 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18184 succs edges as abnormal to prevent splitting
18186 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
18187 /* Make the loopback edge. */
18188 make_edge (bb
, single_succ (cur_region
->entry
),
18191 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18192 corresponds to the case that the body of the loop
18193 is not executed at all. */
18194 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
18195 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
18199 case GIMPLE_OMP_SECTIONS
:
18200 /* Wire up the edges into and out of the nested sections. */
18202 basic_block switch_bb
= single_succ (cur_region
->entry
);
18204 struct omp_region
*i
;
18205 for (i
= cur_region
->inner
; i
; i
= i
->next
)
18207 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
18208 make_edge (switch_bb
, i
->entry
, 0);
18209 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
18212 /* Make the loopback edge to the block with
18213 GIMPLE_OMP_SECTIONS_SWITCH. */
18214 make_edge (bb
, switch_bb
, 0);
18216 /* Make the edge from the switch to exit. */
18217 make_edge (switch_bb
, bb
->next_bb
, 0);
18222 case GIMPLE_OMP_TASK
:
18227 gcc_unreachable ();
18232 gcc_unreachable ();
18235 if (*region
!= cur_region
)
18237 *region
= cur_region
;
18239 *region_idx
= cur_region
->entry
->index
;
18247 static unsigned int
18248 diagnose_omp_structured_block_errors (void)
18250 struct walk_stmt_info wi
;
18251 gimple_seq body
= gimple_body (current_function_decl
);
18253 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
18255 memset (&wi
, 0, sizeof (wi
));
18256 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
18258 memset (&wi
, 0, sizeof (wi
));
18259 wi
.want_locations
= true;
18260 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
18262 gimple_set_body (current_function_decl
, body
);
18264 splay_tree_delete (all_labels
);
18272 const pass_data pass_data_diagnose_omp_blocks
=
18274 GIMPLE_PASS
, /* type */
18275 "*diagnose_omp_blocks", /* name */
18276 OPTGROUP_NONE
, /* optinfo_flags */
18277 TV_NONE
, /* tv_id */
18278 PROP_gimple_any
, /* properties_required */
18279 0, /* properties_provided */
18280 0, /* properties_destroyed */
18281 0, /* todo_flags_start */
18282 0, /* todo_flags_finish */
18285 class pass_diagnose_omp_blocks
: public gimple_opt_pass
18288 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18289 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
18292 /* opt_pass methods: */
18293 virtual bool gate (function
*)
18295 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
18297 virtual unsigned int execute (function
*)
18299 return diagnose_omp_structured_block_errors ();
18302 }; // class pass_diagnose_omp_blocks
18304 } // anon namespace
18307 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18309 return new pass_diagnose_omp_blocks (ctxt
);
18312 /* SIMD clone supporting code. */
18314 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
18315 of arguments to reserve space for. */
18317 static struct cgraph_simd_clone
*
18318 simd_clone_struct_alloc (int nargs
)
18320 struct cgraph_simd_clone
*clone_info
;
18321 size_t len
= (sizeof (struct cgraph_simd_clone
)
18322 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
18323 clone_info
= (struct cgraph_simd_clone
*)
18324 ggc_internal_cleared_alloc (len
);
18328 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
18331 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
18332 struct cgraph_simd_clone
*from
)
18334 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
18335 + ((from
->nargs
- from
->inbranch
)
18336 * sizeof (struct cgraph_simd_clone_arg
))));
18339 /* Return vector of parameter types of function FNDECL. This uses
18340 TYPE_ARG_TYPES if available, otherwise falls back to types of
18341 DECL_ARGUMENTS types. */
18344 simd_clone_vector_of_formal_parm_types (tree fndecl
)
18346 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
18347 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
18348 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
18351 FOR_EACH_VEC_ELT (args
, i
, arg
)
18352 args
[i
] = TREE_TYPE (args
[i
]);
18356 /* Given a simd function in NODE, extract the simd specific
18357 information from the OMP clauses passed in CLAUSES, and return
18358 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
18359 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
18360 otherwise set to FALSE. */
18362 static struct cgraph_simd_clone
*
18363 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
18364 bool *inbranch_specified
)
18366 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
18369 *inbranch_specified
= false;
18371 n
= args
.length ();
18372 if (n
> 0 && args
.last () == void_type_node
)
18375 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
18376 be cloned have a distinctive artificial label in addition to "omp
18380 && lookup_attribute ("cilk simd function",
18381 DECL_ATTRIBUTES (node
->decl
)));
18383 /* Allocate one more than needed just in case this is an in-branch
18384 clone which will require a mask argument. */
18385 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
18386 clone_info
->nargs
= n
;
18387 clone_info
->cilk_elemental
= cilk_clone
;
18394 clauses
= TREE_VALUE (clauses
);
18395 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
18398 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
18400 switch (OMP_CLAUSE_CODE (t
))
18402 case OMP_CLAUSE_INBRANCH
:
18403 clone_info
->inbranch
= 1;
18404 *inbranch_specified
= true;
18406 case OMP_CLAUSE_NOTINBRANCH
:
18407 clone_info
->inbranch
= 0;
18408 *inbranch_specified
= true;
18410 case OMP_CLAUSE_SIMDLEN
:
18411 clone_info
->simdlen
18412 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
18414 case OMP_CLAUSE_LINEAR
:
18416 tree decl
= OMP_CLAUSE_DECL (t
);
18417 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
18418 int argno
= TREE_INT_CST_LOW (decl
);
18419 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
18421 enum cgraph_simd_clone_arg_type arg_type
;
18422 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
18423 switch (OMP_CLAUSE_LINEAR_KIND (t
))
18425 case OMP_CLAUSE_LINEAR_REF
:
18427 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
18429 case OMP_CLAUSE_LINEAR_UVAL
:
18431 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
18433 case OMP_CLAUSE_LINEAR_VAL
:
18434 case OMP_CLAUSE_LINEAR_DEFAULT
:
18436 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
18439 gcc_unreachable ();
18442 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
18443 clone_info
->args
[argno
].arg_type
= arg_type
;
18444 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
18445 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
18446 && clone_info
->args
[argno
].linear_step
< n
);
18450 if (POINTER_TYPE_P (args
[argno
]))
18451 step
= fold_convert (ssizetype
, step
);
18452 if (!tree_fits_shwi_p (step
))
18454 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
18455 "ignoring large linear step");
18459 else if (integer_zerop (step
))
18461 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
18462 "ignoring zero linear step");
18468 enum cgraph_simd_clone_arg_type arg_type
;
18469 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
18470 switch (OMP_CLAUSE_LINEAR_KIND (t
))
18472 case OMP_CLAUSE_LINEAR_REF
:
18474 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
18476 case OMP_CLAUSE_LINEAR_UVAL
:
18478 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
18480 case OMP_CLAUSE_LINEAR_VAL
:
18481 case OMP_CLAUSE_LINEAR_DEFAULT
:
18483 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
18486 gcc_unreachable ();
18489 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
18490 clone_info
->args
[argno
].arg_type
= arg_type
;
18491 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
18496 case OMP_CLAUSE_UNIFORM
:
18498 tree decl
= OMP_CLAUSE_DECL (t
);
18499 int argno
= tree_to_uhwi (decl
);
18500 clone_info
->args
[argno
].arg_type
18501 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
18504 case OMP_CLAUSE_ALIGNED
:
18506 tree decl
= OMP_CLAUSE_DECL (t
);
18507 int argno
= tree_to_uhwi (decl
);
18508 clone_info
->args
[argno
].alignment
18509 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
18520 /* Given a SIMD clone in NODE, calculate the characteristic data
18521 type and return the coresponding type. The characteristic data
18522 type is computed as described in the Intel Vector ABI. */
18525 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
18526 struct cgraph_simd_clone
*clone_info
)
18528 tree type
= integer_type_node
;
18529 tree fndecl
= node
->decl
;
18531 /* a) For non-void function, the characteristic data type is the
18533 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
18534 type
= TREE_TYPE (TREE_TYPE (fndecl
));
18536 /* b) If the function has any non-uniform, non-linear parameters,
18537 then the characteristic data type is the type of the first
18541 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
18542 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
18543 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
18551 /* c) If the characteristic data type determined by a) or b) above
18552 is struct, union, or class type which is pass-by-value (except
18553 for the type that maps to the built-in complex data type), the
18554 characteristic data type is int. */
18555 if (RECORD_OR_UNION_TYPE_P (type
)
18556 && !aggregate_value_p (type
, NULL
)
18557 && TREE_CODE (type
) != COMPLEX_TYPE
)
18558 return integer_type_node
;
18560 /* d) If none of the above three classes is applicable, the
18561 characteristic data type is int. */
18565 /* e) For Intel Xeon Phi native and offload compilation, if the
18566 resulting characteristic data type is 8-bit or 16-bit integer
18567 data type, the characteristic data type is int. */
18568 /* Well, we don't handle Xeon Phi yet. */
18572 simd_clone_mangle (struct cgraph_node
*node
,
18573 struct cgraph_simd_clone
*clone_info
)
18575 char vecsize_mangle
= clone_info
->vecsize_mangle
;
18576 char mask
= clone_info
->inbranch
? 'M' : 'N';
18577 unsigned int simdlen
= clone_info
->simdlen
;
18581 gcc_assert (vecsize_mangle
&& simdlen
);
18583 pp_string (&pp
, "_ZGV");
18584 pp_character (&pp
, vecsize_mangle
);
18585 pp_character (&pp
, mask
);
18586 pp_decimal_int (&pp
, simdlen
);
18588 for (n
= 0; n
< clone_info
->nargs
; ++n
)
18590 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
18592 switch (arg
.arg_type
)
18594 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
18595 pp_character (&pp
, 'u');
18597 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
18598 pp_character (&pp
, 'l');
18599 goto mangle_linear
;
18600 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
18601 pp_character (&pp
, 'R');
18602 goto mangle_linear
;
18603 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
18604 pp_character (&pp
, 'L');
18605 goto mangle_linear
;
18606 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
18607 pp_character (&pp
, 'U');
18608 goto mangle_linear
;
18610 gcc_assert (arg
.linear_step
!= 0);
18611 if (arg
.linear_step
> 1)
18612 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18613 else if (arg
.linear_step
< 0)
18615 pp_character (&pp
, 'n');
18616 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
18620 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
18621 pp_string (&pp
, "ls");
18622 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18624 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
18625 pp_string (&pp
, "Rs");
18626 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18628 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
18629 pp_string (&pp
, "Ls");
18630 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18632 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
18633 pp_string (&pp
, "Us");
18634 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18637 pp_character (&pp
, 'v');
18641 pp_character (&pp
, 'a');
18642 pp_decimal_int (&pp
, arg
.alignment
);
18646 pp_underscore (&pp
);
18647 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
18650 pp_string (&pp
, str
);
18651 str
= pp_formatted_text (&pp
);
18653 /* If there already is a SIMD clone with the same mangled name, don't
18654 add another one. This can happen e.g. for
18655 #pragma omp declare simd
18656 #pragma omp declare simd simdlen(8)
18657 int foo (int, int);
18658 if the simdlen is assumed to be 8 for the first one, etc. */
18659 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
18660 clone
= clone
->simdclone
->next_clone
)
18661 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
18665 return get_identifier (str
);
18668 /* Create a simd clone of OLD_NODE and return it. */
18670 static struct cgraph_node
*
18671 simd_clone_create (struct cgraph_node
*old_node
)
18673 struct cgraph_node
*new_node
;
18674 if (old_node
->definition
)
18676 if (!old_node
->has_gimple_body_p ())
18678 old_node
->get_body ();
18679 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
18685 tree old_decl
= old_node
->decl
;
18686 tree new_decl
= copy_node (old_node
->decl
);
18687 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
18688 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
18689 SET_DECL_RTL (new_decl
, NULL
);
18690 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
18691 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
18692 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
18693 if (old_node
->in_other_partition
)
18694 new_node
->in_other_partition
= 1;
18695 symtab
->call_cgraph_insertion_hooks (new_node
);
18697 if (new_node
== NULL
)
18700 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
18702 /* The function cgraph_function_versioning () will force the new
18703 symbol local. Undo this, and inherit external visability from
18705 new_node
->local
.local
= old_node
->local
.local
;
18706 new_node
->externally_visible
= old_node
->externally_visible
;
18711 /* Adjust the return type of the given function to its appropriate
18712 vector counterpart. Returns a simd array to be used throughout the
18713 function as a return value. */
18716 simd_clone_adjust_return_type (struct cgraph_node
*node
)
18718 tree fndecl
= node
->decl
;
18719 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
18720 unsigned int veclen
;
18723 /* Adjust the function return type. */
18724 if (orig_rettype
== void_type_node
)
18726 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
18727 t
= TREE_TYPE (TREE_TYPE (fndecl
));
18728 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
18729 veclen
= node
->simdclone
->vecsize_int
;
18731 veclen
= node
->simdclone
->vecsize_float
;
18732 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
18733 if (veclen
> node
->simdclone
->simdlen
)
18734 veclen
= node
->simdclone
->simdlen
;
18735 if (POINTER_TYPE_P (t
))
18736 t
= pointer_sized_int_node
;
18737 if (veclen
== node
->simdclone
->simdlen
)
18738 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
18741 t
= build_vector_type (t
, veclen
);
18742 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
18744 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
18745 if (!node
->definition
)
18748 t
= DECL_RESULT (fndecl
);
18749 /* Adjust the DECL_RESULT. */
18750 gcc_assert (TREE_TYPE (t
) != void_type_node
);
18751 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
18754 tree atype
= build_array_type_nelts (orig_rettype
,
18755 node
->simdclone
->simdlen
);
18756 if (veclen
!= node
->simdclone
->simdlen
)
18757 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
18759 /* Set up a SIMD array to use as the return value. */
18760 tree retval
= create_tmp_var_raw (atype
, "retval");
18761 gimple_add_tmp_var (retval
);
18765 /* Each vector argument has a corresponding array to be used locally
18766 as part of the eventual loop. Create such temporary array and
18769 PREFIX is the prefix to be used for the temporary.
18771 TYPE is the inner element type.
18773 SIMDLEN is the number of elements. */
18776 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
18778 tree atype
= build_array_type_nelts (type
, simdlen
);
18779 tree avar
= create_tmp_var_raw (atype
, prefix
);
18780 gimple_add_tmp_var (avar
);
18784 /* Modify the function argument types to their corresponding vector
18785 counterparts if appropriate. Also, create one array for each simd
18786 argument to be used locally when using the function arguments as
18789 NODE is the function whose arguments are to be adjusted.
18791 Returns an adjustment vector that will be filled describing how the
18792 argument types will be adjusted. */
18794 static ipa_parm_adjustment_vec
18795 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
18798 ipa_parm_adjustment_vec adjustments
;
18800 if (node
->definition
)
18801 args
= ipa_get_vector_of_formal_parms (node
->decl
);
18803 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
18804 adjustments
.create (args
.length ());
18805 unsigned i
, j
, veclen
;
18806 struct ipa_parm_adjustment adj
;
18807 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
18809 memset (&adj
, 0, sizeof (adj
));
18810 tree parm
= args
[i
];
18811 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
18812 adj
.base_index
= i
;
18815 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
18816 node
->simdclone
->args
[i
].orig_type
= parm_type
;
18818 switch (node
->simdclone
->args
[i
].arg_type
)
18821 /* No adjustment necessary for scalar arguments. */
18822 adj
.op
= IPA_PARM_OP_COPY
;
18824 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
18825 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
18826 if (node
->definition
)
18827 node
->simdclone
->args
[i
].simd_array
18828 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
18829 TREE_TYPE (parm_type
),
18830 node
->simdclone
->simdlen
);
18831 adj
.op
= IPA_PARM_OP_COPY
;
18833 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
18834 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
18835 case SIMD_CLONE_ARG_TYPE_VECTOR
:
18836 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
18837 veclen
= node
->simdclone
->vecsize_int
;
18839 veclen
= node
->simdclone
->vecsize_float
;
18840 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
18841 if (veclen
> node
->simdclone
->simdlen
)
18842 veclen
= node
->simdclone
->simdlen
;
18843 adj
.arg_prefix
= "simd";
18844 if (POINTER_TYPE_P (parm_type
))
18845 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
18847 adj
.type
= build_vector_type (parm_type
, veclen
);
18848 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
18849 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
18851 adjustments
.safe_push (adj
);
18854 memset (&adj
, 0, sizeof (adj
));
18855 adj
.op
= IPA_PARM_OP_NEW
;
18856 adj
.arg_prefix
= "simd";
18857 adj
.base_index
= i
;
18858 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
18862 if (node
->definition
)
18863 node
->simdclone
->args
[i
].simd_array
18864 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
18865 parm_type
, node
->simdclone
->simdlen
);
18867 adjustments
.safe_push (adj
);
18870 if (node
->simdclone
->inbranch
)
18873 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
18876 memset (&adj
, 0, sizeof (adj
));
18877 adj
.op
= IPA_PARM_OP_NEW
;
18878 adj
.arg_prefix
= "mask";
18880 adj
.base_index
= i
;
18881 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
18882 veclen
= node
->simdclone
->vecsize_int
;
18884 veclen
= node
->simdclone
->vecsize_float
;
18885 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
18886 if (veclen
> node
->simdclone
->simdlen
)
18887 veclen
= node
->simdclone
->simdlen
;
18888 if (POINTER_TYPE_P (base_type
))
18889 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
18891 adj
.type
= build_vector_type (base_type
, veclen
);
18892 adjustments
.safe_push (adj
);
18894 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
18895 adjustments
.safe_push (adj
);
18897 /* We have previously allocated one extra entry for the mask. Use
18899 struct cgraph_simd_clone
*sc
= node
->simdclone
;
18901 if (node
->definition
)
18903 sc
->args
[i
].orig_arg
18904 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
18905 sc
->args
[i
].simd_array
18906 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
18908 sc
->args
[i
].orig_type
= base_type
;
18909 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
18912 if (node
->definition
)
18913 ipa_modify_formal_parameters (node
->decl
, adjustments
);
18916 tree new_arg_types
= NULL_TREE
, new_reversed
;
18917 bool last_parm_void
= false;
18918 if (args
.length () > 0 && args
.last () == void_type_node
)
18919 last_parm_void
= true;
18921 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
18922 j
= adjustments
.length ();
18923 for (i
= 0; i
< j
; i
++)
18925 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
18927 if (adj
->op
== IPA_PARM_OP_COPY
)
18928 ptype
= args
[adj
->base_index
];
18931 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
18933 new_reversed
= nreverse (new_arg_types
);
18934 if (last_parm_void
)
18937 TREE_CHAIN (new_arg_types
) = void_list_node
;
18939 new_reversed
= void_list_node
;
18942 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
18943 TYPE_ARG_TYPES (new_type
) = new_reversed
;
18944 TREE_TYPE (node
->decl
) = new_type
;
18946 adjustments
.release ();
18949 return adjustments
;
18952 /* Initialize and copy the function arguments in NODE to their
18953 corresponding local simd arrays. Returns a fresh gimple_seq with
18954 the instruction sequence generated. */
18957 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
18958 ipa_parm_adjustment_vec adjustments
)
18960 gimple_seq seq
= NULL
;
18961 unsigned i
= 0, j
= 0, k
;
18963 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
18965 arg
= DECL_CHAIN (arg
), i
++, j
++)
18967 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
18968 || POINTER_TYPE_P (TREE_TYPE (arg
)))
18971 node
->simdclone
->args
[i
].vector_arg
= arg
;
18973 tree array
= node
->simdclone
->args
[i
].simd_array
;
18974 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
18976 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
18977 tree ptr
= build_fold_addr_expr (array
);
18978 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
18979 build_int_cst (ptype
, 0));
18980 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
18981 gimplify_and_add (t
, &seq
);
18985 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
18986 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
18987 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
18989 tree ptr
= build_fold_addr_expr (array
);
18993 arg
= DECL_CHAIN (arg
);
18997 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
18998 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
18999 build_int_cst (ptype
, k
* elemsize
));
19000 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
19001 gimplify_and_add (t
, &seq
);
19008 /* Callback info for ipa_simd_modify_stmt_ops below. */
19010 struct modify_stmt_info
{
19011 ipa_parm_adjustment_vec adjustments
;
19013 /* True if the parent statement was modified by
19014 ipa_simd_modify_stmt_ops. */
19018 /* Callback for walk_gimple_op.
19020 Adjust operands from a given statement as specified in the
19021 adjustments vector in the callback data. */
19024 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
19026 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
19027 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
19028 tree
*orig_tp
= tp
;
19029 if (TREE_CODE (*tp
) == ADDR_EXPR
)
19030 tp
= &TREE_OPERAND (*tp
, 0);
19031 struct ipa_parm_adjustment
*cand
= NULL
;
19032 if (TREE_CODE (*tp
) == PARM_DECL
)
19033 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
19037 *walk_subtrees
= 0;
19040 tree repl
= NULL_TREE
;
19042 repl
= unshare_expr (cand
->new_decl
);
19047 *walk_subtrees
= 0;
19048 bool modified
= info
->modified
;
19049 info
->modified
= false;
19050 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
19051 if (!info
->modified
)
19053 info
->modified
= modified
;
19056 info
->modified
= modified
;
19065 repl
= build_fold_addr_expr (repl
);
19067 if (is_gimple_debug (info
->stmt
))
19069 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
19070 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
19071 DECL_ARTIFICIAL (vexpr
) = 1;
19072 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
19073 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
19078 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
19079 repl
= gimple_assign_lhs (stmt
);
19081 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
19082 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
19085 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
19087 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
19093 info
->modified
= true;
19097 /* Traverse the function body and perform all modifications as
19098 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
19099 modified such that the replacement/reduction value will now be an
19100 offset into the corresponding simd_array.
19102 This function will replace all function argument uses with their
19103 corresponding simd array elements, and ajust the return values
19107 ipa_simd_modify_function_body (struct cgraph_node
*node
,
19108 ipa_parm_adjustment_vec adjustments
,
19109 tree retval_array
, tree iter
)
19112 unsigned int i
, j
, l
;
19114 /* Re-use the adjustments array, but this time use it to replace
19115 every function argument use to an offset into the corresponding
19117 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
19119 if (!node
->simdclone
->args
[i
].vector_arg
)
19122 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
19123 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
19124 adjustments
[j
].new_decl
19125 = build4 (ARRAY_REF
,
19127 node
->simdclone
->args
[i
].simd_array
,
19129 NULL_TREE
, NULL_TREE
);
19130 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
19131 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
19132 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
19135 l
= adjustments
.length ();
19136 for (i
= 1; i
< num_ssa_names
; i
++)
19138 tree name
= ssa_name (i
);
19140 && SSA_NAME_VAR (name
)
19141 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
19143 for (j
= 0; j
< l
; j
++)
19144 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
19145 && adjustments
[j
].new_decl
)
19148 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
19151 = copy_var_decl (adjustments
[j
].base
,
19152 DECL_NAME (adjustments
[j
].base
),
19153 TREE_TYPE (adjustments
[j
].base
));
19154 adjustments
[j
].new_ssa_base
= base_var
;
19157 base_var
= adjustments
[j
].new_ssa_base
;
19158 if (SSA_NAME_IS_DEFAULT_DEF (name
))
19160 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19161 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
19162 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
19163 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
19164 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
19165 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
19166 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
19167 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
19170 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
19175 struct modify_stmt_info info
;
19176 info
.adjustments
= adjustments
;
19178 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
19180 gimple_stmt_iterator gsi
;
19182 gsi
= gsi_start_bb (bb
);
19183 while (!gsi_end_p (gsi
))
19185 gimple
*stmt
= gsi_stmt (gsi
);
19187 struct walk_stmt_info wi
;
19189 memset (&wi
, 0, sizeof (wi
));
19190 info
.modified
= false;
19192 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
19194 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
19196 tree retval
= gimple_return_retval (return_stmt
);
19199 gsi_remove (&gsi
, true);
19203 /* Replace `return foo' with `retval_array[iter] = foo'. */
19204 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
19205 retval_array
, iter
, NULL
, NULL
);
19206 stmt
= gimple_build_assign (ref
, retval
);
19207 gsi_replace (&gsi
, stmt
, true);
19208 info
.modified
= true;
19213 update_stmt (stmt
);
19214 if (maybe_clean_eh_stmt (stmt
))
19215 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
19222 /* Helper function of simd_clone_adjust, return linear step addend
19223 of Ith argument. */
19226 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
19227 tree addtype
, basic_block entry_bb
)
19229 tree ptype
= NULL_TREE
;
19230 switch (node
->simdclone
->args
[i
].arg_type
)
19232 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
19233 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
19234 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
19235 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
19236 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
19237 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
19238 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
19239 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
19241 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
19242 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
19243 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
19246 gcc_unreachable ();
19249 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
19250 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
19251 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
19252 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
19255 if (is_gimple_reg (arg
))
19256 ret
= get_or_create_ssa_default_def (cfun
, arg
);
19259 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
19260 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19261 ret
= gimple_assign_lhs (g
);
19263 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
19265 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
19266 build_simple_mem_ref (ret
));
19267 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19268 ret
= gimple_assign_lhs (g
);
19270 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
19272 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
19273 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19274 ret
= gimple_assign_lhs (g
);
19276 if (POINTER_TYPE_P (ptype
))
19278 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
19279 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
19281 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
19282 ret
, fold_convert (addtype
, size
));
19283 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19284 ret
= gimple_assign_lhs (g
);
19290 /* Adjust the argument types in NODE to their appropriate vector
19294 simd_clone_adjust (struct cgraph_node
*node
)
19296 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
19298 targetm
.simd_clone
.adjust (node
);
19300 tree retval
= simd_clone_adjust_return_type (node
);
19301 ipa_parm_adjustment_vec adjustments
19302 = simd_clone_adjust_argument_types (node
);
19304 push_gimplify_context ();
19306 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
19308 /* Adjust all uses of vector arguments accordingly. Adjust all
19309 return values accordingly. */
19310 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
19311 tree iter1
= make_ssa_name (iter
);
19312 tree iter2
= make_ssa_name (iter
);
19313 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
19315 /* Initialize the iteration variable. */
19316 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19317 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
19318 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
19319 /* Insert the SIMD array and iv initialization at function
19321 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
19323 pop_gimplify_context (NULL
);
19325 /* Create a new BB right before the original exit BB, to hold the
19326 iteration increment and the condition/branch. */
19327 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
19328 basic_block incr_bb
= create_empty_bb (orig_exit
);
19329 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
19330 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
19331 flag. Set it now to be a FALLTHRU_EDGE. */
19332 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
19333 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
19334 for (unsigned i
= 0;
19335 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
19337 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
19338 redirect_edge_succ (e
, incr_bb
);
19340 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
19341 e
->probability
= REG_BR_PROB_BASE
;
19342 gsi
= gsi_last_bb (incr_bb
);
19343 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
19344 build_int_cst (unsigned_type_node
, 1));
19345 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19347 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
19348 struct loop
*loop
= alloc_loop ();
19349 cfun
->has_force_vectorize_loops
= true;
19350 loop
->safelen
= node
->simdclone
->simdlen
;
19351 loop
->force_vectorize
= true;
19352 loop
->header
= body_bb
;
19354 /* Branch around the body if the mask applies. */
19355 if (node
->simdclone
->inbranch
)
19357 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
19359 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
19360 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
19361 tree aref
= build4 (ARRAY_REF
,
19362 TREE_TYPE (TREE_TYPE (mask_array
)),
19365 g
= gimple_build_assign (mask
, aref
);
19366 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19367 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
19368 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
19370 aref
= build1 (VIEW_CONVERT_EXPR
,
19371 build_nonstandard_integer_type (bitsize
, 0), mask
);
19372 mask
= make_ssa_name (TREE_TYPE (aref
));
19373 g
= gimple_build_assign (mask
, aref
);
19374 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19377 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
19379 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19380 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
19381 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
19384 /* Generate the condition. */
19385 g
= gimple_build_cond (LT_EXPR
,
19387 build_int_cst (unsigned_type_node
,
19388 node
->simdclone
->simdlen
),
19390 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19391 e
= split_block (incr_bb
, gsi_stmt (gsi
));
19392 basic_block latch_bb
= e
->dest
;
19393 basic_block new_exit_bb
;
19394 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
19395 loop
->latch
= latch_bb
;
19397 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
19399 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
19400 /* The successor of incr_bb is already pointing to latch_bb; just
19402 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
19403 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
19405 gphi
*phi
= create_phi_node (iter1
, body_bb
);
19406 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
19407 edge latch_edge
= single_succ_edge (latch_bb
);
19408 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
19410 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
19412 /* Generate the new return. */
19413 gsi
= gsi_last_bb (new_exit_bb
);
19415 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
19416 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
19417 retval
= TREE_OPERAND (retval
, 0);
19420 retval
= build1 (VIEW_CONVERT_EXPR
,
19421 TREE_TYPE (TREE_TYPE (node
->decl
)),
19423 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
19424 false, GSI_CONTINUE_LINKING
);
19426 g
= gimple_build_return (retval
);
19427 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19429 /* Handle aligned clauses by replacing default defs of the aligned
19430 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
19431 lhs. Handle linear by adding PHIs. */
19432 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
19433 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
19434 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
19435 || !is_gimple_reg_type
19436 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
19438 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19439 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
19440 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
19443 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
19444 gimple_add_tmp_var (iter1
);
19446 gsi
= gsi_after_labels (entry_bb
);
19447 g
= gimple_build_assign (iter1
, orig_arg
);
19448 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19449 gsi
= gsi_after_labels (body_bb
);
19450 g
= gimple_build_assign (orig_arg
, iter1
);
19451 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19453 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
19454 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
19455 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
19457 && TREE_ADDRESSABLE
19458 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
19460 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19461 tree def
= ssa_default_def (cfun
, orig_arg
);
19462 if (def
&& !has_zero_uses (def
))
19464 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
19465 gimple_add_tmp_var (iter1
);
19466 gsi
= gsi_after_labels (entry_bb
);
19467 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
19468 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19469 gsi
= gsi_after_labels (body_bb
);
19470 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
19471 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19474 else if (node
->simdclone
->args
[i
].alignment
19475 && node
->simdclone
->args
[i
].arg_type
19476 == SIMD_CLONE_ARG_TYPE_UNIFORM
19477 && (node
->simdclone
->args
[i
].alignment
19478 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
19479 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
19482 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
19483 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19484 tree def
= ssa_default_def (cfun
, orig_arg
);
19485 if (def
&& !has_zero_uses (def
))
19487 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
19488 gimple_seq seq
= NULL
;
19489 bool need_cvt
= false;
19491 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
19493 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
19496 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
19497 gimple_call_set_lhs (g
, t
);
19498 gimple_seq_add_stmt_without_update (&seq
, g
);
19501 t
= make_ssa_name (orig_arg
);
19502 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
19503 gimple_seq_add_stmt_without_update (&seq
, g
);
19505 gsi_insert_seq_on_edge_immediate
19506 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
19508 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19509 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
19511 node
->create_edge (cgraph_node::get_create (fn
),
19512 call
, entry_bb
->count
, freq
);
19514 imm_use_iterator iter
;
19515 use_operand_p use_p
;
19517 tree repl
= gimple_get_lhs (g
);
19518 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
19519 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
19522 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
19523 SET_USE (use_p
, repl
);
19526 else if ((node
->simdclone
->args
[i
].arg_type
19527 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
19528 || (node
->simdclone
->args
[i
].arg_type
19529 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
19530 || (node
->simdclone
->args
[i
].arg_type
19531 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
19532 || (node
->simdclone
->args
[i
].arg_type
19533 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
19535 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19536 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
19537 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
19538 tree def
= NULL_TREE
;
19539 if (TREE_ADDRESSABLE (orig_arg
))
19541 def
= make_ssa_name (TREE_TYPE (orig_arg
));
19542 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
19543 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
19544 gsi
= gsi_after_labels (entry_bb
);
19545 g
= gimple_build_assign (def
, orig_arg
);
19546 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19550 def
= ssa_default_def (cfun
, orig_arg
);
19551 if (!def
|| has_zero_uses (def
))
19555 iter1
= make_ssa_name (orig_arg
);
19556 iter2
= make_ssa_name (orig_arg
);
19561 phi
= create_phi_node (iter1
, body_bb
);
19562 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
19563 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
19564 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
19565 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
19566 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
19567 ? TREE_TYPE (orig_arg
) : sizetype
;
19568 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
19570 gsi
= gsi_last_bb (incr_bb
);
19571 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
19572 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19574 imm_use_iterator iter
;
19575 use_operand_p use_p
;
19577 if (TREE_ADDRESSABLE (orig_arg
))
19579 gsi
= gsi_after_labels (body_bb
);
19580 g
= gimple_build_assign (orig_arg
, iter1
);
19581 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19584 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
19585 if (use_stmt
== phi
)
19588 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
19589 SET_USE (use_p
, iter1
);
19592 else if (node
->simdclone
->args
[i
].arg_type
19593 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
19594 || (node
->simdclone
->args
[i
].arg_type
19595 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
19597 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19598 tree def
= ssa_default_def (cfun
, orig_arg
);
19599 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
19600 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
19601 if (def
&& !has_zero_uses (def
))
19603 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
19604 iter1
= make_ssa_name (orig_arg
);
19605 iter2
= make_ssa_name (orig_arg
);
19606 tree iter3
= make_ssa_name (rtype
);
19607 tree iter4
= make_ssa_name (rtype
);
19608 tree iter5
= make_ssa_name (rtype
);
19609 gsi
= gsi_after_labels (entry_bb
);
19611 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
19612 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
19614 tree array
= node
->simdclone
->args
[i
].simd_array
;
19615 TREE_ADDRESSABLE (array
) = 1;
19616 tree ptr
= build_fold_addr_expr (array
);
19617 phi
= create_phi_node (iter1
, body_bb
);
19618 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
19619 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
19620 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
19621 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
19622 gsi
= gsi_last_bb (incr_bb
);
19623 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19625 phi
= create_phi_node (iter4
, body_bb
);
19626 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
19627 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
19628 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
19629 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
19630 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
19631 ? TREE_TYPE (iter3
) : sizetype
;
19632 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
19634 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
19635 gsi
= gsi_last_bb (incr_bb
);
19636 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19638 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
19639 gsi
= gsi_after_labels (body_bb
);
19640 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19642 imm_use_iterator iter
;
19643 use_operand_p use_p
;
19645 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
19646 if (use_stmt
== load
)
19649 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
19650 SET_USE (use_p
, iter1
);
19652 if (!TYPE_READONLY (rtype
))
19654 tree v
= make_ssa_name (rtype
);
19655 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
19656 size_zero_node
, NULL_TREE
,
19658 gsi
= gsi_after_labels (new_exit_bb
);
19659 g
= gimple_build_assign (v
, aref
);
19660 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19661 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
19662 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19667 calculate_dominance_info (CDI_DOMINATORS
);
19668 add_loop (loop
, loop
->header
->loop_father
);
19669 update_ssa (TODO_update_ssa
);
19674 /* If the function in NODE is tagged as an elemental SIMD function,
19675 create the appropriate SIMD clones. */
19678 expand_simd_clones (struct cgraph_node
*node
)
19680 tree attr
= lookup_attribute ("omp declare simd",
19681 DECL_ATTRIBUTES (node
->decl
));
19682 if (attr
== NULL_TREE
19683 || node
->global
.inlined_to
19684 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
19688 #pragma omp declare simd
19690 in C, there we don't know the argument types at all. */
19691 if (!node
->definition
19692 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
19695 /* Call this before creating clone_info, as it might ggc_collect. */
19696 if (node
->definition
&& node
->has_gimple_body_p ())
19701 /* Start with parsing the "omp declare simd" attribute(s). */
19702 bool inbranch_clause_specified
;
19703 struct cgraph_simd_clone
*clone_info
19704 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
19705 &inbranch_clause_specified
);
19706 if (clone_info
== NULL
)
19709 int orig_simdlen
= clone_info
->simdlen
;
19710 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
19711 /* The target can return 0 (no simd clones should be created),
19712 1 (just one ISA of simd clones should be created) or higher
19713 count of ISA variants. In that case, clone_info is initialized
19714 for the first ISA variant. */
19716 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
19721 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
19722 also create one inbranch and one !inbranch clone of it. */
19723 for (int i
= 0; i
< count
* 2; i
++)
19725 struct cgraph_simd_clone
*clone
= clone_info
;
19726 if (inbranch_clause_specified
&& (i
& 1) != 0)
19731 clone
= simd_clone_struct_alloc (clone_info
->nargs
19733 simd_clone_struct_copy (clone
, clone_info
);
19734 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
19735 and simd_clone_adjust_argument_types did to the first
19737 clone
->nargs
-= clone_info
->inbranch
;
19738 clone
->simdlen
= orig_simdlen
;
19739 /* And call the target hook again to get the right ISA. */
19740 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
19744 clone
->inbranch
= 1;
19747 /* simd_clone_mangle might fail if such a clone has been created
19749 tree id
= simd_clone_mangle (node
, clone
);
19750 if (id
== NULL_TREE
)
19753 /* Only when we are sure we want to create the clone actually
19754 clone the function (or definitions) or create another
19755 extern FUNCTION_DECL (for prototypes without definitions). */
19756 struct cgraph_node
*n
= simd_clone_create (node
);
19760 n
->simdclone
= clone
;
19761 clone
->origin
= node
;
19762 clone
->next_clone
= NULL
;
19763 if (node
->simd_clones
== NULL
)
19765 clone
->prev_clone
= n
;
19766 node
->simd_clones
= n
;
19770 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
19771 clone
->prev_clone
->simdclone
->next_clone
= n
;
19772 node
->simd_clones
->simdclone
->prev_clone
= n
;
19774 symtab
->change_decl_assembler_name (n
->decl
, id
);
19775 /* And finally adjust the return type, parameters and for
19776 definitions also function body. */
19777 if (node
->definition
)
19778 simd_clone_adjust (n
);
19781 simd_clone_adjust_return_type (n
);
19782 simd_clone_adjust_argument_types (n
);
19786 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
19789 /* Entry point for IPA simd clone creation pass. */
19791 static unsigned int
19792 ipa_omp_simd_clone (void)
19794 struct cgraph_node
*node
;
19795 FOR_EACH_FUNCTION (node
)
19796 expand_simd_clones (node
);
19802 const pass_data pass_data_omp_simd_clone
=
19804 SIMPLE_IPA_PASS
, /* type */
19805 "simdclone", /* name */
19806 OPTGROUP_NONE
, /* optinfo_flags */
19807 TV_NONE
, /* tv_id */
19808 ( PROP_ssa
| PROP_cfg
), /* properties_required */
19809 0, /* properties_provided */
19810 0, /* properties_destroyed */
19811 0, /* todo_flags_start */
19812 0, /* todo_flags_finish */
19815 class pass_omp_simd_clone
: public simple_ipa_opt_pass
19818 pass_omp_simd_clone(gcc::context
*ctxt
)
19819 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
19822 /* opt_pass methods: */
19823 virtual bool gate (function
*);
19824 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
19828 pass_omp_simd_clone::gate (function
*)
19830 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
19833 } // anon namespace
19835 simple_ipa_opt_pass
*
19836 make_pass_omp_simd_clone (gcc::context
*ctxt
)
19838 return new pass_omp_simd_clone (ctxt
);
19841 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
19842 adds their addresses and sizes to constructor-vector V_CTOR. */
19844 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
19845 vec
<constructor_elt
, va_gc
> *v_ctor
)
19847 unsigned len
= vec_safe_length (v_decls
);
19848 for (unsigned i
= 0; i
< len
; i
++)
19850 tree it
= (*v_decls
)[i
];
19851 bool is_var
= TREE_CODE (it
) == VAR_DECL
;
19854 #ifdef ACCEL_COMPILER
19855 && DECL_HAS_VALUE_EXPR_P (it
)
19857 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
19859 tree size
= NULL_TREE
;
19861 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
19865 addr
= build_fold_addr_expr (it
);
19868 #ifdef ACCEL_COMPILER
19869 /* For "omp declare target link" vars add address of the pointer to
19870 the target table, instead of address of the var. */
19871 tree value_expr
= DECL_VALUE_EXPR (it
);
19872 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
19873 varpool_node::finalize_decl (link_ptr_decl
);
19874 addr
= build_fold_addr_expr (link_ptr_decl
);
19876 addr
= build_fold_addr_expr (it
);
19879 /* Most significant bit of the size marks "omp declare target link"
19880 vars in host and target tables. */
19881 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
19882 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
19883 * BITS_PER_UNIT
- 1);
19884 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
19887 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
19889 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
19893 /* Create new symbols containing (address, size) pairs for global variables,
19894 marked with "omp declare target" attribute, as well as addresses for the
19895 functions, which are outlined offloading regions. */
19897 omp_finish_file (void)
19899 unsigned num_funcs
= vec_safe_length (offload_funcs
);
19900 unsigned num_vars
= vec_safe_length (offload_vars
);
19902 if (num_funcs
== 0 && num_vars
== 0)
19905 if (targetm_common
.have_named_sections
)
19907 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
19908 vec_alloc (v_f
, num_funcs
);
19909 vec_alloc (v_v
, num_vars
* 2);
19911 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
19912 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
19914 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
19916 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
19918 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
19919 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
19920 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
19921 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
19922 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
19923 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
19924 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
19925 get_identifier (".offload_func_table"),
19927 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
19928 get_identifier (".offload_var_table"),
19930 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
19931 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
19932 otherwise a joint table in a binary will contain padding between
19933 tables from multiple object files. */
19934 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
19935 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
19936 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
19937 DECL_INITIAL (funcs_decl
) = ctor_f
;
19938 DECL_INITIAL (vars_decl
) = ctor_v
;
19939 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
19940 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
19942 varpool_node::finalize_decl (vars_decl
);
19943 varpool_node::finalize_decl (funcs_decl
);
19947 for (unsigned i
= 0; i
< num_funcs
; i
++)
19949 tree it
= (*offload_funcs
)[i
];
19950 targetm
.record_offload_symbol (it
);
19952 for (unsigned i
= 0; i
< num_vars
; i
++)
19954 tree it
= (*offload_vars
)[i
];
19955 targetm
.record_offload_symbol (it
);
19960 /* Find the number of threads (POS = false), or thread number (POS =
19961 true) for an OpenACC region partitioned as MASK. Setup code
19962 required for the calculation is added to SEQ. */
19965 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
19967 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
19970 /* Start at gang level, and examine relevant dimension indices. */
19971 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
19972 if (GOMP_DIM_MASK (ix
) & mask
)
19974 tree arg
= build_int_cst (unsigned_type_node
, ix
);
19978 /* We had an outer index, so scale that by the size of
19980 tree n
= create_tmp_var (integer_type_node
);
19982 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
19984 gimple_call_set_lhs (call
, n
);
19985 gimple_seq_add_stmt (seq
, call
);
19986 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
19990 /* Determine index in this dimension. */
19991 tree id
= create_tmp_var (integer_type_node
);
19992 gimple
*call
= gimple_build_call_internal
19993 (IFN_GOACC_DIM_POS
, 1, arg
);
19995 gimple_call_set_lhs (call
, id
);
19996 gimple_seq_add_stmt (seq
, call
);
19998 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
20004 if (res
== NULL_TREE
)
20005 res
= integer_zero_node
;
20010 /* Transform IFN_GOACC_LOOP calls to actual code. See
20011 expand_oacc_for for where these are generated. At the vector
20012 level, we stride loops, such that each member of a warp will
20013 operate on adjacent iterations. At the worker and gang level,
20014 each gang/warp executes a set of contiguous iterations. Chunking
20015 can override this such that each iteration engine executes a
20016 contiguous chunk, and then moves on to stride to the next chunk. */
20019 oacc_xform_loop (gcall
*call
)
20021 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
20022 enum ifn_goacc_loop_kind code
20023 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
20024 tree dir
= gimple_call_arg (call
, 1);
20025 tree range
= gimple_call_arg (call
, 2);
20026 tree step
= gimple_call_arg (call
, 3);
20027 tree chunk_size
= NULL_TREE
;
20028 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
20029 tree lhs
= gimple_call_lhs (call
);
20030 tree type
= TREE_TYPE (lhs
);
20031 tree diff_type
= TREE_TYPE (range
);
20032 tree r
= NULL_TREE
;
20033 gimple_seq seq
= NULL
;
20034 bool chunking
= false, striding
= true;
20035 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
20036 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
20038 #ifdef ACCEL_COMPILER
20039 chunk_size
= gimple_call_arg (call
, 4);
20040 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
20041 || integer_zerop (chunk_size
)) /* Default (also static). */
20043 /* If we're at the gang level, we want each to execute a
20044 contiguous run of iterations. Otherwise we want each element
20046 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
20051 /* Chunk of size 1 is striding. */
20052 striding
= integer_onep (chunk_size
);
20053 chunking
= !striding
;
20057 /* striding=true, chunking=true
20059 striding=true, chunking=false
20061 striding=false,chunking=true
20062 -> chunks=ceil (range/(chunksize*threads*step))
20063 striding=false,chunking=false
20064 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
20065 push_gimplify_context (true);
20069 default: gcc_unreachable ();
20071 case IFN_GOACC_LOOP_CHUNKS
:
20073 r
= build_int_cst (type
, 1);
20077 = (range - dir) / (chunks * step * num_threads) + dir */
20078 tree per
= oacc_thread_numbers (false, mask
, &seq
);
20079 per
= fold_convert (type
, per
);
20080 chunk_size
= fold_convert (type
, chunk_size
);
20081 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
20082 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
20083 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
20084 r
= build2 (PLUS_EXPR
, type
, r
, per
);
20085 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
20089 case IFN_GOACC_LOOP_STEP
:
20091 /* If striding, step by the entire compute volume, otherwise
20092 step by the inner volume. */
20093 unsigned volume
= striding
? mask
: inner_mask
;
20095 r
= oacc_thread_numbers (false, volume
, &seq
);
20096 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
20100 case IFN_GOACC_LOOP_OFFSET
:
20103 r
= oacc_thread_numbers (true, mask
, &seq
);
20104 r
= fold_convert (diff_type
, r
);
20108 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
20109 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
20110 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
20111 inner_size
, outer_size
);
20113 volume
= fold_convert (diff_type
, volume
);
20115 chunk_size
= fold_convert (diff_type
, chunk_size
);
20118 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
20120 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
20121 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
20122 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
20125 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
20126 fold_convert (diff_type
, inner_size
));
20127 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
20128 r
= fold_convert (diff_type
, r
);
20129 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
20131 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
20132 inner
= fold_convert (diff_type
, inner
);
20133 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
20137 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
20139 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
20140 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
20142 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
20145 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
20146 if (type
!= diff_type
)
20147 r
= fold_convert (type
, r
);
20150 case IFN_GOACC_LOOP_BOUND
:
20155 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
20156 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
20157 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
20158 inner_size
, outer_size
);
20160 volume
= fold_convert (diff_type
, volume
);
20162 chunk_size
= fold_convert (diff_type
, chunk_size
);
20165 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
20167 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
20168 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
20169 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
20172 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
20173 fold_convert (diff_type
, inner_size
));
20175 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
20177 tree offset
= gimple_call_arg (call
, 6);
20178 r
= build2 (PLUS_EXPR
, diff_type
, r
,
20179 fold_convert (diff_type
, offset
));
20180 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
20181 diff_type
, r
, range
);
20183 if (diff_type
!= type
)
20184 r
= fold_convert (type
, r
);
20188 gimplify_assign (lhs
, r
, &seq
);
20190 pop_gimplify_context (NULL
);
20192 gsi_replace_with_seq (&gsi
, seq
, true);
20195 /* Validate and update the dimensions for offloaded FN. ATTRS is the
20196 raw attribute. DIMS is an array of dimensions, which is returned.
20197 Returns the function level dimensionality -- the level at which an
20198 offload routine wishes to partition a loop. */
20201 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
20203 tree purpose
[GOMP_DIM_MAX
];
20205 tree pos
= TREE_VALUE (attrs
);
20208 /* Make sure the attribute creator attached the dimension
20212 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
20214 purpose
[ix
] = TREE_PURPOSE (pos
);
20218 if (integer_zerop (purpose
[ix
]))
20220 else if (fn_level
< 0)
20224 tree val
= TREE_VALUE (pos
);
20225 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
20226 pos
= TREE_CHAIN (pos
);
20229 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
20231 /* Default anything left to 1. */
20232 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
20241 /* Replace the attribute with new values. */
20243 for (ix
= GOMP_DIM_MAX
; ix
--;)
20244 pos
= tree_cons (purpose
[ix
],
20245 build_int_cst (integer_type_node
, dims
[ix
]),
20247 replace_oacc_fn_attrib (fn
, pos
);
20253 /* Create an empty OpenACC loop structure at LOC. */
20256 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
20258 oacc_loop
*loop
= XCNEW (oacc_loop
);
20260 loop
->parent
= parent
;
20261 loop
->child
= loop
->sibling
= NULL
;
20265 loop
->sibling
= parent
->child
;
20266 parent
->child
= loop
;
20270 loop
->marker
= NULL
;
20271 memset (loop
->heads
, 0, sizeof (loop
->heads
));
20272 memset (loop
->tails
, 0, sizeof (loop
->tails
));
20273 loop
->routine
= NULL_TREE
;
20275 loop
->mask
= loop
->flags
= 0;
20276 loop
->chunk_size
= 0;
20277 loop
->head_end
= NULL
;
20282 /* Create an outermost, dummy OpenACC loop for offloaded function
20286 new_oacc_loop_outer (tree decl
)
20288 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
20291 /* Start a new OpenACC loop structure beginning at head marker HEAD.
20292 Link into PARENT loop. Return the new loop. */
20295 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
20297 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
20299 loop
->marker
= marker
;
20301 /* TODO: This is where device_type flattening would occur for the loop
20304 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
20306 tree chunk_size
= integer_zero_node
;
20307 if (loop
->flags
& OLF_GANG_STATIC
)
20308 chunk_size
= gimple_call_arg (marker
, 4);
20309 loop
->chunk_size
= chunk_size
;
20314 /* Create a dummy loop encompassing a call to a openACC routine.
20315 Extract the routine's partitioning requirements. */
20318 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
20320 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
20321 int dims
[GOMP_DIM_MAX
];
20322 int level
= oacc_validate_dims (decl
, attrs
, dims
);
20324 gcc_assert (level
>= 0);
20326 loop
->marker
= call
;
20327 loop
->routine
= decl
;
20328 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
20329 ^ (GOMP_DIM_MASK (level
) - 1));
20332 /* Finish off the current OpenACC loop ending at tail marker TAIL.
20333 Return the parent loop. */
20336 finish_oacc_loop (oacc_loop
*loop
)
20338 return loop
->parent
;
20341 /* Free all OpenACC loop structures within LOOP (inclusive). */
20344 free_oacc_loop (oacc_loop
*loop
)
20347 free_oacc_loop (loop
->sibling
);
20349 free_oacc_loop (loop
->child
);
20354 /* Dump out the OpenACC loop head or tail beginning at FROM. */
20357 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
20358 const char *title
, int level
)
20360 enum ifn_unique_kind kind
20361 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
20363 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
20364 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
20366 gimple
*stmt
= gsi_stmt (gsi
);
20368 if (is_gimple_call (stmt
)
20369 && gimple_call_internal_p (stmt
)
20370 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
20372 enum ifn_unique_kind k
20373 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
20374 (gimple_call_arg (stmt
, 0)));
20376 if (k
== kind
&& stmt
!= from
)
20379 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
20382 while (gsi_end_p (gsi
))
20383 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
20387 /* Dump OpenACC loops LOOP, its siblings and its children. */
20390 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
20394 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
20395 loop
->flags
, loop
->mask
,
20396 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
20399 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
20402 fprintf (file
, "%*sRoutine %s:%u:%s\n",
20403 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
20404 DECL_SOURCE_LINE (loop
->routine
),
20405 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
20407 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
20408 if (loop
->heads
[ix
])
20409 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
20410 for (ix
= GOMP_DIM_MAX
; ix
--;)
20411 if (loop
->tails
[ix
])
20412 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
20415 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
20417 dump_oacc_loop (file
, loop
->sibling
, depth
);
20420 void debug_oacc_loop (oacc_loop
*);
20422 /* Dump loops to stderr. */
20424 DEBUG_FUNCTION
void
20425 debug_oacc_loop (oacc_loop
*loop
)
20427 dump_oacc_loop (stderr
, loop
, 0);
20430 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
20431 structures as we go. By construction these loops are properly
20435 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
20440 if (bb
->flags
& BB_VISITED
)
20444 bb
->flags
|= BB_VISITED
;
20446 /* Scan for loop markers. */
20447 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
20450 gimple
*stmt
= gsi_stmt (gsi
);
20452 if (!is_gimple_call (stmt
))
20455 gcall
*call
= as_a
<gcall
*> (stmt
);
20457 /* If this is a routine, make a dummy loop for it. */
20458 if (tree decl
= gimple_call_fndecl (call
))
20459 if (tree attrs
= get_oacc_fn_attrib (decl
))
20461 gcc_assert (!marker
);
20462 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
20465 if (!gimple_call_internal_p (call
))
20468 if (gimple_call_internal_fn (call
) != IFN_UNIQUE
)
20471 enum ifn_unique_kind kind
20472 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
20473 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
20474 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
20476 if (gimple_call_num_args (call
) == 2)
20478 gcc_assert (marker
&& !remaining
);
20480 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
20481 loop
= finish_oacc_loop (loop
);
20483 loop
->head_end
= call
;
20487 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
20491 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
20492 loop
= new_oacc_loop (loop
, call
);
20495 gcc_assert (count
== remaining
);
20499 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
20500 loop
->heads
[marker
] = call
;
20502 loop
->tails
[remaining
] = call
;
20508 if (remaining
|| marker
)
20510 bb
= single_succ (bb
);
20511 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
20515 /* Walk successor blocks. */
20519 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
20520 oacc_loop_discover_walk (loop
, e
->dest
);
20523 /* LOOP is the first sibling. Reverse the order in place and return
20524 the new first sibling. Recurse to child loops. */
20527 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
20529 oacc_loop
*last
= NULL
;
20533 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
20535 oacc_loop
*next
= loop
->sibling
;
20536 loop
->sibling
= last
;
20545 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
20546 the current function. */
20549 oacc_loop_discovery ()
20553 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
20554 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
20556 /* The siblings were constructed in reverse order, reverse them so
20557 that diagnostics come out in an unsurprising order. */
20558 top
= oacc_loop_sibling_nreverse (top
);
20560 /* Reset the visited flags. */
20561 FOR_ALL_BB_FN (bb
, cfun
)
20562 bb
->flags
&= ~BB_VISITED
;
20567 /* Transform the abstract internal function markers starting at FROM
20568 to be for partitioning level LEVEL. Stop when we meet another HEAD
20572 oacc_loop_xform_head_tail (gcall
*from
, int level
)
20574 enum ifn_unique_kind kind
20575 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
20576 tree replacement
= build_int_cst (unsigned_type_node
, level
);
20578 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
20580 gimple
*stmt
= gsi_stmt (gsi
);
20582 if (is_gimple_call (stmt
)
20583 && gimple_call_internal_p (stmt
)
20584 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
20586 enum ifn_unique_kind k
20587 = ((enum ifn_unique_kind
)
20588 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
20590 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
20591 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
20592 else if (k
== kind
&& stmt
!= from
)
20595 else if (is_gimple_call (stmt
)
20596 && gimple_call_internal_p (stmt
)
20597 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
20598 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
20601 while (gsi_end_p (gsi
))
20602 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
20606 /* Transform the IFN_GOACC_LOOP internal functions by providing the
20607 determined partitioning mask and chunking argument. */
20610 oacc_loop_xform_loop (gcall
*end_marker
, tree mask_arg
, tree chunk_arg
)
20612 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
20616 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
20618 gimple
*stmt
= gsi_stmt (gsi
);
20620 if (!is_gimple_call (stmt
))
20623 gcall
*call
= as_a
<gcall
*> (stmt
);
20625 if (!gimple_call_internal_p (call
))
20628 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
20631 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
20632 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
20633 if (TREE_INT_CST_LOW (gimple_call_arg (call
, 0))
20634 == IFN_GOACC_LOOP_BOUND
)
20638 /* If we didn't see LOOP_BOUND, it should be in the single
20639 successor block. */
20640 basic_block bb
= single_succ (gsi_bb (gsi
));
20641 gsi
= gsi_start_bb (bb
);
20645 /* Process the discovered OpenACC loops, setting the correct
20646 partitioning level etc. */
20649 oacc_loop_process (oacc_loop
*loop
)
20652 oacc_loop_process (loop
->child
);
20654 if (loop
->mask
&& !loop
->routine
)
20657 unsigned mask
= loop
->mask
;
20658 unsigned dim
= GOMP_DIM_GANG
;
20659 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
20660 tree chunk_arg
= loop
->chunk_size
;
20662 oacc_loop_xform_loop (loop
->head_end
, mask_arg
, chunk_arg
);
20664 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
20668 while (!(GOMP_DIM_MASK (dim
) & mask
))
20671 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
20672 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
20674 mask
^= GOMP_DIM_MASK (dim
);
20679 oacc_loop_process (loop
->sibling
);
20682 /* Walk the OpenACC loop heirarchy checking and assigning the
20683 programmer-specified partitionings. OUTER_MASK is the partitioning
20684 this loop is contained within. Return true if we contain an
20685 auto-partitionable loop. */
20688 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
20690 unsigned this_mask
= loop
->mask
;
20691 bool has_auto
= false;
20694 #ifdef ACCEL_COMPILER
20695 /* When device_type is supported, we want the device compiler to be
20696 noisy, if the loop parameters are device_type-specific. */
20700 if (!loop
->routine
)
20702 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
20703 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
20705 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
20706 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
20708 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
20711 error_at (loop
->loc
,
20713 ? "%<seq%> overrides other OpenACC loop specifiers"
20714 : "%<auto%> conflicts with other OpenACC loop specifiers");
20716 loop
->flags
&= ~OLF_AUTO
;
20720 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
20724 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
20728 if (this_mask
& outer_mask
)
20730 const oacc_loop
*outer
;
20731 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
20732 if (outer
->mask
& this_mask
)
20739 error_at (loop
->loc
,
20740 "%s uses same OpenACC parallelism as containing loop",
20741 loop
->routine
? "routine call" : "inner loop");
20742 inform (outer
->loc
, "containing loop here");
20745 error_at (loop
->loc
,
20746 "%s uses OpenACC parallelism disallowed by containing routine",
20747 loop
->routine
? "routine call" : "loop");
20750 inform (DECL_SOURCE_LOCATION (loop
->routine
),
20751 "routine %qD declared here", loop
->routine
);
20753 this_mask
&= ~outer_mask
;
20757 unsigned outermost
= this_mask
& -this_mask
;
20759 if (outermost
&& outermost
<= outer_mask
)
20763 error_at (loop
->loc
,
20764 "incorrectly nested OpenACC loop parallelism");
20766 const oacc_loop
*outer
;
20767 for (outer
= loop
->parent
;
20768 outer
->flags
&& outer
->flags
< outermost
;
20769 outer
= outer
->parent
)
20771 inform (outer
->loc
, "containing loop here");
20774 this_mask
&= ~outermost
;
20778 loop
->mask
= this_mask
;
20781 && oacc_loop_fixed_partitions (loop
->child
, outer_mask
| this_mask
))
20785 && oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
))
20791 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
20792 OUTER_MASK is the partitioning this loop is contained within.
20793 Return the cumulative partitioning used by this loop, siblings and
20797 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
20799 unsigned inner_mask
= 0;
20802 #ifdef ACCEL_COMPILER
20803 /* When device_type is supported, we want the device compiler to be
20804 noisy, if the loop parameters are device_type-specific. */
20809 inner_mask
|= oacc_loop_auto_partitions (loop
->child
,
20810 outer_mask
| loop
->mask
);
20812 if ((loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
))
20814 unsigned this_mask
= 0;
20816 /* Determine the outermost partitioning used within this loop. */
20817 this_mask
= inner_mask
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
20818 this_mask
= (this_mask
& -this_mask
);
20820 /* Pick the partitioning just inside that one. */
20823 /* And avoid picking one use by an outer loop. */
20824 this_mask
&= ~outer_mask
;
20826 if (!this_mask
&& noisy
)
20827 warning_at (loop
->loc
, 0,
20828 "insufficient partitioning available to parallelize loop");
20830 loop
->mask
= this_mask
;
20832 inner_mask
|= loop
->mask
;
20835 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
20840 /* Walk the OpenACC loop heirarchy to check and assign partitioning
20844 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
20846 if (oacc_loop_fixed_partitions (loop
, outer_mask
))
20847 oacc_loop_auto_partitions (loop
, outer_mask
);
20850 /* Default fork/join early expander. Delete the function calls if
20851 there is no RTL expander. */
20854 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
20855 const int *ARG_UNUSED (dims
), bool is_fork
)
20858 return targetm
.have_oacc_fork ();
20860 return targetm
.have_oacc_join ();
20863 /* Default goacc.reduction early expander.
20865 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
20866 If RES_PTR is not integer-zerop:
20867 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
20868 TEARDOWN - emit '*RES_PTR = VAR'
20870 emit 'LHS = VAR' */
20873 default_goacc_reduction (gcall
*call
)
20875 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
20876 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
20877 tree lhs
= gimple_call_lhs (call
);
20878 tree var
= gimple_call_arg (call
, 2);
20879 gimple_seq seq
= NULL
;
20881 if (code
== IFN_GOACC_REDUCTION_SETUP
20882 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
20884 /* Setup and Teardown need to copy from/to the receiver object,
20885 if there is one. */
20886 tree ref_to_res
= gimple_call_arg (call
, 1);
20888 if (!integer_zerop (ref_to_res
))
20890 tree dst
= build_simple_mem_ref (ref_to_res
);
20893 if (code
== IFN_GOACC_REDUCTION_SETUP
)
20899 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
20903 /* Copy VAR to LHS, if there is an LHS. */
20905 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
20907 gsi_replace_with_seq (&gsi
, seq
, true);
20910 /* Main entry point for oacc transformations which run on the device
20911 compiler after LTO, so we know what the target device is at this
20912 point (including the host fallback). */
20914 static unsigned int
20915 execute_oacc_device_lower ()
20917 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
20918 int dims
[GOMP_DIM_MAX
];
20921 /* Not an offloaded function. */
20924 int fn_level
= oacc_validate_dims (current_function_decl
, attrs
, dims
);
20926 /* Discover, partition and process the loops. */
20927 oacc_loop
*loops
= oacc_loop_discovery ();
20928 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
20929 oacc_loop_partition (loops
, outer_mask
);
20930 oacc_loop_process (loops
);
20933 fprintf (dump_file
, "OpenACC loops\n");
20934 dump_oacc_loop (dump_file
, loops
, 0);
20935 fprintf (dump_file
, "\n");
20938 /* Offloaded targets may introduce new basic blocks, which require
20939 dominance information to update SSA. */
20940 calculate_dominance_info (CDI_DOMINATORS
);
20942 /* Now lower internal loop functions to target-specific code
20945 FOR_ALL_BB_FN (bb
, cfun
)
20946 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
20948 gimple
*stmt
= gsi_stmt (gsi
);
20949 if (!is_gimple_call (stmt
))
20955 gcall
*call
= as_a
<gcall
*> (stmt
);
20956 if (!gimple_call_internal_p (call
))
20962 /* Rewind to allow rescan. */
20964 bool rescan
= false, remove
= false;
20965 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
20971 case IFN_GOACC_LOOP
:
20972 oacc_xform_loop (call
);
20976 case IFN_GOACC_REDUCTION
:
20977 /* Mark the function for SSA renaming. */
20978 mark_virtual_operands_for_renaming (cfun
);
20980 /* If the level is -1, this ended up being an unused
20981 axis. Handle as a default. */
20982 if (integer_minus_onep (gimple_call_arg (call
, 3)))
20983 default_goacc_reduction (call
);
20985 targetm
.goacc
.reduction (call
);
20991 enum ifn_unique_kind kind
20992 = ((enum ifn_unique_kind
)
20993 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
20998 gcc_unreachable ();
21000 case IFN_UNIQUE_OACC_FORK
:
21001 case IFN_UNIQUE_OACC_JOIN
:
21002 if (integer_minus_onep (gimple_call_arg (call
, 2)))
21004 else if (!targetm
.goacc
.fork_join
21005 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
21009 case IFN_UNIQUE_OACC_HEAD_MARK
:
21010 case IFN_UNIQUE_OACC_TAIL_MARK
:
21018 if (gsi_end_p (gsi
))
21019 /* We rewound past the beginning of the BB. */
21020 gsi
= gsi_start_bb (bb
);
21022 /* Undo the rewind. */
21027 if (gimple_vdef (call
))
21028 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
21029 if (gimple_call_lhs (call
))
21031 /* Propagate the data dependency var. */
21032 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
21033 gimple_call_arg (call
, 1));
21034 gsi_replace (&gsi
, ass
, false);
21037 gsi_remove (&gsi
, true);
21040 /* If not rescanning, advance over the call. */
21044 free_oacc_loop (loops
);
21049 /* Default launch dimension validator. Force everything to 1. A
21050 backend that wants to provide larger dimensions must override this
21054 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
21055 int ARG_UNUSED (fn_level
))
21057 bool changed
= false;
21059 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
21071 /* Default dimension bound is unknown on accelerator and 1 on host. */
21074 default_goacc_dim_limit (int ARG_UNUSED (axis
))
21076 #ifdef ACCEL_COMPILER
21085 const pass_data pass_data_oacc_device_lower
=
21087 GIMPLE_PASS
, /* type */
21088 "oaccdevlow", /* name */
21089 OPTGROUP_NONE
, /* optinfo_flags */
21090 TV_NONE
, /* tv_id */
21091 PROP_cfg
, /* properties_required */
21092 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
21093 0, /* properties_destroyed */
21094 0, /* todo_flags_start */
21095 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
21098 class pass_oacc_device_lower
: public gimple_opt_pass
21101 pass_oacc_device_lower (gcc::context
*ctxt
)
21102 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
21105 /* opt_pass methods: */
21106 virtual unsigned int execute (function
*)
21108 bool gate
= flag_openacc
!= 0;
21113 return execute_oacc_device_lower ();
21116 }; // class pass_oacc_device_lower
21118 } // anon namespace
21121 make_pass_oacc_device_lower (gcc::context
*ctxt
)
21123 return new pass_oacc_device_lower (ctxt
);
21126 /* "omp declare target link" handling pass. */
21130 const pass_data pass_data_omp_target_link
=
21132 GIMPLE_PASS
, /* type */
21133 "omptargetlink", /* name */
21134 OPTGROUP_NONE
, /* optinfo_flags */
21135 TV_NONE
, /* tv_id */
21136 PROP_ssa
, /* properties_required */
21137 0, /* properties_provided */
21138 0, /* properties_destroyed */
21139 0, /* todo_flags_start */
21140 TODO_update_ssa
, /* todo_flags_finish */
21143 class pass_omp_target_link
: public gimple_opt_pass
21146 pass_omp_target_link (gcc::context
*ctxt
)
21147 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
21150 /* opt_pass methods: */
21151 virtual bool gate (function
*fun
)
21153 #ifdef ACCEL_COMPILER
21154 tree attrs
= DECL_ATTRIBUTES (fun
->decl
);
21155 return lookup_attribute ("omp declare target", attrs
)
21156 || lookup_attribute ("omp target entrypoint", attrs
);
21163 virtual unsigned execute (function
*);
21166 /* Callback for walk_gimple_stmt used to scan for link var operands. */
21169 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
21173 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
)
21174 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
21176 *walk_subtrees
= 0;
21184 pass_omp_target_link::execute (function
*fun
)
21187 FOR_EACH_BB_FN (bb
, fun
)
21189 gimple_stmt_iterator gsi
;
21190 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
21191 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
21192 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
21198 } // anon namespace
21201 make_pass_omp_target_link (gcc::context
*ctxt
)
21203 return new pass_omp_target_link (ctxt
);
21206 #include "gt-omp-low.h"