1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region
*outer
;
107 /* First child region. */
108 struct omp_region
*inner
;
110 /* Next peer region. */
111 struct omp_region
*next
;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec
<tree
, va_gc
> *ws_args
;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type
;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind
;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered
*ord_stmt
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind
;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v
, n1
, n2
, step
;
204 enum tree_code cond_code
;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop
;
217 bool have_nowait
, have_ordered
, simd_schedule
;
218 unsigned char sched_modifiers
;
219 enum omp_clause_schedule_kind sched_kind
;
220 struct omp_for_data_loop
*loops
;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop
*parent
; /* Containing loop. */
230 oacc_loop
*child
; /* First inner loop. */
232 oacc_loop
*sibling
; /* Next loop within same parent. */
234 location_t loc
; /* Location of the loop start. */
236 gcall
*marker
; /* Initial head marker. */
238 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
239 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
241 tree routine
; /* Pseudo-loop enclosing a routine. */
243 unsigned mask
; /* Partitioning mask. */
244 unsigned inner
; /* Partitioning of inner loops. */
245 unsigned flags
; /* Partitioning flags. */
246 unsigned ifns
; /* Contained loop abstraction functions. */
247 tree chunk_size
; /* Chunk size. */
248 gcall
*head_end
; /* Final marker of head sequence. */
251 /* Flags for an OpenACC loop. */
253 enum oacc_loop_flags
{
254 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
255 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
256 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
257 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
259 /* Explicitly specified loop axes. */
261 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
262 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
263 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
265 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
269 static splay_tree all_contexts
;
270 static int taskreg_nesting_level
;
271 static int target_nesting_level
;
272 static struct omp_region
*root_omp_region
;
273 static bitmap task_shared_vars
;
274 static vec
<omp_context
*> taskreg_contexts
;
275 static bool omp_any_child_fn_dumped
;
277 static void scan_omp (gimple_seq
*, omp_context
*);
278 static tree
scan_omp_1_op (tree
*, int *, void *);
279 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
281 #define WALK_SUBSTMTS \
285 case GIMPLE_EH_FILTER: \
286 case GIMPLE_TRANSACTION: \
287 /* The sub-statements for these should be walked. */ \
288 *handled_ops_p = false; \
291 /* Return true if CTX corresponds to an oacc parallel region. */
294 is_oacc_parallel (omp_context
*ctx
)
296 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
297 return ((outer_type
== GIMPLE_OMP_TARGET
)
298 && (gimple_omp_target_kind (ctx
->stmt
)
299 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
302 /* Return true if CTX corresponds to an oacc kernels region. */
305 is_oacc_kernels (omp_context
*ctx
)
307 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
308 return ((outer_type
== GIMPLE_OMP_TARGET
)
309 && (gimple_omp_target_kind (ctx
->stmt
)
310 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
313 /* If DECL is the artificial dummy VAR_DECL created for non-static
314 data member privatization, return the underlying "this" parameter,
315 otherwise return NULL. */
318 omp_member_access_dummy_var (tree decl
)
321 || !DECL_ARTIFICIAL (decl
)
322 || !DECL_IGNORED_P (decl
)
323 || !DECL_HAS_VALUE_EXPR_P (decl
)
324 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
327 tree v
= DECL_VALUE_EXPR (decl
);
328 if (TREE_CODE (v
) != COMPONENT_REF
)
332 switch (TREE_CODE (v
))
338 case POINTER_PLUS_EXPR
:
339 v
= TREE_OPERAND (v
, 0);
342 if (DECL_CONTEXT (v
) == current_function_decl
343 && DECL_ARTIFICIAL (v
)
344 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
352 /* Helper for unshare_and_remap, called through walk_tree. */
355 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
357 tree
*pair
= (tree
*) data
;
360 *tp
= unshare_expr (pair
[1]);
363 else if (IS_TYPE_OR_DECL_P (*tp
))
368 /* Return unshare_expr (X) with all occurrences of FROM
372 unshare_and_remap (tree x
, tree from
, tree to
)
374 tree pair
[2] = { from
, to
};
375 x
= unshare_expr (x
);
376 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
380 /* Holds offload tables with decls. */
381 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
383 /* Convenience function for calling scan_omp_1_op on tree operands. */
386 scan_omp_op (tree
*tp
, omp_context
*ctx
)
388 struct walk_stmt_info wi
;
390 memset (&wi
, 0, sizeof (wi
));
392 wi
.want_locations
= true;
394 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
397 static void lower_omp (gimple_seq
*, omp_context
*);
398 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
399 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
401 /* Find an OMP clause of type KIND within CLAUSES. */
404 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
406 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
407 if (OMP_CLAUSE_CODE (clauses
) == kind
)
413 /* Return true if CTX is for an omp parallel. */
416 is_parallel_ctx (omp_context
*ctx
)
418 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
422 /* Return true if CTX is for an omp task. */
425 is_task_ctx (omp_context
*ctx
)
427 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
431 /* Return true if CTX is for an omp taskloop. */
434 is_taskloop_ctx (omp_context
*ctx
)
436 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
437 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
441 /* Return true if CTX is for an omp parallel or omp task. */
444 is_taskreg_ctx (omp_context
*ctx
)
446 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
450 /* Return true if REGION is a combined parallel+workshare region. */
453 is_combined_parallel (struct omp_region
*region
)
455 return region
->is_combined_parallel
;
458 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
462 adjust_for_condition (location_t loc
, enum tree_code
*cond_code
, tree
*n2
)
471 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
472 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, 1);
474 *n2
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (*n2
), *n2
,
475 build_int_cst (TREE_TYPE (*n2
), 1));
476 *cond_code
= LT_EXPR
;
479 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
480 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, -1);
482 *n2
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (*n2
), *n2
,
483 build_int_cst (TREE_TYPE (*n2
), 1));
484 *cond_code
= GT_EXPR
;
491 /* Return the looping step from INCR, extracted from the step of a gimple omp
495 get_omp_for_step_from_incr (location_t loc
, tree incr
)
498 switch (TREE_CODE (incr
))
501 step
= TREE_OPERAND (incr
, 1);
503 case POINTER_PLUS_EXPR
:
504 step
= fold_convert (ssizetype
, TREE_OPERAND (incr
, 1));
507 step
= TREE_OPERAND (incr
, 1);
508 step
= fold_build1_loc (loc
, NEGATE_EXPR
, TREE_TYPE (step
), step
);
516 /* Extract the header elements of parallel loop FOR_STMT and store
520 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
521 struct omp_for_data_loop
*loops
)
523 tree t
, var
, *collapse_iter
, *collapse_count
;
524 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
525 struct omp_for_data_loop
*loop
;
527 struct omp_for_data_loop dummy_loop
;
528 location_t loc
= gimple_location (for_stmt
);
529 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
530 bool distribute
= gimple_omp_for_kind (for_stmt
)
531 == GF_OMP_FOR_KIND_DISTRIBUTE
;
532 bool taskloop
= gimple_omp_for_kind (for_stmt
)
533 == GF_OMP_FOR_KIND_TASKLOOP
;
536 fd
->for_stmt
= for_stmt
;
538 if (gimple_omp_for_collapse (for_stmt
) > 1)
541 fd
->loops
= &fd
->loop
;
543 fd
->have_nowait
= distribute
|| simd
;
544 fd
->have_ordered
= false;
547 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
548 fd
->sched_modifiers
= 0;
549 fd
->chunk_size
= NULL_TREE
;
550 fd
->simd_schedule
= false;
551 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
552 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
553 collapse_iter
= NULL
;
554 collapse_count
= NULL
;
556 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
557 switch (OMP_CLAUSE_CODE (t
))
559 case OMP_CLAUSE_NOWAIT
:
560 fd
->have_nowait
= true;
562 case OMP_CLAUSE_ORDERED
:
563 fd
->have_ordered
= true;
564 if (OMP_CLAUSE_ORDERED_EXPR (t
))
565 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
567 case OMP_CLAUSE_SCHEDULE
:
568 gcc_assert (!distribute
&& !taskloop
);
570 = (enum omp_clause_schedule_kind
)
571 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
572 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
573 & ~OMP_CLAUSE_SCHEDULE_MASK
);
574 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
575 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
577 case OMP_CLAUSE_DIST_SCHEDULE
:
578 gcc_assert (distribute
);
579 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
581 case OMP_CLAUSE_COLLAPSE
:
582 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
583 if (fd
->collapse
> 1)
585 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
586 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
592 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
597 collapse_iter
= &iterv
;
598 collapse_count
= &countv
;
601 /* FIXME: for now map schedule(auto) to schedule(static).
602 There should be analysis to determine whether all iterations
603 are approximately the same amount of work (then schedule(static)
604 is best) or if it varies (then schedule(dynamic,N) is better). */
605 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
607 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
608 gcc_assert (fd
->chunk_size
== NULL
);
610 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
612 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
613 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
614 gcc_assert (fd
->chunk_size
== NULL
);
615 else if (fd
->chunk_size
== NULL
)
617 /* We only need to compute a default chunk size for ordered
618 static loops and dynamic loops. */
619 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
621 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
622 ? integer_zero_node
: integer_one_node
;
625 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
626 for (i
= 0; i
< cnt
; i
++)
628 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
630 else if (loops
!= NULL
)
635 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
636 gcc_assert (SSA_VAR_P (loop
->v
));
637 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
638 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
639 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
640 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
642 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
643 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
644 gcc_assert (loop
->cond_code
!= NE_EXPR
645 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKSIMD
646 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
);
647 adjust_for_condition (loc
, &loop
->cond_code
, &loop
->n2
);
649 t
= gimple_omp_for_incr (for_stmt
, i
);
650 gcc_assert (TREE_OPERAND (t
, 0) == var
);
651 loop
->step
= get_omp_for_step_from_incr (loc
, t
);
654 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
655 && !fd
->have_ordered
))
657 if (fd
->collapse
== 1)
658 iter_type
= TREE_TYPE (loop
->v
);
660 || TYPE_PRECISION (iter_type
)
661 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
663 = build_nonstandard_integer_type
664 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
666 else if (iter_type
!= long_long_unsigned_type_node
)
668 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
669 iter_type
= long_long_unsigned_type_node
;
670 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
671 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
672 >= TYPE_PRECISION (iter_type
))
676 if (loop
->cond_code
== LT_EXPR
)
677 n
= fold_build2_loc (loc
,
678 PLUS_EXPR
, TREE_TYPE (loop
->v
),
679 loop
->n2
, loop
->step
);
682 if (TREE_CODE (n
) != INTEGER_CST
683 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
684 iter_type
= long_long_unsigned_type_node
;
686 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
687 > TYPE_PRECISION (iter_type
))
691 if (loop
->cond_code
== LT_EXPR
)
694 n2
= fold_build2_loc (loc
,
695 PLUS_EXPR
, TREE_TYPE (loop
->v
),
696 loop
->n2
, loop
->step
);
700 n1
= fold_build2_loc (loc
,
701 MINUS_EXPR
, TREE_TYPE (loop
->v
),
702 loop
->n2
, loop
->step
);
705 if (TREE_CODE (n1
) != INTEGER_CST
706 || TREE_CODE (n2
) != INTEGER_CST
707 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
708 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
709 iter_type
= long_long_unsigned_type_node
;
713 if (i
>= fd
->collapse
)
716 if (collapse_count
&& *collapse_count
== NULL
)
718 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
719 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
720 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
721 if (t
&& integer_zerop (t
))
722 count
= build_zero_cst (long_long_unsigned_type_node
);
723 else if ((i
== 0 || count
!= NULL_TREE
)
724 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
725 && TREE_CONSTANT (loop
->n1
)
726 && TREE_CONSTANT (loop
->n2
)
727 && TREE_CODE (loop
->step
) == INTEGER_CST
)
729 tree itype
= TREE_TYPE (loop
->v
);
731 if (POINTER_TYPE_P (itype
))
732 itype
= signed_type_for (itype
);
733 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
734 t
= fold_build2_loc (loc
,
736 fold_convert_loc (loc
, itype
, loop
->step
), t
);
737 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
738 fold_convert_loc (loc
, itype
, loop
->n2
));
739 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
740 fold_convert_loc (loc
, itype
, loop
->n1
));
741 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
742 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
743 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
744 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
745 fold_convert_loc (loc
, itype
,
748 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
749 fold_convert_loc (loc
, itype
, loop
->step
));
750 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
751 if (count
!= NULL_TREE
)
752 count
= fold_build2_loc (loc
,
753 MULT_EXPR
, long_long_unsigned_type_node
,
757 if (TREE_CODE (count
) != INTEGER_CST
)
760 else if (count
&& !integer_zerop (count
))
767 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
768 || fd
->have_ordered
))
770 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
771 iter_type
= long_long_unsigned_type_node
;
773 iter_type
= long_integer_type_node
;
775 else if (collapse_iter
&& *collapse_iter
!= NULL
)
776 iter_type
= TREE_TYPE (*collapse_iter
);
777 fd
->iter_type
= iter_type
;
778 if (collapse_iter
&& *collapse_iter
== NULL
)
779 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
780 if (collapse_count
&& *collapse_count
== NULL
)
783 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
785 *collapse_count
= create_tmp_var (iter_type
, ".count");
788 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
790 fd
->loop
.v
= *collapse_iter
;
791 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
792 fd
->loop
.n2
= *collapse_count
;
793 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
794 fd
->loop
.cond_code
= LT_EXPR
;
801 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
802 is the immediate dominator of PAR_ENTRY_BB, return true if there
803 are no data dependencies that would prevent expanding the parallel
804 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
806 When expanding a combined parallel+workshare region, the call to
807 the child function may need additional arguments in the case of
808 GIMPLE_OMP_FOR regions. In some cases, these arguments are
809 computed out of variables passed in from the parent to the child
810 via 'struct .omp_data_s'. For instance:
812 #pragma omp parallel for schedule (guided, i * 4)
817 # BLOCK 2 (PAR_ENTRY_BB)
819 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
821 # BLOCK 3 (WS_ENTRY_BB)
822 .omp_data_i = &.omp_data_o;
823 D.1667 = .omp_data_i->i;
825 #pragma omp for schedule (guided, D.1598)
827 When we outline the parallel region, the call to the child function
828 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
829 that value is computed *after* the call site. So, in principle we
830 cannot do the transformation.
832 To see whether the code in WS_ENTRY_BB blocks the combined
833 parallel+workshare call, we collect all the variables used in the
834 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
835 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
838 FIXME. If we had the SSA form built at this point, we could merely
839 hoist the code in block 3 into block 2 and be done with it. But at
840 this point we don't have dataflow information and though we could
841 hack something up here, it is really not worth the aggravation. */
844 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
846 struct omp_for_data fd
;
847 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
849 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
852 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
854 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
856 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
858 if (fd
.iter_type
!= long_integer_type_node
)
861 /* FIXME. We give up too easily here. If any of these arguments
862 are not constants, they will likely involve variables that have
863 been mapped into fields of .omp_data_s for sharing with the child
864 function. With appropriate data flow, it would be possible to
866 if (!is_gimple_min_invariant (fd
.loop
.n1
)
867 || !is_gimple_min_invariant (fd
.loop
.n2
)
868 || !is_gimple_min_invariant (fd
.loop
.step
)
869 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
876 static int omp_max_vf (void);
878 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
879 presence (SIMD_SCHEDULE). */
882 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
887 int vf
= omp_max_vf ();
891 tree type
= TREE_TYPE (chunk_size
);
892 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
893 build_int_cst (type
, vf
- 1));
894 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
895 build_int_cst (type
, -vf
));
899 /* Collect additional arguments needed to emit a combined
900 parallel+workshare call. WS_STMT is the workshare directive being
903 static vec
<tree
, va_gc
> *
904 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
907 location_t loc
= gimple_location (ws_stmt
);
908 vec
<tree
, va_gc
> *ws_args
;
910 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
912 struct omp_for_data fd
;
915 extract_omp_for_data (for_stmt
, &fd
, NULL
);
919 if (gimple_omp_for_combined_into_p (for_stmt
))
922 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
923 OMP_CLAUSE__LOOPTEMP_
);
925 n1
= OMP_CLAUSE_DECL (innerc
);
926 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
927 OMP_CLAUSE__LOOPTEMP_
);
929 n2
= OMP_CLAUSE_DECL (innerc
);
932 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
934 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
935 ws_args
->quick_push (t
);
937 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
938 ws_args
->quick_push (t
);
940 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
941 ws_args
->quick_push (t
);
945 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
946 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
947 ws_args
->quick_push (t
);
952 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
954 /* Number of sections is equal to the number of edges from the
955 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
956 the exit of the sections region. */
957 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
958 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
959 vec_alloc (ws_args
, 1);
960 ws_args
->quick_push (t
);
968 /* Discover whether REGION is a combined parallel+workshare region. */
971 determine_parallel_type (struct omp_region
*region
)
973 basic_block par_entry_bb
, par_exit_bb
;
974 basic_block ws_entry_bb
, ws_exit_bb
;
976 if (region
== NULL
|| region
->inner
== NULL
977 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
978 || region
->inner
->cont
== NULL
)
981 /* We only support parallel+for and parallel+sections. */
982 if (region
->type
!= GIMPLE_OMP_PARALLEL
983 || (region
->inner
->type
!= GIMPLE_OMP_FOR
984 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
987 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
988 WS_EXIT_BB -> PAR_EXIT_BB. */
989 par_entry_bb
= region
->entry
;
990 par_exit_bb
= region
->exit
;
991 ws_entry_bb
= region
->inner
->entry
;
992 ws_exit_bb
= region
->inner
->exit
;
994 if (single_succ (par_entry_bb
) == ws_entry_bb
995 && single_succ (ws_exit_bb
) == par_exit_bb
996 && workshare_safe_to_combine_p (ws_entry_bb
)
997 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
998 || (last_and_only_stmt (ws_entry_bb
)
999 && last_and_only_stmt (par_exit_bb
))))
1001 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1002 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1004 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1006 /* If this is a combined parallel loop, we need to determine
1007 whether or not to use the combined library calls. There
1008 are two cases where we do not apply the transformation:
1009 static loops and any kind of ordered loop. In the first
1010 case, we already open code the loop so there is no need
1011 to do anything else. In the latter case, the combined
1012 parallel loop call would still need extra synchronization
1013 to implement ordered semantics, so there would not be any
1014 gain in using the combined call. */
1015 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1016 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1018 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1019 == OMP_CLAUSE_SCHEDULE_STATIC
)
1020 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1022 region
->is_combined_parallel
= false;
1023 region
->inner
->is_combined_parallel
= false;
1028 region
->is_combined_parallel
= true;
1029 region
->inner
->is_combined_parallel
= true;
1030 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1035 /* Return true if EXPR is variable sized. */
1038 is_variable_sized (const_tree expr
)
1040 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1043 /* Return true if DECL is a reference type. */
1046 is_reference (tree decl
)
1048 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1051 /* Return the type of a decl. If the decl is reference type,
1052 return its base type. */
1054 get_base_type (tree decl
)
1056 tree type
= TREE_TYPE (decl
);
1057 if (is_reference (decl
))
1058 type
= TREE_TYPE (type
);
1062 /* Lookup variables. The "maybe" form
1063 allows for the variable form to not have been entered, otherwise we
1064 assert that the variable must have been entered. */
1067 lookup_decl (tree var
, omp_context
*ctx
)
1069 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1074 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1076 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1077 return n
? *n
: NULL_TREE
;
1081 lookup_field (tree var
, omp_context
*ctx
)
1084 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1085 return (tree
) n
->value
;
1089 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1092 n
= splay_tree_lookup (ctx
->sfield_map
1093 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1094 return (tree
) n
->value
;
1098 lookup_sfield (tree var
, omp_context
*ctx
)
1100 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1104 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1107 n
= splay_tree_lookup (ctx
->field_map
, key
);
1108 return n
? (tree
) n
->value
: NULL_TREE
;
1112 maybe_lookup_field (tree var
, omp_context
*ctx
)
1114 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1117 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1118 the parallel context if DECL is to be shared. */
1121 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1123 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
))
1124 || TYPE_ATOMIC (TREE_TYPE (decl
)))
1127 /* We can only use copy-in/copy-out semantics for shared variables
1128 when we know the value is not accessible from an outer scope. */
1131 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1133 /* ??? Trivially accessible from anywhere. But why would we even
1134 be passing an address in this case? Should we simply assert
1135 this to be false, or should we have a cleanup pass that removes
1136 these from the list of mappings? */
1137 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1140 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1141 without analyzing the expression whether or not its location
1142 is accessible to anyone else. In the case of nested parallel
1143 regions it certainly may be. */
1144 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1147 /* Do not use copy-in/copy-out for variables that have their
1149 if (TREE_ADDRESSABLE (decl
))
1152 /* lower_send_shared_vars only uses copy-in, but not copy-out
1154 if (TREE_READONLY (decl
)
1155 || ((TREE_CODE (decl
) == RESULT_DECL
1156 || TREE_CODE (decl
) == PARM_DECL
)
1157 && DECL_BY_REFERENCE (decl
)))
1160 /* Disallow copy-in/out in nested parallel if
1161 decl is shared in outer parallel, otherwise
1162 each thread could store the shared variable
1163 in its own copy-in location, making the
1164 variable no longer really shared. */
1165 if (shared_ctx
->is_nested
)
1169 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1170 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1177 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1178 c
; c
= OMP_CLAUSE_CHAIN (c
))
1179 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1180 && OMP_CLAUSE_DECL (c
) == decl
)
1184 goto maybe_mark_addressable_and_ret
;
1188 /* For tasks avoid using copy-in/out. As tasks can be
1189 deferred or executed in different thread, when GOMP_task
1190 returns, the task hasn't necessarily terminated. */
1191 if (is_task_ctx (shared_ctx
))
1194 maybe_mark_addressable_and_ret
:
1195 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1196 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1198 /* Taking address of OUTER in lower_send_shared_vars
1199 might need regimplification of everything that uses the
1201 if (!task_shared_vars
)
1202 task_shared_vars
= BITMAP_ALLOC (NULL
);
1203 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1204 TREE_ADDRESSABLE (outer
) = 1;
1213 /* Construct a new automatic decl similar to VAR. */
1216 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1218 tree copy
= copy_var_decl (var
, name
, type
);
1220 DECL_CONTEXT (copy
) = current_function_decl
;
1221 DECL_CHAIN (copy
) = ctx
->block_vars
;
1222 /* If VAR is listed in task_shared_vars, it means it wasn't
1223 originally addressable and is just because task needs to take
1224 it's address. But we don't need to take address of privatizations
1226 if (TREE_ADDRESSABLE (var
)
1228 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1229 TREE_ADDRESSABLE (copy
) = 0;
1230 ctx
->block_vars
= copy
;
1236 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1238 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1241 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1244 omp_build_component_ref (tree obj
, tree field
)
1246 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1247 if (TREE_THIS_VOLATILE (field
))
1248 TREE_THIS_VOLATILE (ret
) |= 1;
1249 if (TREE_READONLY (field
))
1250 TREE_READONLY (ret
) |= 1;
1254 /* Build tree nodes to access the field for VAR on the receiver side. */
1257 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1259 tree x
, field
= lookup_field (var
, ctx
);
1261 /* If the receiver record type was remapped in the child function,
1262 remap the field into the new record type. */
1263 x
= maybe_lookup_field (field
, ctx
);
1267 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1268 TREE_THIS_NOTRAP (x
) = 1;
1269 x
= omp_build_component_ref (x
, field
);
1272 x
= build_simple_mem_ref (x
);
1273 TREE_THIS_NOTRAP (x
) = 1;
1279 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1280 of a parallel, this is a component reference; for workshare constructs
1281 this is some variable. */
1284 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1288 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1290 else if (is_variable_sized (var
))
1292 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1293 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1294 x
= build_simple_mem_ref (x
);
1296 else if (is_taskreg_ctx (ctx
))
1298 bool by_ref
= use_pointer_for_field (var
, NULL
);
1299 x
= build_receiver_ref (var
, by_ref
, ctx
);
1301 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1302 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1304 /* #pragma omp simd isn't a worksharing construct, and can reference even
1305 private vars in its linear etc. clauses. */
1307 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1308 x
= lookup_decl (var
, ctx
->outer
);
1309 else if (ctx
->outer
)
1310 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1314 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1316 gcc_assert (ctx
->outer
);
1318 = splay_tree_lookup (ctx
->outer
->field_map
,
1319 (splay_tree_key
) &DECL_UID (var
));
1322 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1325 x
= lookup_decl (var
, ctx
->outer
);
1329 tree field
= (tree
) n
->value
;
1330 /* If the receiver record type was remapped in the child function,
1331 remap the field into the new record type. */
1332 x
= maybe_lookup_field (field
, ctx
->outer
);
1336 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1337 x
= omp_build_component_ref (x
, field
);
1338 if (use_pointer_for_field (var
, ctx
->outer
))
1339 x
= build_simple_mem_ref (x
);
1342 else if (ctx
->outer
)
1344 omp_context
*outer
= ctx
->outer
;
1345 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
1347 outer
= outer
->outer
;
1349 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
1351 x
= lookup_decl (var
, outer
);
1353 else if (is_reference (var
))
1354 /* This can happen with orphaned constructs. If var is reference, it is
1355 possible it is shared and as such valid. */
1357 else if (omp_member_access_dummy_var (var
))
1364 tree t
= omp_member_access_dummy_var (var
);
1367 x
= DECL_VALUE_EXPR (var
);
1368 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1370 x
= unshare_and_remap (x
, t
, o
);
1372 x
= unshare_expr (x
);
1376 if (is_reference (var
))
1377 x
= build_simple_mem_ref (x
);
1382 /* Build tree nodes to access the field for VAR on the sender side. */
1385 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1387 tree field
= lookup_sfield (key
, ctx
);
1388 return omp_build_component_ref (ctx
->sender_decl
, field
);
1392 build_sender_ref (tree var
, omp_context
*ctx
)
1394 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1397 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1398 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1401 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
,
1402 bool base_pointers_restrict
= false)
1404 tree field
, type
, sfield
= NULL_TREE
;
1405 splay_tree_key key
= (splay_tree_key
) var
;
1407 if ((mask
& 8) != 0)
1409 key
= (splay_tree_key
) &DECL_UID (var
);
1410 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1412 gcc_assert ((mask
& 1) == 0
1413 || !splay_tree_lookup (ctx
->field_map
, key
));
1414 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1415 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1416 gcc_assert ((mask
& 3) == 3
1417 || !is_gimple_omp_oacc (ctx
->stmt
));
1419 type
= TREE_TYPE (var
);
1420 /* Prevent redeclaring the var in the split-off function with a restrict
1421 pointer type. Note that we only clear type itself, restrict qualifiers in
1422 the pointed-to type will be ignored by points-to analysis. */
1423 if (POINTER_TYPE_P (type
)
1424 && TYPE_RESTRICT (type
))
1425 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
1429 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1430 type
= build_pointer_type (build_pointer_type (type
));
1434 type
= build_pointer_type (type
);
1435 if (base_pointers_restrict
)
1436 type
= build_qualified_type (type
, TYPE_QUAL_RESTRICT
);
1438 else if ((mask
& 3) == 1 && is_reference (var
))
1439 type
= TREE_TYPE (type
);
1441 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1442 FIELD_DECL
, DECL_NAME (var
), type
);
1444 /* Remember what variable this field was created for. This does have a
1445 side effect of making dwarf2out ignore this member, so for helpful
1446 debugging we clear it later in delete_omp_context. */
1447 DECL_ABSTRACT_ORIGIN (field
) = var
;
1448 if (type
== TREE_TYPE (var
))
1450 SET_DECL_ALIGN (field
, DECL_ALIGN (var
));
1451 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1452 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1455 SET_DECL_ALIGN (field
, TYPE_ALIGN (type
));
1457 if ((mask
& 3) == 3)
1459 insert_field_into_struct (ctx
->record_type
, field
);
1460 if (ctx
->srecord_type
)
1462 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1463 FIELD_DECL
, DECL_NAME (var
), type
);
1464 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1465 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
1466 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1467 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1468 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1473 if (ctx
->srecord_type
== NULL_TREE
)
1477 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1478 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1479 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1481 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1482 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1483 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1484 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1485 splay_tree_insert (ctx
->sfield_map
,
1486 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1487 (splay_tree_value
) sfield
);
1491 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1492 : ctx
->srecord_type
, field
);
1496 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1497 if ((mask
& 2) && ctx
->sfield_map
)
1498 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1502 install_var_local (tree var
, omp_context
*ctx
)
1504 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1505 insert_decl_map (&ctx
->cb
, var
, new_var
);
1509 /* Adjust the replacement for DECL in CTX for the new context. This means
1510 copying the DECL_VALUE_EXPR, and fixing up the type. */
1513 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1515 tree new_decl
, size
;
1517 new_decl
= lookup_decl (decl
, ctx
);
1519 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1521 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1522 && DECL_HAS_VALUE_EXPR_P (decl
))
1524 tree ve
= DECL_VALUE_EXPR (decl
);
1525 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1526 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1527 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1530 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1532 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1533 if (size
== error_mark_node
)
1534 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1535 DECL_SIZE (new_decl
) = size
;
1537 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1538 if (size
== error_mark_node
)
1539 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1540 DECL_SIZE_UNIT (new_decl
) = size
;
1544 /* The callback for remap_decl. Search all containing contexts for a
1545 mapping of the variable; this avoids having to duplicate the splay
1546 tree ahead of time. We know a mapping doesn't already exist in the
1547 given context. Create new mappings to implement default semantics. */
1550 omp_copy_decl (tree var
, copy_body_data
*cb
)
1552 omp_context
*ctx
= (omp_context
*) cb
;
1555 if (TREE_CODE (var
) == LABEL_DECL
)
1557 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1558 DECL_CONTEXT (new_var
) = current_function_decl
;
1559 insert_decl_map (&ctx
->cb
, var
, new_var
);
1563 while (!is_taskreg_ctx (ctx
))
1568 new_var
= maybe_lookup_decl (var
, ctx
);
1573 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1576 return error_mark_node
;
1580 /* Debugging dumps for parallel regions. */
1581 void dump_omp_region (FILE *, struct omp_region
*, int);
1582 void debug_omp_region (struct omp_region
*);
1583 void debug_all_omp_regions (void);
1585 /* Dump the parallel region tree rooted at REGION. */
1588 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1590 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1591 gimple_code_name
[region
->type
]);
1594 dump_omp_region (file
, region
->inner
, indent
+ 4);
1598 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1599 region
->cont
->index
);
1603 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1604 region
->exit
->index
);
1606 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1609 dump_omp_region (file
, region
->next
, indent
);
1613 debug_omp_region (struct omp_region
*region
)
1615 dump_omp_region (stderr
, region
, 0);
1619 debug_all_omp_regions (void)
1621 dump_omp_region (stderr
, root_omp_region
, 0);
1625 /* Create a new parallel region starting at STMT inside region PARENT. */
1627 static struct omp_region
*
1628 new_omp_region (basic_block bb
, enum gimple_code type
,
1629 struct omp_region
*parent
)
1631 struct omp_region
*region
= XCNEW (struct omp_region
);
1633 region
->outer
= parent
;
1635 region
->type
= type
;
1639 /* This is a nested region. Add it to the list of inner
1640 regions in PARENT. */
1641 region
->next
= parent
->inner
;
1642 parent
->inner
= region
;
1646 /* This is a toplevel region. Add it to the list of toplevel
1647 regions in ROOT_OMP_REGION. */
1648 region
->next
= root_omp_region
;
1649 root_omp_region
= region
;
1655 /* Release the memory associated with the region tree rooted at REGION. */
1658 free_omp_region_1 (struct omp_region
*region
)
1660 struct omp_region
*i
, *n
;
1662 for (i
= region
->inner
; i
; i
= n
)
1665 free_omp_region_1 (i
);
1671 /* Release the memory for the entire omp region tree. */
1674 free_omp_regions (void)
1676 struct omp_region
*r
, *n
;
1677 for (r
= root_omp_region
; r
; r
= n
)
1680 free_omp_region_1 (r
);
1682 root_omp_region
= NULL
;
1686 /* Create a new context, with OUTER_CTX being the surrounding context. */
1688 static omp_context
*
1689 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1691 omp_context
*ctx
= XCNEW (omp_context
);
1693 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1694 (splay_tree_value
) ctx
);
1699 ctx
->outer
= outer_ctx
;
1700 ctx
->cb
= outer_ctx
->cb
;
1701 ctx
->cb
.block
= NULL
;
1702 ctx
->depth
= outer_ctx
->depth
+ 1;
1706 ctx
->cb
.src_fn
= current_function_decl
;
1707 ctx
->cb
.dst_fn
= current_function_decl
;
1708 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1709 gcc_checking_assert (ctx
->cb
.src_node
);
1710 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1711 ctx
->cb
.src_cfun
= cfun
;
1712 ctx
->cb
.copy_decl
= omp_copy_decl
;
1713 ctx
->cb
.eh_lp_nr
= 0;
1714 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1718 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1723 static gimple_seq
maybe_catch_exception (gimple_seq
);
1725 /* Finalize task copyfn. */
1728 finalize_task_copyfn (gomp_task
*task_stmt
)
1730 struct function
*child_cfun
;
1732 gimple_seq seq
= NULL
, new_seq
;
1735 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1736 if (child_fn
== NULL_TREE
)
1739 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1740 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1742 push_cfun (child_cfun
);
1743 bind
= gimplify_body (child_fn
, false);
1744 gimple_seq_add_stmt (&seq
, bind
);
1745 new_seq
= maybe_catch_exception (seq
);
1748 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1750 gimple_seq_add_stmt (&seq
, bind
);
1752 gimple_set_body (child_fn
, seq
);
1755 /* Inform the callgraph about the new function. */
1756 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1757 node
->parallelized_function
= 1;
1758 cgraph_node::add_new_function (child_fn
, false);
1761 /* Destroy a omp_context data structures. Called through the splay tree
1762 value delete callback. */
1765 delete_omp_context (splay_tree_value value
)
1767 omp_context
*ctx
= (omp_context
*) value
;
1769 delete ctx
->cb
.decl_map
;
1772 splay_tree_delete (ctx
->field_map
);
1773 if (ctx
->sfield_map
)
1774 splay_tree_delete (ctx
->sfield_map
);
1776 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1777 it produces corrupt debug information. */
1778 if (ctx
->record_type
)
1781 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1782 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1784 if (ctx
->srecord_type
)
1787 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1788 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1791 if (is_task_ctx (ctx
))
1792 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1797 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1801 fixup_child_record_type (omp_context
*ctx
)
1803 tree f
, type
= ctx
->record_type
;
1805 if (!ctx
->receiver_decl
)
1807 /* ??? It isn't sufficient to just call remap_type here, because
1808 variably_modified_type_p doesn't work the way we expect for
1809 record types. Testing each field for whether it needs remapping
1810 and creating a new record by hand works, however. */
1811 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1812 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1816 tree name
, new_fields
= NULL
;
1818 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1819 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1820 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1821 TYPE_DECL
, name
, type
);
1822 TYPE_NAME (type
) = name
;
1824 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1826 tree new_f
= copy_node (f
);
1827 DECL_CONTEXT (new_f
) = type
;
1828 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1829 DECL_CHAIN (new_f
) = new_fields
;
1830 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1831 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1833 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1837 /* Arrange to be able to look up the receiver field
1838 given the sender field. */
1839 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1840 (splay_tree_value
) new_f
);
1842 TYPE_FIELDS (type
) = nreverse (new_fields
);
1846 /* In a target region we never modify any of the pointers in *.omp_data_i,
1847 so attempt to help the optimizers. */
1848 if (is_gimple_omp_offloaded (ctx
->stmt
))
1849 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1851 TREE_TYPE (ctx
->receiver_decl
)
1852 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1855 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1856 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1860 scan_sharing_clauses (tree clauses
, omp_context
*ctx
,
1861 bool base_pointers_restrict
= false)
1864 bool scan_array_reductions
= false;
1866 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1870 switch (OMP_CLAUSE_CODE (c
))
1872 case OMP_CLAUSE_PRIVATE
:
1873 decl
= OMP_CLAUSE_DECL (c
);
1874 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1876 else if (!is_variable_sized (decl
))
1877 install_var_local (decl
, ctx
);
1880 case OMP_CLAUSE_SHARED
:
1881 decl
= OMP_CLAUSE_DECL (c
);
1882 /* Ignore shared directives in teams construct. */
1883 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1885 /* Global variables don't need to be copied,
1886 the receiver side will use them directly. */
1887 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1888 if (is_global_var (odecl
))
1890 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1893 gcc_assert (is_taskreg_ctx (ctx
));
1894 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1895 || !is_variable_sized (decl
));
1896 /* Global variables don't need to be copied,
1897 the receiver side will use them directly. */
1898 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1900 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1902 use_pointer_for_field (decl
, ctx
);
1905 by_ref
= use_pointer_for_field (decl
, NULL
);
1906 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1907 || TREE_ADDRESSABLE (decl
)
1909 || is_reference (decl
))
1911 by_ref
= use_pointer_for_field (decl
, ctx
);
1912 install_var_field (decl
, by_ref
, 3, ctx
);
1913 install_var_local (decl
, ctx
);
1916 /* We don't need to copy const scalar vars back. */
1917 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1920 case OMP_CLAUSE_REDUCTION
:
1921 decl
= OMP_CLAUSE_DECL (c
);
1922 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1923 && TREE_CODE (decl
) == MEM_REF
)
1925 tree t
= TREE_OPERAND (decl
, 0);
1926 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1927 t
= TREE_OPERAND (t
, 0);
1928 if (TREE_CODE (t
) == INDIRECT_REF
1929 || TREE_CODE (t
) == ADDR_EXPR
)
1930 t
= TREE_OPERAND (t
, 0);
1931 install_var_local (t
, ctx
);
1932 if (is_taskreg_ctx (ctx
)
1933 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1934 && !is_variable_sized (t
))
1936 by_ref
= use_pointer_for_field (t
, ctx
);
1937 install_var_field (t
, by_ref
, 3, ctx
);
1943 case OMP_CLAUSE_LASTPRIVATE
:
1944 /* Let the corresponding firstprivate clause create
1946 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1950 case OMP_CLAUSE_FIRSTPRIVATE
:
1951 case OMP_CLAUSE_LINEAR
:
1952 decl
= OMP_CLAUSE_DECL (c
);
1954 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1955 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1956 && is_gimple_omp_offloaded (ctx
->stmt
))
1958 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1959 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1960 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1961 install_var_field (decl
, true, 3, ctx
);
1963 install_var_field (decl
, false, 3, ctx
);
1965 if (is_variable_sized (decl
))
1967 if (is_task_ctx (ctx
))
1968 install_var_field (decl
, false, 1, ctx
);
1971 else if (is_taskreg_ctx (ctx
))
1974 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1975 by_ref
= use_pointer_for_field (decl
, NULL
);
1977 if (is_task_ctx (ctx
)
1978 && (global
|| by_ref
|| is_reference (decl
)))
1980 install_var_field (decl
, false, 1, ctx
);
1982 install_var_field (decl
, by_ref
, 2, ctx
);
1985 install_var_field (decl
, by_ref
, 3, ctx
);
1987 install_var_local (decl
, ctx
);
1990 case OMP_CLAUSE_USE_DEVICE_PTR
:
1991 decl
= OMP_CLAUSE_DECL (c
);
1992 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1993 install_var_field (decl
, true, 3, ctx
);
1995 install_var_field (decl
, false, 3, ctx
);
1996 if (DECL_SIZE (decl
)
1997 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1999 tree decl2
= DECL_VALUE_EXPR (decl
);
2000 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2001 decl2
= TREE_OPERAND (decl2
, 0);
2002 gcc_assert (DECL_P (decl2
));
2003 install_var_local (decl2
, ctx
);
2005 install_var_local (decl
, ctx
);
2008 case OMP_CLAUSE_IS_DEVICE_PTR
:
2009 decl
= OMP_CLAUSE_DECL (c
);
2012 case OMP_CLAUSE__LOOPTEMP_
:
2013 gcc_assert (is_taskreg_ctx (ctx
));
2014 decl
= OMP_CLAUSE_DECL (c
);
2015 install_var_field (decl
, false, 3, ctx
);
2016 install_var_local (decl
, ctx
);
2019 case OMP_CLAUSE_COPYPRIVATE
:
2020 case OMP_CLAUSE_COPYIN
:
2021 decl
= OMP_CLAUSE_DECL (c
);
2022 by_ref
= use_pointer_for_field (decl
, NULL
);
2023 install_var_field (decl
, by_ref
, 3, ctx
);
2026 case OMP_CLAUSE_DEFAULT
:
2027 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2030 case OMP_CLAUSE_FINAL
:
2032 case OMP_CLAUSE_NUM_THREADS
:
2033 case OMP_CLAUSE_NUM_TEAMS
:
2034 case OMP_CLAUSE_THREAD_LIMIT
:
2035 case OMP_CLAUSE_DEVICE
:
2036 case OMP_CLAUSE_SCHEDULE
:
2037 case OMP_CLAUSE_DIST_SCHEDULE
:
2038 case OMP_CLAUSE_DEPEND
:
2039 case OMP_CLAUSE_PRIORITY
:
2040 case OMP_CLAUSE_GRAINSIZE
:
2041 case OMP_CLAUSE_NUM_TASKS
:
2042 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2043 case OMP_CLAUSE_NUM_GANGS
:
2044 case OMP_CLAUSE_NUM_WORKERS
:
2045 case OMP_CLAUSE_VECTOR_LENGTH
:
2047 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2051 case OMP_CLAUSE_FROM
:
2052 case OMP_CLAUSE_MAP
:
2054 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2055 decl
= OMP_CLAUSE_DECL (c
);
2056 /* Global variables with "omp declare target" attribute
2057 don't need to be copied, the receiver side will use them
2058 directly. However, global variables with "omp declare target link"
2059 attribute need to be copied. */
2060 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2062 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2063 && (OMP_CLAUSE_MAP_KIND (c
)
2064 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2065 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2066 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2067 && varpool_node::get_create (decl
)->offloadable
2068 && !lookup_attribute ("omp declare target link",
2069 DECL_ATTRIBUTES (decl
)))
2071 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2072 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2074 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2075 not offloaded; there is nothing to map for those. */
2076 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2077 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2078 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2081 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2082 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2083 || (OMP_CLAUSE_MAP_KIND (c
)
2084 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2086 if (TREE_CODE (decl
) == COMPONENT_REF
2087 || (TREE_CODE (decl
) == INDIRECT_REF
2088 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2089 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2090 == REFERENCE_TYPE
)))
2092 if (DECL_SIZE (decl
)
2093 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2095 tree decl2
= DECL_VALUE_EXPR (decl
);
2096 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2097 decl2
= TREE_OPERAND (decl2
, 0);
2098 gcc_assert (DECL_P (decl2
));
2099 install_var_local (decl2
, ctx
);
2101 install_var_local (decl
, ctx
);
2106 if (DECL_SIZE (decl
)
2107 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2109 tree decl2
= DECL_VALUE_EXPR (decl
);
2110 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2111 decl2
= TREE_OPERAND (decl2
, 0);
2112 gcc_assert (DECL_P (decl2
));
2113 install_var_field (decl2
, true, 3, ctx
);
2114 install_var_local (decl2
, ctx
);
2115 install_var_local (decl
, ctx
);
2119 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2120 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2121 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2122 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2123 install_var_field (decl
, true, 7, ctx
);
2125 install_var_field (decl
, true, 3, ctx
,
2126 base_pointers_restrict
);
2127 if (is_gimple_omp_offloaded (ctx
->stmt
)
2128 && !OMP_CLAUSE_MAP_IN_REDUCTION (c
))
2129 install_var_local (decl
, ctx
);
2134 tree base
= get_base_address (decl
);
2135 tree nc
= OMP_CLAUSE_CHAIN (c
);
2138 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2139 && OMP_CLAUSE_DECL (nc
) == base
2140 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2141 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2143 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2144 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2150 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2151 decl
= OMP_CLAUSE_DECL (c
);
2153 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2154 (splay_tree_key
) decl
));
2156 = build_decl (OMP_CLAUSE_LOCATION (c
),
2157 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2158 SET_DECL_ALIGN (field
, TYPE_ALIGN (ptr_type_node
));
2159 insert_field_into_struct (ctx
->record_type
, field
);
2160 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2161 (splay_tree_value
) field
);
2166 case OMP_CLAUSE__GRIDDIM_
:
2169 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
2170 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
2174 case OMP_CLAUSE_NOWAIT
:
2175 case OMP_CLAUSE_ORDERED
:
2176 case OMP_CLAUSE_COLLAPSE
:
2177 case OMP_CLAUSE_UNTIED
:
2178 case OMP_CLAUSE_MERGEABLE
:
2179 case OMP_CLAUSE_PROC_BIND
:
2180 case OMP_CLAUSE_SAFELEN
:
2181 case OMP_CLAUSE_SIMDLEN
:
2182 case OMP_CLAUSE_THREADS
:
2183 case OMP_CLAUSE_SIMD
:
2184 case OMP_CLAUSE_NOGROUP
:
2185 case OMP_CLAUSE_DEFAULTMAP
:
2186 case OMP_CLAUSE_ASYNC
:
2187 case OMP_CLAUSE_WAIT
:
2188 case OMP_CLAUSE_GANG
:
2189 case OMP_CLAUSE_WORKER
:
2190 case OMP_CLAUSE_VECTOR
:
2191 case OMP_CLAUSE_INDEPENDENT
:
2192 case OMP_CLAUSE_AUTO
:
2193 case OMP_CLAUSE_SEQ
:
2196 case OMP_CLAUSE_ALIGNED
:
2197 decl
= OMP_CLAUSE_DECL (c
);
2198 if (is_global_var (decl
)
2199 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2200 install_var_local (decl
, ctx
);
2203 case OMP_CLAUSE_TILE
:
2204 case OMP_CLAUSE__CACHE_
:
2210 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2212 switch (OMP_CLAUSE_CODE (c
))
2214 case OMP_CLAUSE_LASTPRIVATE
:
2215 /* Let the corresponding firstprivate clause create
2217 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2218 scan_array_reductions
= true;
2219 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2223 case OMP_CLAUSE_FIRSTPRIVATE
:
2224 case OMP_CLAUSE_PRIVATE
:
2225 case OMP_CLAUSE_LINEAR
:
2226 case OMP_CLAUSE_IS_DEVICE_PTR
:
2227 decl
= OMP_CLAUSE_DECL (c
);
2228 if (is_variable_sized (decl
))
2230 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2231 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2232 && is_gimple_omp_offloaded (ctx
->stmt
))
2234 tree decl2
= DECL_VALUE_EXPR (decl
);
2235 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2236 decl2
= TREE_OPERAND (decl2
, 0);
2237 gcc_assert (DECL_P (decl2
));
2238 install_var_local (decl2
, ctx
);
2239 fixup_remapped_decl (decl2
, ctx
, false);
2241 install_var_local (decl
, ctx
);
2243 fixup_remapped_decl (decl
, ctx
,
2244 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2245 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2246 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2247 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2248 scan_array_reductions
= true;
2251 case OMP_CLAUSE_REDUCTION
:
2252 decl
= OMP_CLAUSE_DECL (c
);
2253 if (TREE_CODE (decl
) != MEM_REF
)
2255 if (is_variable_sized (decl
))
2256 install_var_local (decl
, ctx
);
2257 fixup_remapped_decl (decl
, ctx
, false);
2259 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2260 scan_array_reductions
= true;
2263 case OMP_CLAUSE_SHARED
:
2264 /* Ignore shared directives in teams construct. */
2265 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2267 decl
= OMP_CLAUSE_DECL (c
);
2268 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2270 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2272 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2275 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2276 install_var_field (decl
, by_ref
, 11, ctx
);
2279 fixup_remapped_decl (decl
, ctx
, false);
2282 case OMP_CLAUSE_MAP
:
2283 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2285 decl
= OMP_CLAUSE_DECL (c
);
2287 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2288 && (OMP_CLAUSE_MAP_KIND (c
)
2289 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2290 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2291 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2292 && varpool_node::get_create (decl
)->offloadable
)
2296 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2297 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2298 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2299 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2301 tree new_decl
= lookup_decl (decl
, ctx
);
2302 TREE_TYPE (new_decl
)
2303 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2305 else if (DECL_SIZE (decl
)
2306 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2308 tree decl2
= DECL_VALUE_EXPR (decl
);
2309 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2310 decl2
= TREE_OPERAND (decl2
, 0);
2311 gcc_assert (DECL_P (decl2
));
2312 fixup_remapped_decl (decl2
, ctx
, false);
2313 fixup_remapped_decl (decl
, ctx
, true);
2316 fixup_remapped_decl (decl
, ctx
, false);
2320 case OMP_CLAUSE_COPYPRIVATE
:
2321 case OMP_CLAUSE_COPYIN
:
2322 case OMP_CLAUSE_DEFAULT
:
2324 case OMP_CLAUSE_NUM_THREADS
:
2325 case OMP_CLAUSE_NUM_TEAMS
:
2326 case OMP_CLAUSE_THREAD_LIMIT
:
2327 case OMP_CLAUSE_DEVICE
:
2328 case OMP_CLAUSE_SCHEDULE
:
2329 case OMP_CLAUSE_DIST_SCHEDULE
:
2330 case OMP_CLAUSE_NOWAIT
:
2331 case OMP_CLAUSE_ORDERED
:
2332 case OMP_CLAUSE_COLLAPSE
:
2333 case OMP_CLAUSE_UNTIED
:
2334 case OMP_CLAUSE_FINAL
:
2335 case OMP_CLAUSE_MERGEABLE
:
2336 case OMP_CLAUSE_PROC_BIND
:
2337 case OMP_CLAUSE_SAFELEN
:
2338 case OMP_CLAUSE_SIMDLEN
:
2339 case OMP_CLAUSE_ALIGNED
:
2340 case OMP_CLAUSE_DEPEND
:
2341 case OMP_CLAUSE__LOOPTEMP_
:
2343 case OMP_CLAUSE_FROM
:
2344 case OMP_CLAUSE_PRIORITY
:
2345 case OMP_CLAUSE_GRAINSIZE
:
2346 case OMP_CLAUSE_NUM_TASKS
:
2347 case OMP_CLAUSE_THREADS
:
2348 case OMP_CLAUSE_SIMD
:
2349 case OMP_CLAUSE_NOGROUP
:
2350 case OMP_CLAUSE_DEFAULTMAP
:
2351 case OMP_CLAUSE_USE_DEVICE_PTR
:
2352 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2353 case OMP_CLAUSE_ASYNC
:
2354 case OMP_CLAUSE_WAIT
:
2355 case OMP_CLAUSE_NUM_GANGS
:
2356 case OMP_CLAUSE_NUM_WORKERS
:
2357 case OMP_CLAUSE_VECTOR_LENGTH
:
2358 case OMP_CLAUSE_GANG
:
2359 case OMP_CLAUSE_WORKER
:
2360 case OMP_CLAUSE_VECTOR
:
2361 case OMP_CLAUSE_INDEPENDENT
:
2362 case OMP_CLAUSE_AUTO
:
2363 case OMP_CLAUSE_SEQ
:
2364 case OMP_CLAUSE__GRIDDIM_
:
2367 case OMP_CLAUSE_TILE
:
2368 case OMP_CLAUSE__CACHE_
:
2374 gcc_checking_assert (!scan_array_reductions
2375 || !is_gimple_omp_oacc (ctx
->stmt
));
2376 if (scan_array_reductions
)
2378 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2379 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2380 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2382 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2383 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2385 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2386 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2387 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2388 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2389 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2390 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2394 /* Create a new name for omp child function. Returns an identifier. If
2395 IS_CILK_FOR is true then the suffix for the child function is
2399 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2402 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2403 return clone_function_name (current_function_decl
,
2404 task_copy
? "_omp_cpyfn" : "_omp_fn");
2407 /* Returns the type of the induction variable for the child function for
2408 _Cilk_for and the types for _high and _low variables based on TYPE. */
2411 cilk_for_check_loop_diff_type (tree type
)
2413 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2415 if (TYPE_UNSIGNED (type
))
2416 return uint32_type_node
;
2418 return integer_type_node
;
2422 if (TYPE_UNSIGNED (type
))
2423 return uint64_type_node
;
2425 return long_long_integer_type_node
;
2429 /* Build a decl for the omp child function. It'll not contain a body
2430 yet, just the bare decl. */
2433 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2435 tree decl
, type
, name
, t
;
2438 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2439 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2440 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2441 tree cilk_var_type
= NULL_TREE
;
2443 name
= create_omp_child_function_name (task_copy
,
2444 cilk_for_count
!= NULL_TREE
);
2446 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2447 ptr_type_node
, NULL_TREE
);
2448 else if (cilk_for_count
)
2450 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2451 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2452 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2453 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2456 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2458 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2460 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2463 ctx
->cb
.dst_fn
= decl
;
2465 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2467 TREE_STATIC (decl
) = 1;
2468 TREE_USED (decl
) = 1;
2469 DECL_ARTIFICIAL (decl
) = 1;
2470 DECL_IGNORED_P (decl
) = 0;
2471 TREE_PUBLIC (decl
) = 0;
2472 DECL_UNINLINABLE (decl
) = 1;
2473 DECL_EXTERNAL (decl
) = 0;
2474 DECL_CONTEXT (decl
) = NULL_TREE
;
2475 DECL_INITIAL (decl
) = make_node (BLOCK
);
2476 BLOCK_SUPERCONTEXT (DECL_INITIAL (decl
)) = decl
;
2477 if (cgraph_node::get (current_function_decl
)->offloadable
)
2478 cgraph_node::get_create (decl
)->offloadable
= 1;
2482 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2483 if (is_gimple_omp_offloaded (octx
->stmt
))
2485 cgraph_node::get_create (decl
)->offloadable
= 1;
2486 if (ENABLE_OFFLOADING
)
2487 g
->have_offload
= true;
2493 if (cgraph_node::get_create (decl
)->offloadable
2494 && !lookup_attribute ("omp declare target",
2495 DECL_ATTRIBUTES (current_function_decl
)))
2496 DECL_ATTRIBUTES (decl
)
2497 = tree_cons (get_identifier ("omp target entrypoint"),
2498 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2500 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2501 RESULT_DECL
, NULL_TREE
, void_type_node
);
2502 DECL_ARTIFICIAL (t
) = 1;
2503 DECL_IGNORED_P (t
) = 1;
2504 DECL_CONTEXT (t
) = decl
;
2505 DECL_RESULT (decl
) = t
;
2507 /* _Cilk_for's child function requires two extra parameters called
2508 __low and __high that are set the by Cilk runtime when it calls this
2512 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2513 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2514 DECL_ARTIFICIAL (t
) = 1;
2515 DECL_NAMELESS (t
) = 1;
2516 DECL_ARG_TYPE (t
) = ptr_type_node
;
2517 DECL_CONTEXT (t
) = current_function_decl
;
2519 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2520 DECL_ARGUMENTS (decl
) = t
;
2522 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2523 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2524 DECL_ARTIFICIAL (t
) = 1;
2525 DECL_NAMELESS (t
) = 1;
2526 DECL_ARG_TYPE (t
) = ptr_type_node
;
2527 DECL_CONTEXT (t
) = current_function_decl
;
2529 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2530 DECL_ARGUMENTS (decl
) = t
;
2533 tree data_name
= get_identifier (".omp_data_i");
2534 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2536 DECL_ARTIFICIAL (t
) = 1;
2537 DECL_NAMELESS (t
) = 1;
2538 DECL_ARG_TYPE (t
) = ptr_type_node
;
2539 DECL_CONTEXT (t
) = current_function_decl
;
2541 TREE_READONLY (t
) = 1;
2543 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2544 DECL_ARGUMENTS (decl
) = t
;
2546 ctx
->receiver_decl
= t
;
2549 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2550 PARM_DECL
, get_identifier (".omp_data_o"),
2552 DECL_ARTIFICIAL (t
) = 1;
2553 DECL_NAMELESS (t
) = 1;
2554 DECL_ARG_TYPE (t
) = ptr_type_node
;
2555 DECL_CONTEXT (t
) = current_function_decl
;
2557 TREE_ADDRESSABLE (t
) = 1;
2558 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2559 DECL_ARGUMENTS (decl
) = t
;
2562 /* Allocate memory for the function structure. The call to
2563 allocate_struct_function clobbers CFUN, so we need to restore
2565 push_struct_function (decl
);
2566 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2567 init_tree_ssa (cfun
);
2571 /* Callback for walk_gimple_seq. Check if combined parallel
2572 contains gimple_omp_for_combined_into_p OMP_FOR. */
2575 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2576 bool *handled_ops_p
,
2577 struct walk_stmt_info
*wi
)
2579 gimple
*stmt
= gsi_stmt (*gsi_p
);
2581 *handled_ops_p
= true;
2582 switch (gimple_code (stmt
))
2586 case GIMPLE_OMP_FOR
:
2587 if (gimple_omp_for_combined_into_p (stmt
)
2588 && gimple_omp_for_kind (stmt
)
2589 == *(const enum gf_mask
*) (wi
->info
))
2592 return integer_zero_node
;
2601 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2604 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2605 omp_context
*outer_ctx
)
2607 struct walk_stmt_info wi
;
2609 memset (&wi
, 0, sizeof (wi
));
2611 wi
.info
= (void *) &msk
;
2612 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2613 if (wi
.info
!= (void *) &msk
)
2615 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2616 struct omp_for_data fd
;
2617 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2618 /* We need two temporaries with fd.loop.v type (istart/iend)
2619 and then (fd.collapse - 1) temporaries with the same
2620 type for count2 ... countN-1 vars if not constant. */
2621 size_t count
= 2, i
;
2622 tree type
= fd
.iter_type
;
2624 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2626 count
+= fd
.collapse
- 1;
2627 /* If there are lastprivate clauses on the inner
2628 GIMPLE_OMP_FOR, add one more temporaries for the total number
2629 of iterations (product of count1 ... countN-1). */
2630 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2631 OMP_CLAUSE_LASTPRIVATE
))
2633 else if (msk
== GF_OMP_FOR_KIND_FOR
2634 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2635 OMP_CLAUSE_LASTPRIVATE
))
2638 for (i
= 0; i
< count
; i
++)
2640 tree temp
= create_tmp_var (type
);
2641 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2642 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2643 OMP_CLAUSE_DECL (c
) = temp
;
2644 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2645 gimple_omp_taskreg_set_clauses (stmt
, c
);
2650 /* Scan an OpenMP parallel directive. */
2653 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2657 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2659 /* Ignore parallel directives with empty bodies, unless there
2660 are copyin clauses. */
2662 && empty_body_p (gimple_omp_body (stmt
))
2663 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2664 OMP_CLAUSE_COPYIN
) == NULL
)
2666 gsi_replace (gsi
, gimple_build_nop (), false);
2670 if (gimple_omp_parallel_combined_p (stmt
))
2671 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2673 ctx
= new_omp_context (stmt
, outer_ctx
);
2674 taskreg_contexts
.safe_push (ctx
);
2675 if (taskreg_nesting_level
> 1)
2676 ctx
->is_nested
= true;
2677 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2678 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2679 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2680 name
= create_tmp_var_name (".omp_data_s");
2681 name
= build_decl (gimple_location (stmt
),
2682 TYPE_DECL
, name
, ctx
->record_type
);
2683 DECL_ARTIFICIAL (name
) = 1;
2684 DECL_NAMELESS (name
) = 1;
2685 TYPE_NAME (ctx
->record_type
) = name
;
2686 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2687 if (!gimple_omp_parallel_grid_phony (stmt
))
2689 create_omp_child_function (ctx
, false);
2690 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2693 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2694 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2696 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2697 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2700 /* Scan an OpenMP task directive. */
2703 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2707 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2709 /* Ignore task directives with empty bodies. */
2711 && empty_body_p (gimple_omp_body (stmt
)))
2713 gsi_replace (gsi
, gimple_build_nop (), false);
2717 if (gimple_omp_task_taskloop_p (stmt
))
2718 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2720 ctx
= new_omp_context (stmt
, outer_ctx
);
2721 taskreg_contexts
.safe_push (ctx
);
2722 if (taskreg_nesting_level
> 1)
2723 ctx
->is_nested
= true;
2724 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2725 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2726 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2727 name
= create_tmp_var_name (".omp_data_s");
2728 name
= build_decl (gimple_location (stmt
),
2729 TYPE_DECL
, name
, ctx
->record_type
);
2730 DECL_ARTIFICIAL (name
) = 1;
2731 DECL_NAMELESS (name
) = 1;
2732 TYPE_NAME (ctx
->record_type
) = name
;
2733 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2734 create_omp_child_function (ctx
, false);
2735 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2737 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2739 if (ctx
->srecord_type
)
2741 name
= create_tmp_var_name (".omp_data_a");
2742 name
= build_decl (gimple_location (stmt
),
2743 TYPE_DECL
, name
, ctx
->srecord_type
);
2744 DECL_ARTIFICIAL (name
) = 1;
2745 DECL_NAMELESS (name
) = 1;
2746 TYPE_NAME (ctx
->srecord_type
) = name
;
2747 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2748 create_omp_child_function (ctx
, true);
2751 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2753 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2755 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2756 t
= build_int_cst (long_integer_type_node
, 0);
2757 gimple_omp_task_set_arg_size (stmt
, t
);
2758 t
= build_int_cst (long_integer_type_node
, 1);
2759 gimple_omp_task_set_arg_align (stmt
, t
);
2764 /* If any decls have been made addressable during scan_omp,
2765 adjust their fields if needed, and layout record types
2766 of parallel/task constructs. */
2769 finish_taskreg_scan (omp_context
*ctx
)
2771 if (ctx
->record_type
== NULL_TREE
)
2774 /* If any task_shared_vars were needed, verify all
2775 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2776 statements if use_pointer_for_field hasn't changed
2777 because of that. If it did, update field types now. */
2778 if (task_shared_vars
)
2782 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2783 c
; c
= OMP_CLAUSE_CHAIN (c
))
2784 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2785 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2787 tree decl
= OMP_CLAUSE_DECL (c
);
2789 /* Global variables don't need to be copied,
2790 the receiver side will use them directly. */
2791 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2793 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2794 || !use_pointer_for_field (decl
, ctx
))
2796 tree field
= lookup_field (decl
, ctx
);
2797 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2798 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2800 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2801 TREE_THIS_VOLATILE (field
) = 0;
2802 DECL_USER_ALIGN (field
) = 0;
2803 SET_DECL_ALIGN (field
, TYPE_ALIGN (TREE_TYPE (field
)));
2804 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2805 SET_TYPE_ALIGN (ctx
->record_type
, DECL_ALIGN (field
));
2806 if (ctx
->srecord_type
)
2808 tree sfield
= lookup_sfield (decl
, ctx
);
2809 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2810 TREE_THIS_VOLATILE (sfield
) = 0;
2811 DECL_USER_ALIGN (sfield
) = 0;
2812 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
2813 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2814 SET_TYPE_ALIGN (ctx
->srecord_type
, DECL_ALIGN (sfield
));
2819 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2821 layout_type (ctx
->record_type
);
2822 fixup_child_record_type (ctx
);
2826 location_t loc
= gimple_location (ctx
->stmt
);
2827 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2828 /* Move VLA fields to the end. */
2829 p
= &TYPE_FIELDS (ctx
->record_type
);
2831 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2832 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2835 *p
= TREE_CHAIN (*p
);
2836 TREE_CHAIN (*q
) = NULL_TREE
;
2837 q
= &TREE_CHAIN (*q
);
2840 p
= &DECL_CHAIN (*p
);
2842 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2844 /* Move fields corresponding to first and second _looptemp_
2845 clause first. There are filled by GOMP_taskloop
2846 and thus need to be in specific positions. */
2847 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2848 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2849 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2850 OMP_CLAUSE__LOOPTEMP_
);
2851 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2852 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2853 p
= &TYPE_FIELDS (ctx
->record_type
);
2855 if (*p
== f1
|| *p
== f2
)
2856 *p
= DECL_CHAIN (*p
);
2858 p
= &DECL_CHAIN (*p
);
2859 DECL_CHAIN (f1
) = f2
;
2860 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2861 TYPE_FIELDS (ctx
->record_type
) = f1
;
2862 if (ctx
->srecord_type
)
2864 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2865 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2866 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2868 if (*p
== f1
|| *p
== f2
)
2869 *p
= DECL_CHAIN (*p
);
2871 p
= &DECL_CHAIN (*p
);
2872 DECL_CHAIN (f1
) = f2
;
2873 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2874 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2877 layout_type (ctx
->record_type
);
2878 fixup_child_record_type (ctx
);
2879 if (ctx
->srecord_type
)
2880 layout_type (ctx
->srecord_type
);
2881 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2882 TYPE_SIZE_UNIT (ctx
->record_type
));
2883 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2884 t
= build_int_cst (long_integer_type_node
,
2885 TYPE_ALIGN_UNIT (ctx
->record_type
));
2886 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2890 /* Find the enclosing offload context. */
2892 static omp_context
*
2893 enclosing_target_ctx (omp_context
*ctx
)
2895 for (; ctx
; ctx
= ctx
->outer
)
2896 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2902 /* Return true if ctx is part of an oacc kernels region. */
2905 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2907 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2909 gimple
*stmt
= ctx
->stmt
;
2910 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2911 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2918 /* Check the parallelism clauses inside a kernels regions.
2919 Until kernels handling moves to use the same loop indirection
2920 scheme as parallel, we need to do this checking early. */
2923 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2925 bool checking
= true;
2926 unsigned outer_mask
= 0;
2927 unsigned this_mask
= 0;
2928 bool has_seq
= false, has_auto
= false;
2931 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2935 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2937 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2940 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2942 switch (OMP_CLAUSE_CODE (c
))
2944 case OMP_CLAUSE_GANG
:
2945 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2947 case OMP_CLAUSE_WORKER
:
2948 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2950 case OMP_CLAUSE_VECTOR
:
2951 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2953 case OMP_CLAUSE_SEQ
:
2956 case OMP_CLAUSE_AUTO
:
2966 if (has_seq
&& (this_mask
|| has_auto
))
2967 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2968 " OpenACC loop specifiers");
2969 else if (has_auto
&& this_mask
)
2970 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2971 " OpenACC loop specifiers");
2973 if (this_mask
& outer_mask
)
2974 error_at (gimple_location (stmt
), "inner loop uses same"
2975 " OpenACC parallelism as containing loop");
2978 return outer_mask
| this_mask
;
2981 /* Scan a GIMPLE_OMP_FOR. */
2984 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2988 tree clauses
= gimple_omp_for_clauses (stmt
);
2990 ctx
= new_omp_context (stmt
, outer_ctx
);
2992 if (is_gimple_omp_oacc (stmt
))
2994 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2996 if (!tgt
|| is_oacc_parallel (tgt
))
2997 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2999 char const *check
= NULL
;
3001 switch (OMP_CLAUSE_CODE (c
))
3003 case OMP_CLAUSE_GANG
:
3007 case OMP_CLAUSE_WORKER
:
3011 case OMP_CLAUSE_VECTOR
:
3019 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3020 error_at (gimple_location (stmt
),
3021 "argument not permitted on %qs clause in"
3022 " OpenACC %<parallel%>", check
);
3025 if (tgt
&& is_oacc_kernels (tgt
))
3027 /* Strip out reductions, as they are not handled yet. */
3028 tree
*prev_ptr
= &clauses
;
3030 while (tree probe
= *prev_ptr
)
3032 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3034 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3035 *prev_ptr
= *next_ptr
;
3037 prev_ptr
= next_ptr
;
3040 gimple_omp_for_set_clauses (stmt
, clauses
);
3041 check_oacc_kernel_gwv (stmt
, ctx
);
3045 scan_sharing_clauses (clauses
, ctx
);
3047 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3048 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3050 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3051 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3052 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3053 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3055 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3058 /* Scan an OpenMP sections directive. */
3061 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3065 ctx
= new_omp_context (stmt
, outer_ctx
);
3066 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3067 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3070 /* Scan an OpenMP single directive. */
3073 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3078 ctx
= new_omp_context (stmt
, outer_ctx
);
3079 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3080 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3081 name
= create_tmp_var_name (".omp_copy_s");
3082 name
= build_decl (gimple_location (stmt
),
3083 TYPE_DECL
, name
, ctx
->record_type
);
3084 TYPE_NAME (ctx
->record_type
) = name
;
3086 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3087 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3089 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3090 ctx
->record_type
= NULL
;
3092 layout_type (ctx
->record_type
);
3095 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3096 used in the corresponding offloaded function are restrict. */
3099 omp_target_base_pointers_restrict_p (tree clauses
)
3101 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3103 if (flag_openacc
== 0)
3106 /* I. Basic example:
3110 unsigned int a[2], b[2];
3112 #pragma acc kernels \
3121 After gimplification, we have:
3123 #pragma omp target oacc_kernels \
3124 map(force_from:a [len: 8]) \
3125 map(force_from:b [len: 8])
3131 Because both mappings have the force prefix, we know that they will be
3132 allocated when calling the corresponding offloaded function, which means we
3133 can mark the base pointers for a and b in the offloaded function as
3137 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3139 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
3142 switch (OMP_CLAUSE_MAP_KIND (c
))
3144 case GOMP_MAP_FORCE_ALLOC
:
3145 case GOMP_MAP_FORCE_TO
:
3146 case GOMP_MAP_FORCE_FROM
:
3147 case GOMP_MAP_FORCE_TOFROM
:
3157 /* Scan a GIMPLE_OMP_TARGET. */
3160 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3164 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3165 tree clauses
= gimple_omp_target_clauses (stmt
);
3167 ctx
= new_omp_context (stmt
, outer_ctx
);
3168 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3169 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3170 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3171 name
= create_tmp_var_name (".omp_data_t");
3172 name
= build_decl (gimple_location (stmt
),
3173 TYPE_DECL
, name
, ctx
->record_type
);
3174 DECL_ARTIFICIAL (name
) = 1;
3175 DECL_NAMELESS (name
) = 1;
3176 TYPE_NAME (ctx
->record_type
) = name
;
3177 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3179 bool base_pointers_restrict
= false;
3182 create_omp_child_function (ctx
, false);
3183 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3185 base_pointers_restrict
= omp_target_base_pointers_restrict_p (clauses
);
3186 if (base_pointers_restrict
3187 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3189 "Base pointers in offloaded function are restrict\n");
3192 scan_sharing_clauses (clauses
, ctx
, base_pointers_restrict
);
3193 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3195 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3196 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3199 TYPE_FIELDS (ctx
->record_type
)
3200 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3203 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3204 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3206 field
= DECL_CHAIN (field
))
3207 gcc_assert (DECL_ALIGN (field
) == align
);
3209 layout_type (ctx
->record_type
);
3211 fixup_child_record_type (ctx
);
3215 /* Scan an OpenMP teams directive. */
3218 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3220 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3221 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3222 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3225 /* Check nesting restrictions. */
3227 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3231 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
3232 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3233 the original copy of its contents. */
3236 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3237 inside an OpenACC CTX. */
3238 if (!(is_gimple_omp (stmt
)
3239 && is_gimple_omp_oacc (stmt
))
3240 /* Except for atomic codes that we share with OpenMP. */
3241 && !(gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3242 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3244 if (get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3246 error_at (gimple_location (stmt
),
3247 "non-OpenACC construct inside of OpenACC routine");
3251 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3252 if (is_gimple_omp (octx
->stmt
)
3253 && is_gimple_omp_oacc (octx
->stmt
))
3255 error_at (gimple_location (stmt
),
3256 "non-OpenACC construct inside of OpenACC region");
3263 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3264 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3267 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3269 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3270 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3272 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3273 && (ctx
->outer
== NULL
3274 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3275 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3276 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3277 != GF_OMP_FOR_KIND_FOR
)
3278 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3280 error_at (gimple_location (stmt
),
3281 "%<ordered simd threads%> must be closely "
3282 "nested inside of %<for simd%> region");
3288 error_at (gimple_location (stmt
),
3289 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3290 " may not be nested inside %<simd%> region");
3293 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3295 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3296 || (gimple_omp_for_kind (stmt
)
3297 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3298 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3300 error_at (gimple_location (stmt
),
3301 "only %<distribute%> or %<parallel%> regions are "
3302 "allowed to be strictly nested inside %<teams%> "
3308 switch (gimple_code (stmt
))
3310 case GIMPLE_OMP_FOR
:
3311 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3313 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3315 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3317 error_at (gimple_location (stmt
),
3318 "%<distribute%> region must be strictly nested "
3319 "inside %<teams%> construct");
3324 /* We split taskloop into task and nested taskloop in it. */
3325 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3327 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3332 switch (gimple_code (ctx
->stmt
))
3334 case GIMPLE_OMP_FOR
:
3335 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3336 == GF_OMP_FOR_KIND_OACC_LOOP
);
3339 case GIMPLE_OMP_TARGET
:
3340 switch (gimple_omp_target_kind (ctx
->stmt
))
3342 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3343 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3354 else if (get_oacc_fn_attrib (current_function_decl
))
3358 error_at (gimple_location (stmt
),
3359 "OpenACC loop directive must be associated with"
3360 " an OpenACC compute region");
3366 if (is_gimple_call (stmt
)
3367 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3368 == BUILT_IN_GOMP_CANCEL
3369 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3370 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3372 const char *bad
= NULL
;
3373 const char *kind
= NULL
;
3374 const char *construct
3375 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3376 == BUILT_IN_GOMP_CANCEL
)
3377 ? "#pragma omp cancel"
3378 : "#pragma omp cancellation point";
3381 error_at (gimple_location (stmt
), "orphaned %qs construct",
3385 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3386 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3390 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3391 bad
= "#pragma omp parallel";
3392 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3393 == BUILT_IN_GOMP_CANCEL
3394 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3395 ctx
->cancellable
= true;
3399 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3400 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3401 bad
= "#pragma omp for";
3402 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3403 == BUILT_IN_GOMP_CANCEL
3404 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3406 ctx
->cancellable
= true;
3407 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3409 warning_at (gimple_location (stmt
), 0,
3410 "%<#pragma omp cancel for%> inside "
3411 "%<nowait%> for construct");
3412 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3413 OMP_CLAUSE_ORDERED
))
3414 warning_at (gimple_location (stmt
), 0,
3415 "%<#pragma omp cancel for%> inside "
3416 "%<ordered%> for construct");
3421 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3422 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3423 bad
= "#pragma omp sections";
3424 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3425 == BUILT_IN_GOMP_CANCEL
3426 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3428 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3430 ctx
->cancellable
= true;
3431 if (find_omp_clause (gimple_omp_sections_clauses
3434 warning_at (gimple_location (stmt
), 0,
3435 "%<#pragma omp cancel sections%> inside "
3436 "%<nowait%> sections construct");
3440 gcc_assert (ctx
->outer
3441 && gimple_code (ctx
->outer
->stmt
)
3442 == GIMPLE_OMP_SECTIONS
);
3443 ctx
->outer
->cancellable
= true;
3444 if (find_omp_clause (gimple_omp_sections_clauses
3447 warning_at (gimple_location (stmt
), 0,
3448 "%<#pragma omp cancel sections%> inside "
3449 "%<nowait%> sections construct");
3455 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3456 bad
= "#pragma omp task";
3459 for (omp_context
*octx
= ctx
->outer
;
3460 octx
; octx
= octx
->outer
)
3462 switch (gimple_code (octx
->stmt
))
3464 case GIMPLE_OMP_TASKGROUP
:
3466 case GIMPLE_OMP_TARGET
:
3467 if (gimple_omp_target_kind (octx
->stmt
)
3468 != GF_OMP_TARGET_KIND_REGION
)
3471 case GIMPLE_OMP_PARALLEL
:
3472 case GIMPLE_OMP_TEAMS
:
3473 error_at (gimple_location (stmt
),
3474 "%<%s taskgroup%> construct not closely "
3475 "nested inside of %<taskgroup%> region",
3483 ctx
->cancellable
= true;
3488 error_at (gimple_location (stmt
), "invalid arguments");
3493 error_at (gimple_location (stmt
),
3494 "%<%s %s%> construct not closely nested inside of %qs",
3495 construct
, kind
, bad
);
3500 case GIMPLE_OMP_SECTIONS
:
3501 case GIMPLE_OMP_SINGLE
:
3502 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3503 switch (gimple_code (ctx
->stmt
))
3505 case GIMPLE_OMP_FOR
:
3506 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3507 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3510 case GIMPLE_OMP_SECTIONS
:
3511 case GIMPLE_OMP_SINGLE
:
3512 case GIMPLE_OMP_ORDERED
:
3513 case GIMPLE_OMP_MASTER
:
3514 case GIMPLE_OMP_TASK
:
3515 case GIMPLE_OMP_CRITICAL
:
3516 if (is_gimple_call (stmt
))
3518 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3519 != BUILT_IN_GOMP_BARRIER
)
3521 error_at (gimple_location (stmt
),
3522 "barrier region may not be closely nested inside "
3523 "of work-sharing, %<critical%>, %<ordered%>, "
3524 "%<master%>, explicit %<task%> or %<taskloop%> "
3528 error_at (gimple_location (stmt
),
3529 "work-sharing region may not be closely nested inside "
3530 "of work-sharing, %<critical%>, %<ordered%>, "
3531 "%<master%>, explicit %<task%> or %<taskloop%> region");
3533 case GIMPLE_OMP_PARALLEL
:
3534 case GIMPLE_OMP_TEAMS
:
3536 case GIMPLE_OMP_TARGET
:
3537 if (gimple_omp_target_kind (ctx
->stmt
)
3538 == GF_OMP_TARGET_KIND_REGION
)
3545 case GIMPLE_OMP_MASTER
:
3546 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3547 switch (gimple_code (ctx
->stmt
))
3549 case GIMPLE_OMP_FOR
:
3550 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3551 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3554 case GIMPLE_OMP_SECTIONS
:
3555 case GIMPLE_OMP_SINGLE
:
3556 case GIMPLE_OMP_TASK
:
3557 error_at (gimple_location (stmt
),
3558 "%<master%> region may not be closely nested inside "
3559 "of work-sharing, explicit %<task%> or %<taskloop%> "
3562 case GIMPLE_OMP_PARALLEL
:
3563 case GIMPLE_OMP_TEAMS
:
3565 case GIMPLE_OMP_TARGET
:
3566 if (gimple_omp_target_kind (ctx
->stmt
)
3567 == GF_OMP_TARGET_KIND_REGION
)
3574 case GIMPLE_OMP_TASK
:
3575 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3576 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3577 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3578 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3580 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3581 error_at (OMP_CLAUSE_LOCATION (c
),
3582 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3583 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3587 case GIMPLE_OMP_ORDERED
:
3588 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3589 c
; c
= OMP_CLAUSE_CHAIN (c
))
3591 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3593 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3594 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3597 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3598 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3599 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3602 /* Look for containing ordered(N) loop. */
3604 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3606 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3607 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3609 error_at (OMP_CLAUSE_LOCATION (c
),
3610 "%<ordered%> construct with %<depend%> clause "
3611 "must be closely nested inside an %<ordered%> "
3615 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3617 error_at (OMP_CLAUSE_LOCATION (c
),
3618 "%<ordered%> construct with %<depend%> clause "
3619 "must be closely nested inside a loop with "
3620 "%<ordered%> clause with a parameter");
3626 error_at (OMP_CLAUSE_LOCATION (c
),
3627 "invalid depend kind in omp %<ordered%> %<depend%>");
3631 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3632 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3634 /* ordered simd must be closely nested inside of simd region,
3635 and simd region must not encounter constructs other than
3636 ordered simd, therefore ordered simd may be either orphaned,
3637 or ctx->stmt must be simd. The latter case is handled already
3641 error_at (gimple_location (stmt
),
3642 "%<ordered%> %<simd%> must be closely nested inside "
3647 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3648 switch (gimple_code (ctx
->stmt
))
3650 case GIMPLE_OMP_CRITICAL
:
3651 case GIMPLE_OMP_TASK
:
3652 case GIMPLE_OMP_ORDERED
:
3653 ordered_in_taskloop
:
3654 error_at (gimple_location (stmt
),
3655 "%<ordered%> region may not be closely nested inside "
3656 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3657 "%<taskloop%> region");
3659 case GIMPLE_OMP_FOR
:
3660 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3661 goto ordered_in_taskloop
;
3662 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3663 OMP_CLAUSE_ORDERED
) == NULL
)
3665 error_at (gimple_location (stmt
),
3666 "%<ordered%> region must be closely nested inside "
3667 "a loop region with an %<ordered%> clause");
3671 case GIMPLE_OMP_TARGET
:
3672 if (gimple_omp_target_kind (ctx
->stmt
)
3673 != GF_OMP_TARGET_KIND_REGION
)
3676 case GIMPLE_OMP_PARALLEL
:
3677 case GIMPLE_OMP_TEAMS
:
3678 error_at (gimple_location (stmt
),
3679 "%<ordered%> region must be closely nested inside "
3680 "a loop region with an %<ordered%> clause");
3686 case GIMPLE_OMP_CRITICAL
:
3689 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3690 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3691 if (gomp_critical
*other_crit
3692 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3693 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3695 error_at (gimple_location (stmt
),
3696 "%<critical%> region may not be nested inside "
3697 "a %<critical%> region with the same name");
3702 case GIMPLE_OMP_TEAMS
:
3704 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3705 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3707 error_at (gimple_location (stmt
),
3708 "%<teams%> construct not closely nested inside of "
3709 "%<target%> construct");
3713 case GIMPLE_OMP_TARGET
:
3714 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3715 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3716 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3717 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3719 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3720 error_at (OMP_CLAUSE_LOCATION (c
),
3721 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3722 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3725 if (is_gimple_omp_offloaded (stmt
)
3726 && get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3728 error_at (gimple_location (stmt
),
3729 "OpenACC region inside of OpenACC routine, nested "
3730 "parallelism not supported yet");
3733 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3735 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3737 if (is_gimple_omp (stmt
)
3738 && is_gimple_omp_oacc (stmt
)
3739 && is_gimple_omp (ctx
->stmt
))
3741 error_at (gimple_location (stmt
),
3742 "OpenACC construct inside of non-OpenACC region");
3748 const char *stmt_name
, *ctx_stmt_name
;
3749 switch (gimple_omp_target_kind (stmt
))
3751 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3752 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3753 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3754 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3755 stmt_name
= "target enter data"; break;
3756 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3757 stmt_name
= "target exit data"; break;
3758 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3759 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3760 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3761 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3762 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3763 stmt_name
= "enter/exit data"; break;
3764 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3766 default: gcc_unreachable ();
3768 switch (gimple_omp_target_kind (ctx
->stmt
))
3770 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3771 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3772 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3773 ctx_stmt_name
= "parallel"; break;
3774 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3775 ctx_stmt_name
= "kernels"; break;
3776 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3777 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3778 ctx_stmt_name
= "host_data"; break;
3779 default: gcc_unreachable ();
3782 /* OpenACC/OpenMP mismatch? */
3783 if (is_gimple_omp_oacc (stmt
)
3784 != is_gimple_omp_oacc (ctx
->stmt
))
3786 error_at (gimple_location (stmt
),
3787 "%s %qs construct inside of %s %qs region",
3788 (is_gimple_omp_oacc (stmt
)
3789 ? "OpenACC" : "OpenMP"), stmt_name
,
3790 (is_gimple_omp_oacc (ctx
->stmt
)
3791 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3794 if (is_gimple_omp_offloaded (ctx
->stmt
))
3796 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3797 if (is_gimple_omp_oacc (ctx
->stmt
))
3799 error_at (gimple_location (stmt
),
3800 "%qs construct inside of %qs region",
3801 stmt_name
, ctx_stmt_name
);
3806 warning_at (gimple_location (stmt
), 0,
3807 "%qs construct inside of %qs region",
3808 stmt_name
, ctx_stmt_name
);
3820 /* Helper function scan_omp.
3822 Callback for walk_tree or operators in walk_gimple_stmt used to
3823 scan for OMP directives in TP. */
3826 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3828 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3829 omp_context
*ctx
= (omp_context
*) wi
->info
;
3832 switch (TREE_CODE (t
))
3840 tree repl
= remap_decl (t
, &ctx
->cb
);
3841 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3847 if (ctx
&& TYPE_P (t
))
3848 *tp
= remap_type (t
, &ctx
->cb
);
3849 else if (!DECL_P (t
))
3854 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3855 if (tem
!= TREE_TYPE (t
))
3857 if (TREE_CODE (t
) == INTEGER_CST
)
3858 *tp
= wide_int_to_tree (tem
, t
);
3860 TREE_TYPE (t
) = tem
;
3870 /* Return true if FNDECL is a setjmp or a longjmp. */
3873 setjmp_or_longjmp_p (const_tree fndecl
)
3875 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3876 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3877 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3880 tree declname
= DECL_NAME (fndecl
);
3883 const char *name
= IDENTIFIER_POINTER (declname
);
3884 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3888 /* Helper function for scan_omp.
3890 Callback for walk_gimple_stmt used to scan for OMP directives in
3891 the current statement in GSI. */
3894 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3895 struct walk_stmt_info
*wi
)
3897 gimple
*stmt
= gsi_stmt (*gsi
);
3898 omp_context
*ctx
= (omp_context
*) wi
->info
;
3900 if (gimple_has_location (stmt
))
3901 input_location
= gimple_location (stmt
);
3903 /* Check the nesting restrictions. */
3904 bool remove
= false;
3905 if (is_gimple_omp (stmt
))
3906 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3907 else if (is_gimple_call (stmt
))
3909 tree fndecl
= gimple_call_fndecl (stmt
);
3912 if (setjmp_or_longjmp_p (fndecl
)
3914 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3915 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3918 error_at (gimple_location (stmt
),
3919 "setjmp/longjmp inside simd construct");
3921 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3922 switch (DECL_FUNCTION_CODE (fndecl
))
3924 case BUILT_IN_GOMP_BARRIER
:
3925 case BUILT_IN_GOMP_CANCEL
:
3926 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3927 case BUILT_IN_GOMP_TASKYIELD
:
3928 case BUILT_IN_GOMP_TASKWAIT
:
3929 case BUILT_IN_GOMP_TASKGROUP_START
:
3930 case BUILT_IN_GOMP_TASKGROUP_END
:
3931 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3940 stmt
= gimple_build_nop ();
3941 gsi_replace (gsi
, stmt
, false);
3944 *handled_ops_p
= true;
3946 switch (gimple_code (stmt
))
3948 case GIMPLE_OMP_PARALLEL
:
3949 taskreg_nesting_level
++;
3950 scan_omp_parallel (gsi
, ctx
);
3951 taskreg_nesting_level
--;
3954 case GIMPLE_OMP_TASK
:
3955 taskreg_nesting_level
++;
3956 scan_omp_task (gsi
, ctx
);
3957 taskreg_nesting_level
--;
3960 case GIMPLE_OMP_FOR
:
3961 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3964 case GIMPLE_OMP_SECTIONS
:
3965 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3968 case GIMPLE_OMP_SINGLE
:
3969 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3972 case GIMPLE_OMP_SECTION
:
3973 case GIMPLE_OMP_MASTER
:
3974 case GIMPLE_OMP_TASKGROUP
:
3975 case GIMPLE_OMP_ORDERED
:
3976 case GIMPLE_OMP_CRITICAL
:
3977 case GIMPLE_OMP_GRID_BODY
:
3978 ctx
= new_omp_context (stmt
, ctx
);
3979 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3982 case GIMPLE_OMP_TARGET
:
3983 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3986 case GIMPLE_OMP_TEAMS
:
3987 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3994 *handled_ops_p
= false;
3996 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3998 var
= DECL_CHAIN (var
))
3999 insert_decl_map (&ctx
->cb
, var
, var
);
4003 *handled_ops_p
= false;
4011 /* Scan all the statements starting at the current statement. CTX
4012 contains context information about the OMP directives and
4013 clauses found during the scan. */
4016 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
4018 location_t saved_location
;
4019 struct walk_stmt_info wi
;
4021 memset (&wi
, 0, sizeof (wi
));
4023 wi
.want_locations
= true;
4025 saved_location
= input_location
;
4026 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
4027 input_location
= saved_location
;
4030 /* Re-gimplification and code generation routines. */
4032 /* Build a call to GOMP_barrier. */
4035 build_omp_barrier (tree lhs
)
4037 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
4038 : BUILT_IN_GOMP_BARRIER
);
4039 gcall
*g
= gimple_build_call (fndecl
, 0);
4041 gimple_call_set_lhs (g
, lhs
);
4045 /* If a context was created for STMT when it was scanned, return it. */
4047 static omp_context
*
4048 maybe_lookup_ctx (gimple
*stmt
)
4051 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
4052 return n
? (omp_context
*) n
->value
: NULL
;
4056 /* Find the mapping for DECL in CTX or the immediately enclosing
4057 context that has a mapping for DECL.
4059 If CTX is a nested parallel directive, we may have to use the decl
4060 mappings created in CTX's parent context. Suppose that we have the
4061 following parallel nesting (variable UIDs showed for clarity):
4064 #omp parallel shared(iD.1562) -> outer parallel
4065 iD.1562 = iD.1562 + 1;
4067 #omp parallel shared (iD.1562) -> inner parallel
4068 iD.1562 = iD.1562 - 1;
4070 Each parallel structure will create a distinct .omp_data_s structure
4071 for copying iD.1562 in/out of the directive:
4073 outer parallel .omp_data_s.1.i -> iD.1562
4074 inner parallel .omp_data_s.2.i -> iD.1562
4076 A shared variable mapping will produce a copy-out operation before
4077 the parallel directive and a copy-in operation after it. So, in
4078 this case we would have:
4081 .omp_data_o.1.i = iD.1562;
4082 #omp parallel shared(iD.1562) -> outer parallel
4083 .omp_data_i.1 = &.omp_data_o.1
4084 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4086 .omp_data_o.2.i = iD.1562; -> **
4087 #omp parallel shared(iD.1562) -> inner parallel
4088 .omp_data_i.2 = &.omp_data_o.2
4089 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4092 ** This is a problem. The symbol iD.1562 cannot be referenced
4093 inside the body of the outer parallel region. But since we are
4094 emitting this copy operation while expanding the inner parallel
4095 directive, we need to access the CTX structure of the outer
4096 parallel directive to get the correct mapping:
4098 .omp_data_o.2.i = .omp_data_i.1->i
4100 Since there may be other workshare or parallel directives enclosing
4101 the parallel directive, it may be necessary to walk up the context
4102 parent chain. This is not a problem in general because nested
4103 parallelism happens only rarely. */
4106 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4111 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4112 t
= maybe_lookup_decl (decl
, up
);
4114 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
4116 return t
? t
: decl
;
4120 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4121 in outer contexts. */
4124 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4129 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4130 t
= maybe_lookup_decl (decl
, up
);
4132 return t
? t
: decl
;
4136 /* Construct the initialization value for reduction operation OP. */
4139 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4148 case TRUTH_ORIF_EXPR
:
4149 case TRUTH_XOR_EXPR
:
4151 return build_zero_cst (type
);
4154 case TRUTH_AND_EXPR
:
4155 case TRUTH_ANDIF_EXPR
:
4157 return fold_convert_loc (loc
, type
, integer_one_node
);
4160 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4163 if (SCALAR_FLOAT_TYPE_P (type
))
4165 REAL_VALUE_TYPE max
, min
;
4166 if (HONOR_INFINITIES (type
))
4169 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4172 real_maxval (&min
, 1, TYPE_MODE (type
));
4173 return build_real (type
, min
);
4175 else if (POINTER_TYPE_P (type
))
4178 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4179 return wide_int_to_tree (type
, min
);
4183 gcc_assert (INTEGRAL_TYPE_P (type
));
4184 return TYPE_MIN_VALUE (type
);
4188 if (SCALAR_FLOAT_TYPE_P (type
))
4190 REAL_VALUE_TYPE max
;
4191 if (HONOR_INFINITIES (type
))
4194 real_maxval (&max
, 0, TYPE_MODE (type
));
4195 return build_real (type
, max
);
4197 else if (POINTER_TYPE_P (type
))
4200 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4201 return wide_int_to_tree (type
, max
);
4205 gcc_assert (INTEGRAL_TYPE_P (type
));
4206 return TYPE_MAX_VALUE (type
);
4214 /* Construct the initialization value for reduction CLAUSE. */
4217 omp_reduction_init (tree clause
, tree type
)
4219 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4220 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4223 /* Return alignment to be assumed for var in CLAUSE, which should be
4224 OMP_CLAUSE_ALIGNED. */
4227 omp_clause_aligned_alignment (tree clause
)
4229 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4230 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4232 /* Otherwise return implementation defined alignment. */
4233 unsigned int al
= 1;
4234 machine_mode mode
, vmode
;
4235 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4237 vs
= 1 << floor_log2 (vs
);
4238 static enum mode_class classes
[]
4239 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4240 for (int i
= 0; i
< 4; i
+= 2)
4241 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4243 mode
= GET_MODE_WIDER_MODE (mode
))
4245 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4246 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4249 && GET_MODE_SIZE (vmode
) < vs
4250 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4251 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4253 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4254 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4256 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4257 / GET_MODE_SIZE (mode
));
4258 if (TYPE_MODE (type
) != vmode
)
4260 if (TYPE_ALIGN_UNIT (type
) > al
)
4261 al
= TYPE_ALIGN_UNIT (type
);
4263 return build_int_cst (integer_type_node
, al
);
4266 /* Return maximum possible vectorization factor for the target. */
4273 || !flag_tree_loop_optimize
4274 || (!flag_tree_loop_vectorize
4275 && (global_options_set
.x_flag_tree_loop_vectorize
4276 || global_options_set
.x_flag_tree_vectorize
)))
4279 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4282 vs
= 1 << floor_log2 (vs
);
4285 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4286 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4287 return GET_MODE_NUNITS (vqimode
);
4291 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4295 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4296 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4300 max_vf
= omp_max_vf ();
4303 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4304 OMP_CLAUSE_SAFELEN
);
4306 && (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
4307 || tree_int_cst_sgn (OMP_CLAUSE_SAFELEN_EXPR (c
)) != 1))
4309 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4311 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4315 idx
= create_tmp_var (unsigned_type_node
);
4316 lane
= create_tmp_var (unsigned_type_node
);
4322 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4323 tree avar
= create_tmp_var_raw (atype
);
4324 if (TREE_ADDRESSABLE (new_var
))
4325 TREE_ADDRESSABLE (avar
) = 1;
4326 DECL_ATTRIBUTES (avar
)
4327 = tree_cons (get_identifier ("omp simd array"), NULL
,
4328 DECL_ATTRIBUTES (avar
));
4329 gimple_add_tmp_var (avar
);
4330 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4331 NULL_TREE
, NULL_TREE
);
4332 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4333 NULL_TREE
, NULL_TREE
);
4334 if (DECL_P (new_var
))
4336 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4337 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4342 /* Helper function of lower_rec_input_clauses. For a reference
4343 in simd reduction, add an underlying variable it will reference. */
4346 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4348 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4349 if (TREE_CONSTANT (z
))
4351 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4352 get_name (new_vard
));
4353 gimple_add_tmp_var (z
);
4354 TREE_ADDRESSABLE (z
) = 1;
4355 z
= build_fold_addr_expr_loc (loc
, z
);
4356 gimplify_assign (new_vard
, z
, ilist
);
4360 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4361 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4362 private variables. Initialization statements go in ILIST, while calls
4363 to destructors go in DLIST. */
4366 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4367 omp_context
*ctx
, struct omp_for_data
*fd
)
4369 tree c
, dtor
, copyin_seq
, x
, ptr
;
4370 bool copyin_by_ref
= false;
4371 bool lastprivate_firstprivate
= false;
4372 bool reduction_omp_orig_ref
= false;
4374 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4375 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4377 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4378 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4379 gimple_seq llist
[2] = { NULL
, NULL
};
4383 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4384 with data sharing clauses referencing variable sized vars. That
4385 is unnecessarily hard to support and very unlikely to result in
4386 vectorized code anyway. */
4388 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4389 switch (OMP_CLAUSE_CODE (c
))
4391 case OMP_CLAUSE_LINEAR
:
4392 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4395 case OMP_CLAUSE_PRIVATE
:
4396 case OMP_CLAUSE_FIRSTPRIVATE
:
4397 case OMP_CLAUSE_LASTPRIVATE
:
4398 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4401 case OMP_CLAUSE_REDUCTION
:
4402 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4403 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4410 /* Do all the fixed sized types in the first pass, and the variable sized
4411 types in the second pass. This makes sure that the scalar arguments to
4412 the variable sized types are processed before we use them in the
4413 variable sized operations. */
4414 for (pass
= 0; pass
< 2; ++pass
)
4416 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4418 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4421 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4425 case OMP_CLAUSE_PRIVATE
:
4426 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4429 case OMP_CLAUSE_SHARED
:
4430 /* Ignore shared directives in teams construct. */
4431 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4433 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4435 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4436 || is_global_var (OMP_CLAUSE_DECL (c
)));
4439 case OMP_CLAUSE_FIRSTPRIVATE
:
4440 case OMP_CLAUSE_COPYIN
:
4442 case OMP_CLAUSE_LINEAR
:
4443 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4444 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4445 lastprivate_firstprivate
= true;
4447 case OMP_CLAUSE_REDUCTION
:
4448 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4449 reduction_omp_orig_ref
= true;
4451 case OMP_CLAUSE__LOOPTEMP_
:
4452 /* Handle _looptemp_ clauses only on parallel/task. */
4456 case OMP_CLAUSE_LASTPRIVATE
:
4457 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4459 lastprivate_firstprivate
= true;
4460 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4463 /* Even without corresponding firstprivate, if
4464 decl is Fortran allocatable, it needs outer var
4467 && lang_hooks
.decls
.omp_private_outer_ref
4468 (OMP_CLAUSE_DECL (c
)))
4469 lastprivate_firstprivate
= true;
4471 case OMP_CLAUSE_ALIGNED
:
4474 var
= OMP_CLAUSE_DECL (c
);
4475 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4476 && !is_global_var (var
))
4478 new_var
= maybe_lookup_decl (var
, ctx
);
4479 if (new_var
== NULL_TREE
)
4480 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4481 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4482 tree alarg
= omp_clause_aligned_alignment (c
);
4483 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4484 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
, alarg
);
4485 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4486 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4487 gimplify_and_add (x
, ilist
);
4489 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4490 && is_global_var (var
))
4492 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4493 new_var
= lookup_decl (var
, ctx
);
4494 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4495 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4496 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4497 tree alarg
= omp_clause_aligned_alignment (c
);
4498 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4499 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
, alarg
);
4500 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4501 x
= create_tmp_var (ptype
);
4502 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4503 gimplify_and_add (t
, ilist
);
4504 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4505 SET_DECL_VALUE_EXPR (new_var
, t
);
4506 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4513 new_var
= var
= OMP_CLAUSE_DECL (c
);
4514 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4516 var
= TREE_OPERAND (var
, 0);
4517 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4518 var
= TREE_OPERAND (var
, 0);
4519 if (TREE_CODE (var
) == INDIRECT_REF
4520 || TREE_CODE (var
) == ADDR_EXPR
)
4521 var
= TREE_OPERAND (var
, 0);
4522 if (is_variable_sized (var
))
4524 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4525 var
= DECL_VALUE_EXPR (var
);
4526 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4527 var
= TREE_OPERAND (var
, 0);
4528 gcc_assert (DECL_P (var
));
4532 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4533 new_var
= lookup_decl (var
, ctx
);
4535 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4540 /* C/C++ array section reductions. */
4541 else if (c_kind
== OMP_CLAUSE_REDUCTION
4542 && var
!= OMP_CLAUSE_DECL (c
))
4547 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4548 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4549 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4551 tree b
= TREE_OPERAND (orig_var
, 1);
4552 b
= maybe_lookup_decl (b
, ctx
);
4555 b
= TREE_OPERAND (orig_var
, 1);
4556 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4558 if (integer_zerop (bias
))
4562 bias
= fold_convert_loc (clause_loc
,
4563 TREE_TYPE (b
), bias
);
4564 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4565 TREE_TYPE (b
), b
, bias
);
4567 orig_var
= TREE_OPERAND (orig_var
, 0);
4569 if (TREE_CODE (orig_var
) == INDIRECT_REF
4570 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4571 orig_var
= TREE_OPERAND (orig_var
, 0);
4572 tree d
= OMP_CLAUSE_DECL (c
);
4573 tree type
= TREE_TYPE (d
);
4574 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4575 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4576 const char *name
= get_name (orig_var
);
4577 if (TREE_CONSTANT (v
))
4579 x
= create_tmp_var_raw (type
, name
);
4580 gimple_add_tmp_var (x
);
4581 TREE_ADDRESSABLE (x
) = 1;
4582 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4587 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4588 tree t
= maybe_lookup_decl (v
, ctx
);
4592 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4593 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4594 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4596 build_int_cst (TREE_TYPE (v
), 1));
4597 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4599 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4600 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4601 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4604 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4605 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4606 tree y
= create_tmp_var (ptype
, name
);
4607 gimplify_assign (y
, x
, ilist
);
4611 if (!integer_zerop (bias
))
4613 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4615 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4617 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4618 pointer_sized_int_node
, yb
, bias
);
4619 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4620 yb
= create_tmp_var (ptype
, name
);
4621 gimplify_assign (yb
, x
, ilist
);
4625 d
= TREE_OPERAND (d
, 0);
4626 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4627 d
= TREE_OPERAND (d
, 0);
4628 if (TREE_CODE (d
) == ADDR_EXPR
)
4630 if (orig_var
!= var
)
4632 gcc_assert (is_variable_sized (orig_var
));
4633 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4635 gimplify_assign (new_var
, x
, ilist
);
4636 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4637 tree t
= build_fold_indirect_ref (new_var
);
4638 DECL_IGNORED_P (new_var
) = 0;
4639 TREE_THIS_NOTRAP (t
);
4640 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4641 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4645 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4646 build_int_cst (ptype
, 0));
4647 SET_DECL_VALUE_EXPR (new_var
, x
);
4648 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4653 gcc_assert (orig_var
== var
);
4654 if (TREE_CODE (d
) == INDIRECT_REF
)
4656 x
= create_tmp_var (ptype
, name
);
4657 TREE_ADDRESSABLE (x
) = 1;
4658 gimplify_assign (x
, yb
, ilist
);
4659 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4661 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4662 gimplify_assign (new_var
, x
, ilist
);
4664 tree y1
= create_tmp_var (ptype
, NULL
);
4665 gimplify_assign (y1
, y
, ilist
);
4666 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4667 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4668 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4669 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4671 y2
= create_tmp_var (ptype
, NULL
);
4672 gimplify_assign (y2
, y
, ilist
);
4673 tree ref
= build_outer_var_ref (var
, ctx
);
4674 /* For ref build_outer_var_ref already performs this. */
4675 if (TREE_CODE (d
) == INDIRECT_REF
)
4676 gcc_assert (is_reference (var
));
4677 else if (TREE_CODE (d
) == ADDR_EXPR
)
4678 ref
= build_fold_addr_expr (ref
);
4679 else if (is_reference (var
))
4680 ref
= build_fold_addr_expr (ref
);
4681 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4682 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4683 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4685 y3
= create_tmp_var (ptype
, NULL
);
4686 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4690 y4
= create_tmp_var (ptype
, NULL
);
4691 gimplify_assign (y4
, ref
, dlist
);
4694 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4695 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4696 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4697 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4698 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4701 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4702 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4703 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4704 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4705 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4707 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4709 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4710 tree decl_placeholder
4711 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4712 SET_DECL_VALUE_EXPR (decl_placeholder
,
4713 build_simple_mem_ref (y1
));
4714 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4715 SET_DECL_VALUE_EXPR (placeholder
,
4716 y3
? build_simple_mem_ref (y3
)
4718 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4719 x
= lang_hooks
.decls
.omp_clause_default_ctor
4720 (c
, build_simple_mem_ref (y1
),
4721 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4723 gimplify_and_add (x
, ilist
);
4724 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4726 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4727 lower_omp (&tseq
, ctx
);
4728 gimple_seq_add_seq (ilist
, tseq
);
4730 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4733 SET_DECL_VALUE_EXPR (decl_placeholder
,
4734 build_simple_mem_ref (y2
));
4735 SET_DECL_VALUE_EXPR (placeholder
,
4736 build_simple_mem_ref (y4
));
4737 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4738 lower_omp (&tseq
, ctx
);
4739 gimple_seq_add_seq (dlist
, tseq
);
4740 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4742 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4743 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4744 x
= lang_hooks
.decls
.omp_clause_dtor
4745 (c
, build_simple_mem_ref (y2
));
4748 gimple_seq tseq
= NULL
;
4750 gimplify_stmt (&dtor
, &tseq
);
4751 gimple_seq_add_seq (dlist
, tseq
);
4756 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4757 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4759 /* reduction(-:var) sums up the partial results, so it
4760 acts identically to reduction(+:var). */
4761 if (code
== MINUS_EXPR
)
4764 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4767 x
= build2 (code
, TREE_TYPE (type
),
4768 build_simple_mem_ref (y4
),
4769 build_simple_mem_ref (y2
));
4770 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4774 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4775 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4776 gimple_seq_add_stmt (ilist
, g
);
4779 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4780 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4781 gimple_seq_add_stmt (ilist
, g
);
4783 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4784 build_int_cst (TREE_TYPE (i
), 1));
4785 gimple_seq_add_stmt (ilist
, g
);
4786 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4787 gimple_seq_add_stmt (ilist
, g
);
4788 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4791 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4792 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4793 gimple_seq_add_stmt (dlist
, g
);
4796 g
= gimple_build_assign
4797 (y4
, POINTER_PLUS_EXPR
, y4
,
4798 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4799 gimple_seq_add_stmt (dlist
, g
);
4801 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4802 build_int_cst (TREE_TYPE (i2
), 1));
4803 gimple_seq_add_stmt (dlist
, g
);
4804 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4805 gimple_seq_add_stmt (dlist
, g
);
4806 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4810 else if (is_variable_sized (var
))
4812 /* For variable sized types, we need to allocate the
4813 actual storage here. Call alloca and store the
4814 result in the pointer decl that we created elsewhere. */
4818 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4823 ptr
= DECL_VALUE_EXPR (new_var
);
4824 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4825 ptr
= TREE_OPERAND (ptr
, 0);
4826 gcc_assert (DECL_P (ptr
));
4827 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4829 /* void *tmp = __builtin_alloca */
4830 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4831 stmt
= gimple_build_call (atmp
, 2, x
,
4832 size_int (DECL_ALIGN (var
)));
4833 tmp
= create_tmp_var_raw (ptr_type_node
);
4834 gimple_add_tmp_var (tmp
);
4835 gimple_call_set_lhs (stmt
, tmp
);
4837 gimple_seq_add_stmt (ilist
, stmt
);
4839 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4840 gimplify_assign (ptr
, x
, ilist
);
4843 else if (is_reference (var
))
4845 /* For references that are being privatized for Fortran,
4846 allocate new backing storage for the new pointer
4847 variable. This allows us to avoid changing all the
4848 code that expects a pointer to something that expects
4849 a direct variable. */
4853 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4854 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4856 x
= build_receiver_ref (var
, false, ctx
);
4857 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4859 else if (TREE_CONSTANT (x
))
4861 /* For reduction in SIMD loop, defer adding the
4862 initialization of the reference, because if we decide
4863 to use SIMD array for it, the initilization could cause
4865 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4869 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4871 gimple_add_tmp_var (x
);
4872 TREE_ADDRESSABLE (x
) = 1;
4873 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4879 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4880 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4881 tree al
= size_int (TYPE_ALIGN (rtype
));
4882 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4887 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4888 gimplify_assign (new_var
, x
, ilist
);
4891 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4893 else if (c_kind
== OMP_CLAUSE_REDUCTION
4894 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4902 switch (OMP_CLAUSE_CODE (c
))
4904 case OMP_CLAUSE_SHARED
:
4905 /* Ignore shared directives in teams construct. */
4906 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4908 /* Shared global vars are just accessed directly. */
4909 if (is_global_var (new_var
))
4911 /* For taskloop firstprivate/lastprivate, represented
4912 as firstprivate and shared clause on the task, new_var
4913 is the firstprivate var. */
4914 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4916 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4917 needs to be delayed until after fixup_child_record_type so
4918 that we get the correct type during the dereference. */
4919 by_ref
= use_pointer_for_field (var
, ctx
);
4920 x
= build_receiver_ref (var
, by_ref
, ctx
);
4921 SET_DECL_VALUE_EXPR (new_var
, x
);
4922 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4924 /* ??? If VAR is not passed by reference, and the variable
4925 hasn't been initialized yet, then we'll get a warning for
4926 the store into the omp_data_s structure. Ideally, we'd be
4927 able to notice this and not store anything at all, but
4928 we're generating code too early. Suppress the warning. */
4930 TREE_NO_WARNING (var
) = 1;
4933 case OMP_CLAUSE_LASTPRIVATE
:
4934 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4938 case OMP_CLAUSE_PRIVATE
:
4939 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4940 x
= build_outer_var_ref (var
, ctx
);
4941 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4943 if (is_task_ctx (ctx
))
4944 x
= build_receiver_ref (var
, false, ctx
);
4946 x
= build_outer_var_ref (var
, ctx
);
4952 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4953 (c
, unshare_expr (new_var
), x
);
4956 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4957 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4958 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4959 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4960 idx
, lane
, ivar
, lvar
))
4963 x
= lang_hooks
.decls
.omp_clause_default_ctor
4964 (c
, unshare_expr (ivar
), x
);
4966 gimplify_and_add (x
, &llist
[0]);
4969 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4972 gimple_seq tseq
= NULL
;
4975 gimplify_stmt (&dtor
, &tseq
);
4976 gimple_seq_add_seq (&llist
[1], tseq
);
4983 gimplify_and_add (nx
, ilist
);
4987 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4990 gimple_seq tseq
= NULL
;
4993 gimplify_stmt (&dtor
, &tseq
);
4994 gimple_seq_add_seq (dlist
, tseq
);
4998 case OMP_CLAUSE_LINEAR
:
4999 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
5000 goto do_firstprivate
;
5001 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
5004 x
= build_outer_var_ref (var
, ctx
);
5007 case OMP_CLAUSE_FIRSTPRIVATE
:
5008 if (is_task_ctx (ctx
))
5010 if (is_reference (var
) || is_variable_sized (var
))
5012 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
5014 || use_pointer_for_field (var
, NULL
))
5016 x
= build_receiver_ref (var
, false, ctx
);
5017 SET_DECL_VALUE_EXPR (new_var
, x
);
5018 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5023 x
= build_outer_var_ref (var
, ctx
);
5026 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5027 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5029 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5030 tree stept
= TREE_TYPE (t
);
5031 tree ct
= find_omp_clause (clauses
,
5032 OMP_CLAUSE__LOOPTEMP_
);
5034 tree l
= OMP_CLAUSE_DECL (ct
);
5035 tree n1
= fd
->loop
.n1
;
5036 tree step
= fd
->loop
.step
;
5037 tree itype
= TREE_TYPE (l
);
5038 if (POINTER_TYPE_P (itype
))
5039 itype
= signed_type_for (itype
);
5040 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5041 if (TYPE_UNSIGNED (itype
)
5042 && fd
->loop
.cond_code
== GT_EXPR
)
5043 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5044 fold_build1 (NEGATE_EXPR
, itype
, l
),
5045 fold_build1 (NEGATE_EXPR
,
5048 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5049 t
= fold_build2 (MULT_EXPR
, stept
,
5050 fold_convert (stept
, l
), t
);
5052 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5054 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5056 gimplify_and_add (x
, ilist
);
5060 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5061 x
= fold_build2 (POINTER_PLUS_EXPR
,
5062 TREE_TYPE (x
), x
, t
);
5064 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5067 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5068 || TREE_ADDRESSABLE (new_var
))
5069 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5070 idx
, lane
, ivar
, lvar
))
5072 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5074 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5075 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5076 gimplify_and_add (x
, ilist
);
5077 gimple_stmt_iterator gsi
5078 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5080 = gimple_build_assign (unshare_expr (lvar
), iv
);
5081 gsi_insert_before_without_update (&gsi
, g
,
5083 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5084 enum tree_code code
= PLUS_EXPR
;
5085 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5086 code
= POINTER_PLUS_EXPR
;
5087 g
= gimple_build_assign (iv
, code
, iv
, t
);
5088 gsi_insert_before_without_update (&gsi
, g
,
5092 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5093 (c
, unshare_expr (ivar
), x
);
5094 gimplify_and_add (x
, &llist
[0]);
5095 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5098 gimple_seq tseq
= NULL
;
5101 gimplify_stmt (&dtor
, &tseq
);
5102 gimple_seq_add_seq (&llist
[1], tseq
);
5107 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5108 (c
, unshare_expr (new_var
), x
);
5109 gimplify_and_add (x
, ilist
);
5112 case OMP_CLAUSE__LOOPTEMP_
:
5113 gcc_assert (is_taskreg_ctx (ctx
));
5114 x
= build_outer_var_ref (var
, ctx
);
5115 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5116 gimplify_and_add (x
, ilist
);
5119 case OMP_CLAUSE_COPYIN
:
5120 by_ref
= use_pointer_for_field (var
, NULL
);
5121 x
= build_receiver_ref (var
, by_ref
, ctx
);
5122 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5123 append_to_statement_list (x
, ©in_seq
);
5124 copyin_by_ref
|= by_ref
;
5127 case OMP_CLAUSE_REDUCTION
:
5128 /* OpenACC reductions are initialized using the
5129 GOACC_REDUCTION internal function. */
5130 if (is_gimple_omp_oacc (ctx
->stmt
))
5132 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5134 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5136 x
= build_outer_var_ref (var
, ctx
);
5138 if (is_reference (var
)
5139 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5141 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5142 SET_DECL_VALUE_EXPR (placeholder
, x
);
5143 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5144 tree new_vard
= new_var
;
5145 if (is_reference (var
))
5147 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5148 new_vard
= TREE_OPERAND (new_var
, 0);
5149 gcc_assert (DECL_P (new_vard
));
5152 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5153 idx
, lane
, ivar
, lvar
))
5155 if (new_vard
== new_var
)
5157 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5158 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5162 SET_DECL_VALUE_EXPR (new_vard
,
5163 build_fold_addr_expr (ivar
));
5164 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5166 x
= lang_hooks
.decls
.omp_clause_default_ctor
5167 (c
, unshare_expr (ivar
),
5168 build_outer_var_ref (var
, ctx
));
5170 gimplify_and_add (x
, &llist
[0]);
5171 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5173 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5174 lower_omp (&tseq
, ctx
);
5175 gimple_seq_add_seq (&llist
[0], tseq
);
5177 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5178 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5179 lower_omp (&tseq
, ctx
);
5180 gimple_seq_add_seq (&llist
[1], tseq
);
5181 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5182 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5183 if (new_vard
== new_var
)
5184 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5186 SET_DECL_VALUE_EXPR (new_vard
,
5187 build_fold_addr_expr (lvar
));
5188 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5193 gimplify_stmt (&dtor
, &tseq
);
5194 gimple_seq_add_seq (&llist
[1], tseq
);
5198 /* If this is a reference to constant size reduction var
5199 with placeholder, we haven't emitted the initializer
5200 for it because it is undesirable if SIMD arrays are used.
5201 But if they aren't used, we need to emit the deferred
5202 initialization now. */
5203 else if (is_reference (var
) && is_simd
)
5204 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5205 x
= lang_hooks
.decls
.omp_clause_default_ctor
5206 (c
, unshare_expr (new_var
),
5207 build_outer_var_ref (var
, ctx
));
5209 gimplify_and_add (x
, ilist
);
5210 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5212 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5213 lower_omp (&tseq
, ctx
);
5214 gimple_seq_add_seq (ilist
, tseq
);
5216 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5219 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5220 lower_omp (&tseq
, ctx
);
5221 gimple_seq_add_seq (dlist
, tseq
);
5222 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5224 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5229 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5230 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5231 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5233 /* reduction(-:var) sums up the partial results, so it
5234 acts identically to reduction(+:var). */
5235 if (code
== MINUS_EXPR
)
5238 tree new_vard
= new_var
;
5239 if (is_simd
&& is_reference (var
))
5241 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5242 new_vard
= TREE_OPERAND (new_var
, 0);
5243 gcc_assert (DECL_P (new_vard
));
5246 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5247 idx
, lane
, ivar
, lvar
))
5249 tree ref
= build_outer_var_ref (var
, ctx
);
5251 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5253 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5254 ref
= build_outer_var_ref (var
, ctx
);
5255 gimplify_assign (ref
, x
, &llist
[1]);
5257 if (new_vard
!= new_var
)
5259 SET_DECL_VALUE_EXPR (new_vard
,
5260 build_fold_addr_expr (lvar
));
5261 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5266 if (is_reference (var
) && is_simd
)
5267 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5268 gimplify_assign (new_var
, x
, ilist
);
5271 tree ref
= build_outer_var_ref (var
, ctx
);
5273 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5274 ref
= build_outer_var_ref (var
, ctx
);
5275 gimplify_assign (ref
, x
, dlist
);
5289 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5290 /* Don't want uninit warnings on simduid, it is always uninitialized,
5291 but we use it not for the value, but for the DECL_UID only. */
5292 TREE_NO_WARNING (uid
) = 1;
5294 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5295 gimple_call_set_lhs (g
, lane
);
5296 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5297 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5298 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5299 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5300 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5301 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5302 g
= gimple_build_assign (lane
, INTEGER_CST
,
5303 build_int_cst (unsigned_type_node
, 0));
5304 gimple_seq_add_stmt (ilist
, g
);
5305 for (int i
= 0; i
< 2; i
++)
5308 tree vf
= create_tmp_var (unsigned_type_node
);
5309 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5310 gimple_call_set_lhs (g
, vf
);
5311 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5312 gimple_seq_add_stmt (seq
, g
);
5313 tree t
= build_int_cst (unsigned_type_node
, 0);
5314 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5315 gimple_seq_add_stmt (seq
, g
);
5316 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5317 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5318 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5319 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5320 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5321 gimple_seq_add_seq (seq
, llist
[i
]);
5322 t
= build_int_cst (unsigned_type_node
, 1);
5323 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5324 gimple_seq_add_stmt (seq
, g
);
5325 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5326 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5327 gimple_seq_add_stmt (seq
, g
);
5328 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5332 /* The copyin sequence is not to be executed by the main thread, since
5333 that would result in self-copies. Perhaps not visible to scalars,
5334 but it certainly is to C++ operator=. */
5337 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5339 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5340 build_int_cst (TREE_TYPE (x
), 0));
5341 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5342 gimplify_and_add (x
, ilist
);
5345 /* If any copyin variable is passed by reference, we must ensure the
5346 master thread doesn't modify it before it is copied over in all
5347 threads. Similarly for variables in both firstprivate and
5348 lastprivate clauses we need to ensure the lastprivate copying
5349 happens after firstprivate copying in all threads. And similarly
5350 for UDRs if initializer expression refers to omp_orig. */
5351 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5353 /* Don't add any barrier for #pragma omp simd or
5354 #pragma omp distribute. */
5355 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5356 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5357 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5360 /* If max_vf is non-zero, then we can use only a vectorization factor
5361 up to the max_vf we chose. So stick it into the safelen clause. */
5364 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5365 OMP_CLAUSE_SAFELEN
);
5367 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5368 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5371 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5372 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5374 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5375 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5381 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5382 both parallel and workshare constructs. PREDICATE may be NULL if it's
5386 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5389 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5390 bool par_clauses
= false;
5391 tree simduid
= NULL
, lastlane
= NULL
;
5393 /* Early exit if there are no lastprivate or linear clauses. */
5394 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5395 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5396 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5397 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5399 if (clauses
== NULL
)
5401 /* If this was a workshare clause, see if it had been combined
5402 with its parallel. In that case, look for the clauses on the
5403 parallel statement itself. */
5404 if (is_parallel_ctx (ctx
))
5408 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5411 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5412 OMP_CLAUSE_LASTPRIVATE
);
5413 if (clauses
== NULL
)
5421 tree label_true
, arm1
, arm2
;
5423 label
= create_artificial_label (UNKNOWN_LOCATION
);
5424 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5425 arm1
= TREE_OPERAND (predicate
, 0);
5426 arm2
= TREE_OPERAND (predicate
, 1);
5427 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5428 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5429 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5431 gimple_seq_add_stmt (stmt_list
, stmt
);
5432 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5435 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5436 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5438 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5440 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5443 for (c
= clauses
; c
;)
5446 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5448 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5449 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5450 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5452 var
= OMP_CLAUSE_DECL (c
);
5453 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5454 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5455 && is_taskloop_ctx (ctx
))
5457 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5458 new_var
= lookup_decl (var
, ctx
->outer
);
5462 new_var
= lookup_decl (var
, ctx
);
5463 /* Avoid uninitialized warnings for lastprivate and
5464 for linear iterators. */
5466 && (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5467 || OMP_CLAUSE_LINEAR_NO_COPYIN (c
)))
5468 TREE_NO_WARNING (new_var
) = 1;
5471 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5473 tree val
= DECL_VALUE_EXPR (new_var
);
5474 if (TREE_CODE (val
) == ARRAY_REF
5475 && VAR_P (TREE_OPERAND (val
, 0))
5476 && lookup_attribute ("omp simd array",
5477 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5480 if (lastlane
== NULL
)
5482 lastlane
= create_tmp_var (unsigned_type_node
);
5484 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5486 TREE_OPERAND (val
, 1));
5487 gimple_call_set_lhs (g
, lastlane
);
5488 gimple_seq_add_stmt (stmt_list
, g
);
5490 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5491 TREE_OPERAND (val
, 0), lastlane
,
5492 NULL_TREE
, NULL_TREE
);
5496 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5497 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5499 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5500 gimple_seq_add_seq (stmt_list
,
5501 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5502 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5504 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5505 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5507 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5508 gimple_seq_add_seq (stmt_list
,
5509 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5510 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5514 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5515 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5517 gcc_checking_assert (is_taskloop_ctx (ctx
));
5518 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5520 if (is_global_var (ovar
))
5524 x
= build_outer_var_ref (var
, ctx
, true);
5525 if (is_reference (var
))
5526 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5527 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5528 gimplify_and_add (x
, stmt_list
);
5530 c
= OMP_CLAUSE_CHAIN (c
);
5531 if (c
== NULL
&& !par_clauses
)
5533 /* If this was a workshare clause, see if it had been combined
5534 with its parallel. In that case, continue looking for the
5535 clauses also on the parallel statement itself. */
5536 if (is_parallel_ctx (ctx
))
5540 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5543 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5544 OMP_CLAUSE_LASTPRIVATE
);
5550 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5553 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5554 (which might be a placeholder). INNER is true if this is an inner
5555 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5556 join markers. Generate the before-loop forking sequence in
5557 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5558 general form of these sequences is
5560 GOACC_REDUCTION_SETUP
5562 GOACC_REDUCTION_INIT
5564 GOACC_REDUCTION_FINI
5566 GOACC_REDUCTION_TEARDOWN. */
5569 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5570 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5571 gimple_seq
*join_seq
, omp_context
*ctx
)
5573 gimple_seq before_fork
= NULL
;
5574 gimple_seq after_fork
= NULL
;
5575 gimple_seq before_join
= NULL
;
5576 gimple_seq after_join
= NULL
;
5577 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5578 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5579 unsigned offset
= 0;
5581 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5582 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5584 tree orig
= OMP_CLAUSE_DECL (c
);
5585 tree var
= maybe_lookup_decl (orig
, ctx
);
5586 tree ref_to_res
= NULL_TREE
;
5587 tree incoming
, outgoing
, v1
, v2
, v3
;
5588 bool is_private
= false;
5590 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5591 if (rcode
== MINUS_EXPR
)
5593 else if (rcode
== TRUTH_ANDIF_EXPR
)
5594 rcode
= BIT_AND_EXPR
;
5595 else if (rcode
== TRUTH_ORIF_EXPR
)
5596 rcode
= BIT_IOR_EXPR
;
5597 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5602 incoming
= outgoing
= var
;
5606 /* See if an outer construct also reduces this variable. */
5607 omp_context
*outer
= ctx
;
5609 while (omp_context
*probe
= outer
->outer
)
5611 enum gimple_code type
= gimple_code (probe
->stmt
);
5616 case GIMPLE_OMP_FOR
:
5617 cls
= gimple_omp_for_clauses (probe
->stmt
);
5620 case GIMPLE_OMP_TARGET
:
5621 if (gimple_omp_target_kind (probe
->stmt
)
5622 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5625 cls
= gimple_omp_target_clauses (probe
->stmt
);
5633 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5634 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5635 && orig
== OMP_CLAUSE_DECL (cls
))
5637 incoming
= outgoing
= lookup_decl (orig
, probe
);
5638 goto has_outer_reduction
;
5640 else if ((OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_FIRSTPRIVATE
5641 || OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_PRIVATE
)
5642 && orig
== OMP_CLAUSE_DECL (cls
))
5650 /* This is the outermost construct with this reduction,
5651 see if there's a mapping for it. */
5652 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5653 && maybe_lookup_field (orig
, outer
) && !is_private
)
5655 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5656 if (is_reference (orig
))
5657 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5659 tree type
= TREE_TYPE (var
);
5660 if (POINTER_TYPE_P (type
))
5661 type
= TREE_TYPE (type
);
5664 incoming
= omp_reduction_init_op (loc
, rcode
, type
);
5668 /* Try to look at enclosing contexts for reduction var,
5669 use original if no mapping found. */
5671 omp_context
*c
= ctx
->outer
;
5674 t
= maybe_lookup_decl (orig
, c
);
5677 incoming
= outgoing
= (t
? t
: orig
);
5680 has_outer_reduction
:;
5684 ref_to_res
= integer_zero_node
;
5686 if (is_reference (orig
))
5688 tree type
= TREE_TYPE (var
);
5689 const char *id
= IDENTIFIER_POINTER (DECL_NAME (var
));
5693 tree x
= create_tmp_var (TREE_TYPE (type
), id
);
5694 gimplify_assign (var
, build_fold_addr_expr (x
), fork_seq
);
5697 v1
= create_tmp_var (type
, id
);
5698 v2
= create_tmp_var (type
, id
);
5699 v3
= create_tmp_var (type
, id
);
5701 gimplify_assign (v1
, var
, fork_seq
);
5702 gimplify_assign (v2
, var
, fork_seq
);
5703 gimplify_assign (v3
, var
, fork_seq
);
5705 var
= build_simple_mem_ref (var
);
5706 v1
= build_simple_mem_ref (v1
);
5707 v2
= build_simple_mem_ref (v2
);
5708 v3
= build_simple_mem_ref (v3
);
5709 outgoing
= build_simple_mem_ref (outgoing
);
5711 if (!TREE_CONSTANT (incoming
))
5712 incoming
= build_simple_mem_ref (incoming
);
5717 /* Determine position in reduction buffer, which may be used
5719 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5720 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5721 offset
= (offset
+ align
- 1) & ~(align
- 1);
5722 tree off
= build_int_cst (sizetype
, offset
);
5723 offset
+= GET_MODE_SIZE (mode
);
5727 init_code
= build_int_cst (integer_type_node
,
5728 IFN_GOACC_REDUCTION_INIT
);
5729 fini_code
= build_int_cst (integer_type_node
,
5730 IFN_GOACC_REDUCTION_FINI
);
5731 setup_code
= build_int_cst (integer_type_node
,
5732 IFN_GOACC_REDUCTION_SETUP
);
5733 teardown_code
= build_int_cst (integer_type_node
,
5734 IFN_GOACC_REDUCTION_TEARDOWN
);
5738 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5739 TREE_TYPE (var
), 6, setup_code
,
5740 unshare_expr (ref_to_res
),
5741 incoming
, level
, op
, off
);
5743 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5744 TREE_TYPE (var
), 6, init_code
,
5745 unshare_expr (ref_to_res
),
5746 v1
, level
, op
, off
);
5748 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5749 TREE_TYPE (var
), 6, fini_code
,
5750 unshare_expr (ref_to_res
),
5751 v2
, level
, op
, off
);
5753 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5754 TREE_TYPE (var
), 6, teardown_code
,
5755 ref_to_res
, v3
, level
, op
, off
);
5757 gimplify_assign (v1
, setup_call
, &before_fork
);
5758 gimplify_assign (v2
, init_call
, &after_fork
);
5759 gimplify_assign (v3
, fini_call
, &before_join
);
5760 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5763 /* Now stitch things together. */
5764 gimple_seq_add_seq (fork_seq
, before_fork
);
5766 gimple_seq_add_stmt (fork_seq
, fork
);
5767 gimple_seq_add_seq (fork_seq
, after_fork
);
5769 gimple_seq_add_seq (join_seq
, before_join
);
5771 gimple_seq_add_stmt (join_seq
, join
);
5772 gimple_seq_add_seq (join_seq
, after_join
);
5775 /* Generate code to implement the REDUCTION clauses. */
5778 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5780 gimple_seq sub_seq
= NULL
;
5785 /* OpenACC loop reductions are handled elsewhere. */
5786 if (is_gimple_omp_oacc (ctx
->stmt
))
5789 /* SIMD reductions are handled in lower_rec_input_clauses. */
5790 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5791 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5794 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5795 update in that case, otherwise use a lock. */
5796 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5797 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5799 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5800 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5802 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5812 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5814 tree var
, ref
, new_var
, orig_var
;
5815 enum tree_code code
;
5816 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5818 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5821 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5822 if (TREE_CODE (var
) == MEM_REF
)
5824 var
= TREE_OPERAND (var
, 0);
5825 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5826 var
= TREE_OPERAND (var
, 0);
5827 if (TREE_CODE (var
) == INDIRECT_REF
5828 || TREE_CODE (var
) == ADDR_EXPR
)
5829 var
= TREE_OPERAND (var
, 0);
5831 if (is_variable_sized (var
))
5833 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5834 var
= DECL_VALUE_EXPR (var
);
5835 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5836 var
= TREE_OPERAND (var
, 0);
5837 gcc_assert (DECL_P (var
));
5840 new_var
= lookup_decl (var
, ctx
);
5841 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5842 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5843 ref
= build_outer_var_ref (var
, ctx
);
5844 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5846 /* reduction(-:var) sums up the partial results, so it acts
5847 identically to reduction(+:var). */
5848 if (code
== MINUS_EXPR
)
5853 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5855 addr
= save_expr (addr
);
5856 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5857 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5858 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5859 gimplify_and_add (x
, stmt_seqp
);
5862 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5864 tree d
= OMP_CLAUSE_DECL (c
);
5865 tree type
= TREE_TYPE (d
);
5866 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5867 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5868 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5869 tree bias
= TREE_OPERAND (d
, 1);
5870 d
= TREE_OPERAND (d
, 0);
5871 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5873 tree b
= TREE_OPERAND (d
, 1);
5874 b
= maybe_lookup_decl (b
, ctx
);
5877 b
= TREE_OPERAND (d
, 1);
5878 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5880 if (integer_zerop (bias
))
5884 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5885 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5886 TREE_TYPE (b
), b
, bias
);
5888 d
= TREE_OPERAND (d
, 0);
5890 /* For ref build_outer_var_ref already performs this, so
5891 only new_var needs a dereference. */
5892 if (TREE_CODE (d
) == INDIRECT_REF
)
5894 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5895 gcc_assert (is_reference (var
) && var
== orig_var
);
5897 else if (TREE_CODE (d
) == ADDR_EXPR
)
5899 if (orig_var
== var
)
5901 new_var
= build_fold_addr_expr (new_var
);
5902 ref
= build_fold_addr_expr (ref
);
5907 gcc_assert (orig_var
== var
);
5908 if (is_reference (var
))
5909 ref
= build_fold_addr_expr (ref
);
5913 tree t
= maybe_lookup_decl (v
, ctx
);
5917 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5918 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5920 if (!integer_zerop (bias
))
5922 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5923 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5924 TREE_TYPE (new_var
), new_var
,
5925 unshare_expr (bias
));
5926 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5927 TREE_TYPE (ref
), ref
, bias
);
5929 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5930 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5931 tree m
= create_tmp_var (ptype
, NULL
);
5932 gimplify_assign (m
, new_var
, stmt_seqp
);
5934 m
= create_tmp_var (ptype
, NULL
);
5935 gimplify_assign (m
, ref
, stmt_seqp
);
5937 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5938 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5939 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5940 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5941 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5942 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5943 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5945 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5946 tree decl_placeholder
5947 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5948 SET_DECL_VALUE_EXPR (placeholder
, out
);
5949 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5950 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5951 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5952 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5953 gimple_seq_add_seq (&sub_seq
,
5954 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5955 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5956 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5957 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5961 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5962 out
= unshare_expr (out
);
5963 gimplify_assign (out
, x
, &sub_seq
);
5965 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5966 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5967 gimple_seq_add_stmt (&sub_seq
, g
);
5968 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5969 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5970 gimple_seq_add_stmt (&sub_seq
, g
);
5971 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5972 build_int_cst (TREE_TYPE (i
), 1));
5973 gimple_seq_add_stmt (&sub_seq
, g
);
5974 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5975 gimple_seq_add_stmt (&sub_seq
, g
);
5976 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5978 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5980 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5982 if (is_reference (var
)
5983 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5985 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5986 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5987 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5988 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5989 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5990 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5991 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5995 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5996 ref
= build_outer_var_ref (var
, ctx
);
5997 gimplify_assign (ref
, x
, &sub_seq
);
6001 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
6003 gimple_seq_add_stmt (stmt_seqp
, stmt
);
6005 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
6007 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
6009 gimple_seq_add_stmt (stmt_seqp
, stmt
);
6013 /* Generate code to implement the COPYPRIVATE clauses. */
6016 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
6021 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6023 tree var
, new_var
, ref
, x
;
6025 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6027 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
6030 var
= OMP_CLAUSE_DECL (c
);
6031 by_ref
= use_pointer_for_field (var
, NULL
);
6033 ref
= build_sender_ref (var
, ctx
);
6034 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
6037 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
6038 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
6040 gimplify_assign (ref
, x
, slist
);
6042 ref
= build_receiver_ref (var
, false, ctx
);
6045 ref
= fold_convert_loc (clause_loc
,
6046 build_pointer_type (TREE_TYPE (new_var
)),
6048 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
6050 if (is_reference (var
))
6052 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
6053 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
6054 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
6056 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
6057 gimplify_and_add (x
, rlist
);
6062 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
6063 and REDUCTION from the sender (aka parent) side. */
6066 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
6070 int ignored_looptemp
= 0;
6071 bool is_taskloop
= false;
6073 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
6074 by GOMP_taskloop. */
6075 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
6077 ignored_looptemp
= 2;
6081 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6083 tree val
, ref
, x
, var
;
6084 bool by_ref
, do_in
= false, do_out
= false;
6085 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6087 switch (OMP_CLAUSE_CODE (c
))
6089 case OMP_CLAUSE_PRIVATE
:
6090 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
6093 case OMP_CLAUSE_FIRSTPRIVATE
:
6094 case OMP_CLAUSE_COPYIN
:
6095 case OMP_CLAUSE_LASTPRIVATE
:
6096 case OMP_CLAUSE_REDUCTION
:
6098 case OMP_CLAUSE_SHARED
:
6099 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6102 case OMP_CLAUSE__LOOPTEMP_
:
6103 if (ignored_looptemp
)
6113 val
= OMP_CLAUSE_DECL (c
);
6114 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6115 && TREE_CODE (val
) == MEM_REF
)
6117 val
= TREE_OPERAND (val
, 0);
6118 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
6119 val
= TREE_OPERAND (val
, 0);
6120 if (TREE_CODE (val
) == INDIRECT_REF
6121 || TREE_CODE (val
) == ADDR_EXPR
)
6122 val
= TREE_OPERAND (val
, 0);
6123 if (is_variable_sized (val
))
6127 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6128 outer taskloop region. */
6129 omp_context
*ctx_for_o
= ctx
;
6131 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
6132 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6133 ctx_for_o
= ctx
->outer
;
6135 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
6137 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
6138 && is_global_var (var
))
6141 t
= omp_member_access_dummy_var (var
);
6144 var
= DECL_VALUE_EXPR (var
);
6145 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
6147 var
= unshare_and_remap (var
, t
, o
);
6149 var
= unshare_expr (var
);
6152 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
6154 /* Handle taskloop firstprivate/lastprivate, where the
6155 lastprivate on GIMPLE_OMP_TASK is represented as
6156 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6157 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
6158 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
6159 if (use_pointer_for_field (val
, ctx
))
6160 var
= build_fold_addr_expr (var
);
6161 gimplify_assign (x
, var
, ilist
);
6162 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
6166 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6167 || val
== OMP_CLAUSE_DECL (c
))
6168 && is_variable_sized (val
))
6170 by_ref
= use_pointer_for_field (val
, NULL
);
6172 switch (OMP_CLAUSE_CODE (c
))
6174 case OMP_CLAUSE_FIRSTPRIVATE
:
6175 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
)
6177 && is_task_ctx (ctx
))
6178 TREE_NO_WARNING (var
) = 1;
6182 case OMP_CLAUSE_PRIVATE
:
6183 case OMP_CLAUSE_COPYIN
:
6184 case OMP_CLAUSE__LOOPTEMP_
:
6188 case OMP_CLAUSE_LASTPRIVATE
:
6189 if (by_ref
|| is_reference (val
))
6191 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
6198 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
6203 case OMP_CLAUSE_REDUCTION
:
6205 if (val
== OMP_CLAUSE_DECL (c
))
6206 do_out
= !(by_ref
|| is_reference (val
));
6208 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
6217 ref
= build_sender_ref (val
, ctx
);
6218 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
6219 gimplify_assign (ref
, x
, ilist
);
6220 if (is_task_ctx (ctx
))
6221 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
6226 ref
= build_sender_ref (val
, ctx
);
6227 gimplify_assign (var
, ref
, olist
);
6232 /* Generate code to implement SHARED from the sender (aka parent)
6233 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6234 list things that got automatically shared. */
6237 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6239 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6241 if (ctx
->record_type
== NULL
)
6244 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6245 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6247 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6248 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6251 nvar
= maybe_lookup_decl (ovar
, ctx
);
6252 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6255 /* If CTX is a nested parallel directive. Find the immediately
6256 enclosing parallel or workshare construct that contains a
6257 mapping for OVAR. */
6258 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6260 t
= omp_member_access_dummy_var (var
);
6263 var
= DECL_VALUE_EXPR (var
);
6264 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6266 var
= unshare_and_remap (var
, t
, o
);
6268 var
= unshare_expr (var
);
6271 if (use_pointer_for_field (ovar
, ctx
))
6273 x
= build_sender_ref (ovar
, ctx
);
6274 var
= build_fold_addr_expr (var
);
6275 gimplify_assign (x
, var
, ilist
);
6279 x
= build_sender_ref (ovar
, ctx
);
6280 gimplify_assign (x
, var
, ilist
);
6282 if (!TREE_READONLY (var
)
6283 /* We don't need to receive a new reference to a result
6284 or parm decl. In fact we may not store to it as we will
6285 invalidate any pending RSO and generate wrong gimple
6287 && !((TREE_CODE (var
) == RESULT_DECL
6288 || TREE_CODE (var
) == PARM_DECL
)
6289 && DECL_BY_REFERENCE (var
)))
6291 x
= build_sender_ref (ovar
, ctx
);
6292 gimplify_assign (var
, x
, olist
);
6298 /* Emit an OpenACC head marker call, encapulating the partitioning and
6299 other information that must be processed by the target compiler.
6300 Return the maximum number of dimensions the associated loop might
6301 be partitioned over. */
6304 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6305 gimple_seq
*seq
, omp_context
*ctx
)
6307 unsigned levels
= 0;
6309 tree gang_static
= NULL_TREE
;
6310 auto_vec
<tree
, 5> args
;
6312 args
.quick_push (build_int_cst
6313 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6314 args
.quick_push (ddvar
);
6315 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6317 switch (OMP_CLAUSE_CODE (c
))
6319 case OMP_CLAUSE_GANG
:
6320 tag
|= OLF_DIM_GANG
;
6321 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6322 /* static:* is represented by -1, and we can ignore it, as
6323 scheduling is always static. */
6324 if (gang_static
&& integer_minus_onep (gang_static
))
6325 gang_static
= NULL_TREE
;
6329 case OMP_CLAUSE_WORKER
:
6330 tag
|= OLF_DIM_WORKER
;
6334 case OMP_CLAUSE_VECTOR
:
6335 tag
|= OLF_DIM_VECTOR
;
6339 case OMP_CLAUSE_SEQ
:
6343 case OMP_CLAUSE_AUTO
:
6347 case OMP_CLAUSE_INDEPENDENT
:
6348 tag
|= OLF_INDEPENDENT
;
6358 if (DECL_P (gang_static
))
6359 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6360 tag
|= OLF_GANG_STATIC
;
6363 /* In a parallel region, loops are implicitly INDEPENDENT. */
6364 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6365 if (!tgt
|| is_oacc_parallel (tgt
))
6366 tag
|= OLF_INDEPENDENT
;
6368 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6369 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6373 /* Ensure at least one level. */
6377 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6378 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6380 args
.quick_push (gang_static
);
6382 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6383 gimple_set_location (call
, loc
);
6384 gimple_set_lhs (call
, ddvar
);
6385 gimple_seq_add_stmt (seq
, call
);
6390 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6391 partitioning level of the enclosed region. */
6394 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6395 tree tofollow
, gimple_seq
*seq
)
6397 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6398 : IFN_UNIQUE_OACC_TAIL_MARK
);
6399 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6400 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6401 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6402 marker
, ddvar
, tofollow
);
6403 gimple_set_location (call
, loc
);
6404 gimple_set_lhs (call
, ddvar
);
6405 gimple_seq_add_stmt (seq
, call
);
6408 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6409 the loop clauses, from which we extract reductions. Initialize
6413 lower_oacc_head_tail (location_t loc
, tree clauses
,
6414 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6417 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6418 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6420 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6421 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6422 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6425 for (unsigned done
= 1; count
; count
--, done
++)
6427 gimple_seq fork_seq
= NULL
;
6428 gimple_seq join_seq
= NULL
;
6430 tree place
= build_int_cst (integer_type_node
, -1);
6431 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6432 fork_kind
, ddvar
, place
);
6433 gimple_set_location (fork
, loc
);
6434 gimple_set_lhs (fork
, ddvar
);
6436 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6437 join_kind
, ddvar
, place
);
6438 gimple_set_location (join
, loc
);
6439 gimple_set_lhs (join
, ddvar
);
6441 /* Mark the beginning of this level sequence. */
6443 lower_oacc_loop_marker (loc
, ddvar
, true,
6444 build_int_cst (integer_type_node
, count
),
6446 lower_oacc_loop_marker (loc
, ddvar
, false,
6447 build_int_cst (integer_type_node
, done
),
6450 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6451 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6453 /* Append this level to head. */
6454 gimple_seq_add_seq (head
, fork_seq
);
6455 /* Prepend it to tail. */
6456 gimple_seq_add_seq (&join_seq
, *tail
);
6462 /* Mark the end of the sequence. */
6463 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6464 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6467 /* A convenience function to build an empty GIMPLE_COND with just the
6471 gimple_build_cond_empty (tree cond
)
6473 enum tree_code pred_code
;
6476 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6477 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6480 /* Return true if a parallel REGION is within a declare target function or
6481 within a target region and is not a part of a gridified target. */
6484 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
6486 bool indirect
= false;
6487 for (region
= region
->outer
; region
; region
= region
->outer
)
6489 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6491 else if (region
->type
== GIMPLE_OMP_TARGET
)
6493 gomp_target
*tgt_stmt
6494 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
6496 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
6497 OMP_CLAUSE__GRIDDIM_
))
6504 if (lookup_attribute ("omp declare target",
6505 DECL_ATTRIBUTES (current_function_decl
)))
6511 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6514 /* Build the function calls to GOMP_parallel_start etc to actually
6515 generate the parallel operation. REGION is the parallel region
6516 being expanded. BB is the block where to insert the code. WS_ARGS
6517 will be set if this is a call to a combined parallel+workshare
6518 construct, it contains the list of additional arguments needed by
6519 the workshare construct. */
6522 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6523 gomp_parallel
*entry_stmt
,
6524 vec
<tree
, va_gc
> *ws_args
)
6526 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6527 gimple_stmt_iterator gsi
;
6529 enum built_in_function start_ix
;
6531 location_t clause_loc
;
6532 vec
<tree
, va_gc
> *args
;
6534 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6536 /* Determine what flavor of GOMP_parallel we will be
6538 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6539 if (is_combined_parallel (region
))
6541 switch (region
->inner
->type
)
6543 case GIMPLE_OMP_FOR
:
6544 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6545 switch (region
->inner
->sched_kind
)
6547 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6550 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6551 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6552 if (region
->inner
->sched_modifiers
6553 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6555 start_ix2
= 3 + region
->inner
->sched_kind
;
6560 start_ix2
= region
->inner
->sched_kind
;
6563 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6564 start_ix
= (enum built_in_function
) start_ix2
;
6566 case GIMPLE_OMP_SECTIONS
:
6567 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6574 /* By default, the value of NUM_THREADS is zero (selected at run time)
6575 and there is no conditional. */
6577 val
= build_int_cst (unsigned_type_node
, 0);
6578 flags
= build_int_cst (unsigned_type_node
, 0);
6580 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6582 cond
= OMP_CLAUSE_IF_EXPR (c
);
6584 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6587 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6588 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6591 clause_loc
= gimple_location (entry_stmt
);
6593 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6595 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6597 /* Ensure 'val' is of the correct type. */
6598 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6600 /* If we found the clause 'if (cond)', build either
6601 (cond != 0) or (cond ? val : 1u). */
6604 cond
= gimple_boolify (cond
);
6606 if (integer_zerop (val
))
6607 val
= fold_build2_loc (clause_loc
,
6608 EQ_EXPR
, unsigned_type_node
, cond
,
6609 build_int_cst (TREE_TYPE (cond
), 0));
6612 basic_block cond_bb
, then_bb
, else_bb
;
6613 edge e
, e_then
, e_else
;
6614 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6616 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6617 if (gimple_in_ssa_p (cfun
))
6619 tmp_then
= make_ssa_name (tmp_var
);
6620 tmp_else
= make_ssa_name (tmp_var
);
6621 tmp_join
= make_ssa_name (tmp_var
);
6630 e
= split_block_after_labels (bb
);
6635 then_bb
= create_empty_bb (cond_bb
);
6636 else_bb
= create_empty_bb (then_bb
);
6637 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6638 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6640 stmt
= gimple_build_cond_empty (cond
);
6641 gsi
= gsi_start_bb (cond_bb
);
6642 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6644 gsi
= gsi_start_bb (then_bb
);
6645 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6647 gsi
= gsi_start_bb (else_bb
);
6648 expand_omp_build_assign (&gsi
, tmp_else
,
6649 build_int_cst (unsigned_type_node
, 1),
6652 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6653 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6654 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6655 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6656 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6657 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6659 if (gimple_in_ssa_p (cfun
))
6661 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6662 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6663 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6669 gsi
= gsi_start_bb (bb
);
6670 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6671 false, GSI_CONTINUE_LINKING
);
6674 gsi
= gsi_last_bb (bb
);
6675 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6677 t1
= null_pointer_node
;
6679 t1
= build_fold_addr_expr (t
);
6680 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
6681 t2
= build_fold_addr_expr (child_fndecl
);
6683 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6684 args
->quick_push (t2
);
6685 args
->quick_push (t1
);
6686 args
->quick_push (val
);
6688 args
->splice (*ws_args
);
6689 args
->quick_push (flags
);
6691 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6692 builtin_decl_explicit (start_ix
), args
);
6694 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6695 false, GSI_CONTINUE_LINKING
);
6697 if (hsa_gen_requested_p ()
6698 && parallel_needs_hsa_kernel_p (region
))
6700 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
6701 hsa_register_kernel (child_cnode
);
6705 /* Insert a function call whose name is FUNC_NAME with the information from
6706 ENTRY_STMT into the basic_block BB. */
6709 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6710 vec
<tree
, va_gc
> *ws_args
)
6713 gimple_stmt_iterator gsi
;
6714 vec
<tree
, va_gc
> *args
;
6716 gcc_assert (vec_safe_length (ws_args
) == 2);
6717 tree func_name
= (*ws_args
)[0];
6718 tree grain
= (*ws_args
)[1];
6720 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6721 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6722 gcc_assert (count
!= NULL_TREE
);
6723 count
= OMP_CLAUSE_OPERAND (count
, 0);
6725 gsi
= gsi_last_bb (bb
);
6726 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6728 t1
= null_pointer_node
;
6730 t1
= build_fold_addr_expr (t
);
6731 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6733 vec_alloc (args
, 4);
6734 args
->quick_push (t2
);
6735 args
->quick_push (t1
);
6736 args
->quick_push (count
);
6737 args
->quick_push (grain
);
6738 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6740 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6741 GSI_CONTINUE_LINKING
);
6744 /* Build the function call to GOMP_task to actually
6745 generate the task operation. BB is the block where to insert the code. */
6748 expand_task_call (struct omp_region
*region
, basic_block bb
,
6749 gomp_task
*entry_stmt
)
6752 gimple_stmt_iterator gsi
;
6753 location_t loc
= gimple_location (entry_stmt
);
6755 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6757 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6758 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6759 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6760 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6761 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6762 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6765 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6766 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6767 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6769 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6770 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6771 tree num_tasks
= NULL_TREE
;
6775 gimple
*g
= last_stmt (region
->outer
->entry
);
6776 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6777 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6778 struct omp_for_data fd
;
6779 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6780 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6781 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6782 OMP_CLAUSE__LOOPTEMP_
);
6783 startvar
= OMP_CLAUSE_DECL (startvar
);
6784 endvar
= OMP_CLAUSE_DECL (endvar
);
6785 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6786 if (fd
.loop
.cond_code
== LT_EXPR
)
6787 iflags
|= GOMP_TASK_FLAG_UP
;
6788 tree tclauses
= gimple_omp_for_clauses (g
);
6789 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6791 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6794 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6797 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6798 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6801 num_tasks
= integer_zero_node
;
6803 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6804 if (ifc
== NULL_TREE
)
6805 iflags
|= GOMP_TASK_FLAG_IF
;
6806 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6807 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6808 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6811 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6813 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6815 tree cond
= boolean_true_node
;
6820 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6821 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6822 build_int_cst (unsigned_type_node
,
6824 build_int_cst (unsigned_type_node
, 0));
6825 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6829 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6834 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6835 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6836 build_int_cst (unsigned_type_node
,
6837 GOMP_TASK_FLAG_FINAL
),
6838 build_int_cst (unsigned_type_node
, 0));
6839 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6842 depend
= OMP_CLAUSE_DECL (depend
);
6844 depend
= build_int_cst (ptr_type_node
, 0);
6846 priority
= fold_convert (integer_type_node
,
6847 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6849 priority
= integer_zero_node
;
6851 gsi
= gsi_last_bb (bb
);
6852 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6854 t2
= null_pointer_node
;
6856 t2
= build_fold_addr_expr_loc (loc
, t
);
6857 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6858 t
= gimple_omp_task_copy_fn (entry_stmt
);
6860 t3
= null_pointer_node
;
6862 t3
= build_fold_addr_expr_loc (loc
, t
);
6865 t
= build_call_expr (ull
6866 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6867 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6869 gimple_omp_task_arg_size (entry_stmt
),
6870 gimple_omp_task_arg_align (entry_stmt
), flags
,
6871 num_tasks
, priority
, startvar
, endvar
, step
);
6873 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6875 gimple_omp_task_arg_size (entry_stmt
),
6876 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6879 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6880 false, GSI_CONTINUE_LINKING
);
6884 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6885 catch handler and return it. This prevents programs from violating the
6886 structured block semantics with throws. */
6889 maybe_catch_exception (gimple_seq body
)
6894 if (!flag_exceptions
)
6897 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6898 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6900 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6902 g
= gimple_build_eh_must_not_throw (decl
);
6903 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6906 return gimple_seq_alloc_with_stmt (g
);
6909 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6912 vec2chain (vec
<tree
, va_gc
> *v
)
6914 tree chain
= NULL_TREE
, t
;
6917 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6919 DECL_CHAIN (t
) = chain
;
6927 /* Remove barriers in REGION->EXIT's block. Note that this is only
6928 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6929 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6930 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6934 remove_exit_barrier (struct omp_region
*region
)
6936 gimple_stmt_iterator gsi
;
6937 basic_block exit_bb
;
6941 int any_addressable_vars
= -1;
6943 exit_bb
= region
->exit
;
6945 /* If the parallel region doesn't return, we don't have REGION->EXIT
6950 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6951 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6952 statements that can appear in between are extremely limited -- no
6953 memory operations at all. Here, we allow nothing at all, so the
6954 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6955 gsi
= gsi_last_bb (exit_bb
);
6956 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6958 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6961 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6963 gsi
= gsi_last_bb (e
->src
);
6964 if (gsi_end_p (gsi
))
6966 stmt
= gsi_stmt (gsi
);
6967 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6968 && !gimple_omp_return_nowait_p (stmt
))
6970 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6971 in many cases. If there could be tasks queued, the barrier
6972 might be needed to let the tasks run before some local
6973 variable of the parallel that the task uses as shared
6974 runs out of scope. The task can be spawned either
6975 from within current function (this would be easy to check)
6976 or from some function it calls and gets passed an address
6977 of such a variable. */
6978 if (any_addressable_vars
< 0)
6980 gomp_parallel
*parallel_stmt
6981 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6982 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6983 tree local_decls
, block
, decl
;
6986 any_addressable_vars
= 0;
6987 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6988 if (TREE_ADDRESSABLE (decl
))
6990 any_addressable_vars
= 1;
6993 for (block
= gimple_block (stmt
);
6994 !any_addressable_vars
6996 && TREE_CODE (block
) == BLOCK
;
6997 block
= BLOCK_SUPERCONTEXT (block
))
6999 for (local_decls
= BLOCK_VARS (block
);
7001 local_decls
= DECL_CHAIN (local_decls
))
7002 if (TREE_ADDRESSABLE (local_decls
))
7004 any_addressable_vars
= 1;
7007 if (block
== gimple_block (parallel_stmt
))
7011 if (!any_addressable_vars
)
7012 gimple_omp_return_set_nowait (stmt
);
7018 remove_exit_barriers (struct omp_region
*region
)
7020 if (region
->type
== GIMPLE_OMP_PARALLEL
)
7021 remove_exit_barrier (region
);
7025 region
= region
->inner
;
7026 remove_exit_barriers (region
);
7027 while (region
->next
)
7029 region
= region
->next
;
7030 remove_exit_barriers (region
);
7035 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
7036 calls. These can't be declared as const functions, but
7037 within one parallel body they are constant, so they can be
7038 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
7039 which are declared const. Similarly for task body, except
7040 that in untied task omp_get_thread_num () can change at any task
7041 scheduling point. */
7044 optimize_omp_library_calls (gimple
*entry_stmt
)
7047 gimple_stmt_iterator gsi
;
7048 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7049 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
7050 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7051 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
7052 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
7053 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
7054 OMP_CLAUSE_UNTIED
) != NULL
);
7056 FOR_EACH_BB_FN (bb
, cfun
)
7057 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
7059 gimple
*call
= gsi_stmt (gsi
);
7062 if (is_gimple_call (call
)
7063 && (decl
= gimple_call_fndecl (call
))
7064 && DECL_EXTERNAL (decl
)
7065 && TREE_PUBLIC (decl
)
7066 && DECL_INITIAL (decl
) == NULL
)
7070 if (DECL_NAME (decl
) == thr_num_id
)
7072 /* In #pragma omp task untied omp_get_thread_num () can change
7073 during the execution of the task region. */
7076 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7078 else if (DECL_NAME (decl
) == num_thr_id
)
7079 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7083 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
7084 || gimple_call_num_args (call
) != 0)
7087 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
7090 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
7091 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
7092 TREE_TYPE (TREE_TYPE (built_in
))))
7095 gimple_call_set_fndecl (call
, built_in
);
7100 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7104 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
7108 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7109 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
7112 if (TREE_CODE (t
) == ADDR_EXPR
)
7113 recompute_tree_invariant_for_addr_expr (t
);
7115 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
7119 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7122 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
7125 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
7126 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
7127 !after
, after
? GSI_CONTINUE_LINKING
7129 gimple
*stmt
= gimple_build_assign (to
, from
);
7131 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
7133 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
7134 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
7135 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
7137 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
7138 gimple_regimplify_operands (stmt
, &gsi
);
7142 /* Expand the OpenMP parallel or task directive starting at REGION. */
7145 expand_omp_taskreg (struct omp_region
*region
)
7147 basic_block entry_bb
, exit_bb
, new_bb
;
7148 struct function
*child_cfun
;
7149 tree child_fn
, block
, t
;
7150 gimple_stmt_iterator gsi
;
7151 gimple
*entry_stmt
, *stmt
;
7153 vec
<tree
, va_gc
> *ws_args
;
7155 entry_stmt
= last_stmt (region
->entry
);
7156 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
7157 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7159 entry_bb
= region
->entry
;
7160 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
7161 exit_bb
= region
->cont
;
7163 exit_bb
= region
->exit
;
7167 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
7168 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
7169 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
7172 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7173 and the inner statement contains the name of the built-in function
7175 ws_args
= region
->inner
->ws_args
;
7176 else if (is_combined_parallel (region
))
7177 ws_args
= region
->ws_args
;
7181 if (child_cfun
->cfg
)
7183 /* Due to inlining, it may happen that we have already outlined
7184 the region, in which case all we need to do is make the
7185 sub-graph unreachable and emit the parallel call. */
7186 edge entry_succ_e
, exit_succ_e
;
7188 entry_succ_e
= single_succ_edge (entry_bb
);
7190 gsi
= gsi_last_bb (entry_bb
);
7191 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
7192 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
7193 gsi_remove (&gsi
, true);
7198 exit_succ_e
= single_succ_edge (exit_bb
);
7199 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
7201 remove_edge_and_dominated_blocks (entry_succ_e
);
7205 unsigned srcidx
, dstidx
, num
;
7207 /* If the parallel region needs data sent from the parent
7208 function, then the very first statement (except possible
7209 tree profile counter updates) of the parallel body
7210 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7211 &.OMP_DATA_O is passed as an argument to the child function,
7212 we need to replace it with the argument as seen by the child
7215 In most cases, this will end up being the identity assignment
7216 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7217 a function call that has been inlined, the original PARM_DECL
7218 .OMP_DATA_I may have been converted into a different local
7219 variable. In which case, we need to keep the assignment. */
7220 if (gimple_omp_taskreg_data_arg (entry_stmt
))
7222 basic_block entry_succ_bb
7223 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
7224 : FALLTHRU_EDGE (entry_bb
)->dest
;
7226 gimple
*parcopy_stmt
= NULL
;
7228 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7232 gcc_assert (!gsi_end_p (gsi
));
7233 stmt
= gsi_stmt (gsi
);
7234 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7237 if (gimple_num_ops (stmt
) == 2)
7239 tree arg
= gimple_assign_rhs1 (stmt
);
7241 /* We're ignore the subcode because we're
7242 effectively doing a STRIP_NOPS. */
7244 if (TREE_CODE (arg
) == ADDR_EXPR
7245 && TREE_OPERAND (arg
, 0)
7246 == gimple_omp_taskreg_data_arg (entry_stmt
))
7248 parcopy_stmt
= stmt
;
7254 gcc_assert (parcopy_stmt
!= NULL
);
7255 arg
= DECL_ARGUMENTS (child_fn
);
7257 if (!gimple_in_ssa_p (cfun
))
7259 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
7260 gsi_remove (&gsi
, true);
7263 /* ?? Is setting the subcode really necessary ?? */
7264 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
7265 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7270 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
7271 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
7272 /* We'd like to set the rhs to the default def in the child_fn,
7273 but it's too early to create ssa names in the child_fn.
7274 Instead, we set the rhs to the parm. In
7275 move_sese_region_to_fn, we introduce a default def for the
7276 parm, map the parm to it's default def, and once we encounter
7277 this stmt, replace the parm with the default def. */
7278 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7279 update_stmt (parcopy_stmt
);
7283 /* Declare local variables needed in CHILD_CFUN. */
7284 block
= DECL_INITIAL (child_fn
);
7285 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7286 /* The gimplifier could record temporaries in parallel/task block
7287 rather than in containing function's local_decls chain,
7288 which would mean cgraph missed finalizing them. Do it now. */
7289 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7290 if (TREE_CODE (t
) == VAR_DECL
7292 && !DECL_EXTERNAL (t
))
7293 varpool_node::finalize_decl (t
);
7294 DECL_SAVED_TREE (child_fn
) = NULL
;
7295 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7296 gimple_set_body (child_fn
, NULL
);
7297 TREE_USED (block
) = 1;
7299 /* Reset DECL_CONTEXT on function arguments. */
7300 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7301 DECL_CONTEXT (t
) = child_fn
;
7303 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7304 so that it can be moved to the child function. */
7305 gsi
= gsi_last_bb (entry_bb
);
7306 stmt
= gsi_stmt (gsi
);
7307 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7308 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7309 e
= split_block (entry_bb
, stmt
);
7310 gsi_remove (&gsi
, true);
7313 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7314 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7317 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7318 gcc_assert (e2
->dest
== region
->exit
);
7319 remove_edge (BRANCH_EDGE (entry_bb
));
7320 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7321 gsi
= gsi_last_bb (region
->exit
);
7322 gcc_assert (!gsi_end_p (gsi
)
7323 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7324 gsi_remove (&gsi
, true);
7327 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7330 gsi
= gsi_last_bb (exit_bb
);
7331 gcc_assert (!gsi_end_p (gsi
)
7332 && (gimple_code (gsi_stmt (gsi
))
7333 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7334 stmt
= gimple_build_return (NULL
);
7335 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7336 gsi_remove (&gsi
, true);
7339 /* Move the parallel region into CHILD_CFUN. */
7341 if (gimple_in_ssa_p (cfun
))
7343 init_tree_ssa (child_cfun
);
7344 init_ssa_operands (child_cfun
);
7345 child_cfun
->gimple_df
->in_ssa_p
= true;
7349 block
= gimple_block (entry_stmt
);
7351 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7353 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7356 basic_block dest_bb
= e2
->dest
;
7358 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7360 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7362 /* When the OMP expansion process cannot guarantee an up-to-date
7363 loop tree arrange for the child function to fixup loops. */
7364 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7365 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7367 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7368 num
= vec_safe_length (child_cfun
->local_decls
);
7369 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7371 t
= (*child_cfun
->local_decls
)[srcidx
];
7372 if (DECL_CONTEXT (t
) == cfun
->decl
)
7374 if (srcidx
!= dstidx
)
7375 (*child_cfun
->local_decls
)[dstidx
] = t
;
7379 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7381 /* Inform the callgraph about the new function. */
7382 child_cfun
->curr_properties
= cfun
->curr_properties
;
7383 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7384 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7385 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7386 node
->parallelized_function
= 1;
7387 cgraph_node::add_new_function (child_fn
, true);
7389 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7390 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7392 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7393 fixed in a following pass. */
7394 push_cfun (child_cfun
);
7396 assign_assembler_name_if_neeeded (child_fn
);
7399 optimize_omp_library_calls (entry_stmt
);
7400 cgraph_edge::rebuild_edges ();
7402 /* Some EH regions might become dead, see PR34608. If
7403 pass_cleanup_cfg isn't the first pass to happen with the
7404 new child, these dead EH edges might cause problems.
7405 Clean them up now. */
7406 if (flag_exceptions
)
7409 bool changed
= false;
7411 FOR_EACH_BB_FN (bb
, cfun
)
7412 changed
|= gimple_purge_dead_eh_edges (bb
);
7414 cleanup_tree_cfg ();
7416 if (gimple_in_ssa_p (cfun
))
7417 update_ssa (TODO_update_ssa
);
7418 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7419 verify_loop_structure ();
7422 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7424 omp_any_child_fn_dumped
= true;
7425 dump_function_header (dump_file
, child_fn
, dump_flags
);
7426 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7430 /* Emit a library call to launch the children threads. */
7432 expand_cilk_for_call (new_bb
,
7433 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7434 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7435 expand_parallel_call (region
, new_bb
,
7436 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7438 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7439 if (gimple_in_ssa_p (cfun
))
7440 update_ssa (TODO_update_ssa_only_virtuals
);
7443 /* Information about members of an OpenACC collapsed loop nest. */
7445 struct oacc_collapse
7447 tree base
; /* Base value. */
7448 tree iters
; /* Number of steps. */
7449 tree step
; /* step size. */
7452 /* Helper for expand_oacc_for. Determine collapsed loop information.
7453 Fill in COUNTS array. Emit any initialization code before GSI.
7454 Return the calculated outer loop bound of BOUND_TYPE. */
7457 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7458 gimple_stmt_iterator
*gsi
,
7459 oacc_collapse
*counts
, tree bound_type
)
7461 tree total
= build_int_cst (bound_type
, 1);
7464 gcc_assert (integer_onep (fd
->loop
.step
));
7465 gcc_assert (integer_zerop (fd
->loop
.n1
));
7467 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7469 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7471 tree iter_type
= TREE_TYPE (loop
->v
);
7472 tree diff_type
= iter_type
;
7473 tree plus_type
= iter_type
;
7475 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7477 if (POINTER_TYPE_P (iter_type
))
7478 plus_type
= sizetype
;
7479 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7480 diff_type
= signed_type_for (diff_type
);
7484 tree s
= loop
->step
;
7485 bool up
= loop
->cond_code
== LT_EXPR
;
7486 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7490 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7491 true, GSI_SAME_STMT
);
7492 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7493 true, GSI_SAME_STMT
);
7495 /* Convert the step, avoiding possible unsigned->signed overflow. */
7496 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7498 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7499 s
= fold_convert (diff_type
, s
);
7501 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7502 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7503 true, GSI_SAME_STMT
);
7505 /* Determine the range, avoiding possible unsigned->signed overflow. */
7506 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7507 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7508 fold_convert (plus_type
, negating
? b
: e
),
7509 fold_convert (plus_type
, negating
? e
: b
));
7510 expr
= fold_convert (diff_type
, expr
);
7512 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7513 tree range
= force_gimple_operand_gsi
7514 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7516 /* Determine number of iterations. */
7517 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7518 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7519 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7521 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7522 true, GSI_SAME_STMT
);
7524 counts
[ix
].base
= b
;
7525 counts
[ix
].iters
= iters
;
7526 counts
[ix
].step
= s
;
7528 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7529 fold_convert (bound_type
, iters
));
7535 /* Emit initializers for collapsed loop members. IVAR is the outer
7536 loop iteration variable, from which collapsed loop iteration values
7537 are calculated. COUNTS array has been initialized by
7538 expand_oacc_collapse_inits. */
7541 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7542 gimple_stmt_iterator
*gsi
,
7543 const oacc_collapse
*counts
, tree ivar
)
7545 tree ivar_type
= TREE_TYPE (ivar
);
7547 /* The most rapidly changing iteration variable is the innermost
7549 for (int ix
= fd
->collapse
; ix
--;)
7551 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7552 const oacc_collapse
*collapse
= &counts
[ix
];
7553 tree iter_type
= TREE_TYPE (loop
->v
);
7554 tree diff_type
= TREE_TYPE (collapse
->step
);
7555 tree plus_type
= iter_type
;
7556 enum tree_code plus_code
= PLUS_EXPR
;
7559 if (POINTER_TYPE_P (iter_type
))
7561 plus_code
= POINTER_PLUS_EXPR
;
7562 plus_type
= sizetype
;
7565 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7566 fold_convert (ivar_type
, collapse
->iters
));
7567 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7569 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7570 fold_convert (plus_type
, expr
));
7571 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7572 true, GSI_SAME_STMT
);
7573 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7574 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7578 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7579 fold_convert (ivar_type
, collapse
->iters
));
7580 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7581 true, GSI_SAME_STMT
);
7587 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7588 of the combined collapse > 1 loop constructs, generate code like:
7589 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7594 count3 = (adj + N32 - N31) / STEP3;
7595 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7600 count2 = (adj + N22 - N21) / STEP2;
7601 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7606 count1 = (adj + N12 - N11) / STEP1;
7607 count = count1 * count2 * count3;
7608 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7610 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7611 of the combined loop constructs, just initialize COUNTS array
7612 from the _looptemp_ clauses. */
7614 /* NOTE: It *could* be better to moosh all of the BBs together,
7615 creating one larger BB with all the computation and the unexpected
7616 jump at the end. I.e.
7618 bool zero3, zero2, zero1, zero;
7621 count3 = (N32 - N31) /[cl] STEP3;
7623 count2 = (N22 - N21) /[cl] STEP2;
7625 count1 = (N12 - N11) /[cl] STEP1;
7626 zero = zero3 || zero2 || zero1;
7627 count = count1 * count2 * count3;
7628 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7630 After all, we expect the zero=false, and thus we expect to have to
7631 evaluate all of the comparison expressions, so short-circuiting
7632 oughtn't be a win. Since the condition isn't protecting a
7633 denominator, we're not concerned about divide-by-zero, so we can
7634 fully evaluate count even if a numerator turned out to be wrong.
7636 It seems like putting this all together would create much better
7637 scheduling opportunities, and less pressure on the chip's branch
7641 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7642 basic_block
&entry_bb
, tree
*counts
,
7643 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7644 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7645 basic_block
&l2_dom_bb
)
7647 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7651 /* Collapsed loops need work for expansion into SSA form. */
7652 gcc_assert (!gimple_in_ssa_p (cfun
));
7654 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7655 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7657 gcc_assert (fd
->ordered
== 0);
7658 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7659 isn't supposed to be handled, as the inner loop doesn't
7661 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7662 OMP_CLAUSE__LOOPTEMP_
);
7663 gcc_assert (innerc
);
7664 for (i
= 0; i
< fd
->collapse
; i
++)
7666 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7667 OMP_CLAUSE__LOOPTEMP_
);
7668 gcc_assert (innerc
);
7670 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7672 counts
[0] = NULL_TREE
;
7677 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7679 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7680 counts
[i
] = NULL_TREE
;
7681 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7682 fold_convert (itype
, fd
->loops
[i
].n1
),
7683 fold_convert (itype
, fd
->loops
[i
].n2
));
7684 if (t
&& integer_zerop (t
))
7686 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7687 counts
[i
] = build_int_cst (type
, 0);
7691 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7693 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7695 if (i
>= fd
->collapse
&& counts
[i
])
7697 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7698 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7699 fold_convert (itype
, fd
->loops
[i
].n1
),
7700 fold_convert (itype
, fd
->loops
[i
].n2
)))
7701 == NULL_TREE
|| !integer_onep (t
)))
7705 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7706 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7707 true, GSI_SAME_STMT
);
7708 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7709 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7710 true, GSI_SAME_STMT
);
7711 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7712 NULL_TREE
, NULL_TREE
);
7713 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7714 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7715 expand_omp_regimplify_p
, NULL
, NULL
)
7716 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7717 expand_omp_regimplify_p
, NULL
, NULL
))
7719 *gsi
= gsi_for_stmt (cond_stmt
);
7720 gimple_regimplify_operands (cond_stmt
, gsi
);
7722 e
= split_block (entry_bb
, cond_stmt
);
7723 basic_block
&zero_iter_bb
7724 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7725 int &first_zero_iter
7726 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7727 if (zero_iter_bb
== NULL
)
7729 gassign
*assign_stmt
;
7730 first_zero_iter
= i
;
7731 zero_iter_bb
= create_empty_bb (entry_bb
);
7732 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7733 *gsi
= gsi_after_labels (zero_iter_bb
);
7734 if (i
< fd
->collapse
)
7735 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7736 build_zero_cst (type
));
7739 counts
[i
] = create_tmp_reg (type
, ".count");
7741 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7743 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7744 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7747 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7748 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7749 e
->flags
= EDGE_TRUE_VALUE
;
7750 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7751 if (l2_dom_bb
== NULL
)
7752 l2_dom_bb
= entry_bb
;
7754 *gsi
= gsi_last_bb (entry_bb
);
7757 if (POINTER_TYPE_P (itype
))
7758 itype
= signed_type_for (itype
);
7759 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7761 t
= fold_build2 (PLUS_EXPR
, itype
,
7762 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7763 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7764 fold_convert (itype
, fd
->loops
[i
].n2
));
7765 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7766 fold_convert (itype
, fd
->loops
[i
].n1
));
7767 /* ?? We could probably use CEIL_DIV_EXPR instead of
7768 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7769 generate the same code in the end because generically we
7770 don't know that the values involved must be negative for
7772 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7773 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7774 fold_build1 (NEGATE_EXPR
, itype
, t
),
7775 fold_build1 (NEGATE_EXPR
, itype
,
7776 fold_convert (itype
,
7777 fd
->loops
[i
].step
)));
7779 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7780 fold_convert (itype
, fd
->loops
[i
].step
));
7781 t
= fold_convert (type
, t
);
7782 if (TREE_CODE (t
) == INTEGER_CST
)
7786 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7787 counts
[i
] = create_tmp_reg (type
, ".count");
7788 expand_omp_build_assign (gsi
, counts
[i
], t
);
7790 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7795 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7796 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7802 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7804 V3 = N31 + (T % count3) * STEP3;
7806 V2 = N21 + (T % count2) * STEP2;
7808 V1 = N11 + T * STEP1;
7809 if this loop doesn't have an inner loop construct combined with it.
7810 If it does have an inner loop construct combined with it and the
7811 iteration count isn't known constant, store values from counts array
7812 into its _looptemp_ temporaries instead. */
7815 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7816 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7819 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7821 /* If fd->loop.n2 is constant, then no propagation of the counts
7822 is needed, they are constant. */
7823 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7826 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7827 ? gimple_omp_taskreg_clauses (inner_stmt
)
7828 : gimple_omp_for_clauses (inner_stmt
);
7829 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7830 isn't supposed to be handled, as the inner loop doesn't
7832 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7833 gcc_assert (innerc
);
7834 for (i
= 0; i
< fd
->collapse
; i
++)
7836 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7837 OMP_CLAUSE__LOOPTEMP_
);
7838 gcc_assert (innerc
);
7841 tree tem
= OMP_CLAUSE_DECL (innerc
);
7842 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7843 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7844 false, GSI_CONTINUE_LINKING
);
7845 gassign
*stmt
= gimple_build_assign (tem
, t
);
7846 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7852 tree type
= TREE_TYPE (fd
->loop
.v
);
7853 tree tem
= create_tmp_reg (type
, ".tem");
7854 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7855 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7857 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7859 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7861 if (POINTER_TYPE_P (vtype
))
7862 itype
= signed_type_for (vtype
);
7864 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7867 t
= fold_convert (itype
, t
);
7868 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7869 fold_convert (itype
, fd
->loops
[i
].step
));
7870 if (POINTER_TYPE_P (vtype
))
7871 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7873 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7874 t
= force_gimple_operand_gsi (gsi
, t
,
7875 DECL_P (fd
->loops
[i
].v
)
7876 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7878 GSI_CONTINUE_LINKING
);
7879 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7880 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7883 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7884 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7885 false, GSI_CONTINUE_LINKING
);
7886 stmt
= gimple_build_assign (tem
, t
);
7887 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7893 /* Helper function for expand_omp_for_*. Generate code like:
7896 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7900 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7907 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7908 basic_block body_bb
)
7910 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7912 gimple_stmt_iterator gsi
;
7918 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7920 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7922 bb
= create_empty_bb (last_bb
);
7923 add_bb_to_loop (bb
, last_bb
->loop_father
);
7924 gsi
= gsi_start_bb (bb
);
7926 if (i
< fd
->collapse
- 1)
7928 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7929 e
->probability
= REG_BR_PROB_BASE
/ 8;
7931 t
= fd
->loops
[i
+ 1].n1
;
7932 t
= force_gimple_operand_gsi (&gsi
, t
,
7933 DECL_P (fd
->loops
[i
+ 1].v
)
7934 && TREE_ADDRESSABLE (fd
->loops
[i
7937 GSI_CONTINUE_LINKING
);
7938 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7939 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7944 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7946 if (POINTER_TYPE_P (vtype
))
7947 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7949 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7950 t
= force_gimple_operand_gsi (&gsi
, t
,
7951 DECL_P (fd
->loops
[i
].v
)
7952 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7953 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7954 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7955 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7959 t
= fd
->loops
[i
].n2
;
7960 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7961 false, GSI_CONTINUE_LINKING
);
7962 tree v
= fd
->loops
[i
].v
;
7963 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7964 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7965 false, GSI_CONTINUE_LINKING
);
7966 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7967 stmt
= gimple_build_cond_empty (t
);
7968 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7969 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7970 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7973 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7981 /* Expand #pragma omp ordered depend(source). */
7984 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7985 tree
*counts
, location_t loc
)
7987 enum built_in_function source_ix
7988 = fd
->iter_type
== long_integer_type_node
7989 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7991 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7992 build_fold_addr_expr (counts
[fd
->ordered
]));
7993 gimple_set_location (g
, loc
);
7994 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7997 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
8000 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
8001 tree
*counts
, tree c
, location_t loc
)
8003 auto_vec
<tree
, 10> args
;
8004 enum built_in_function sink_ix
8005 = fd
->iter_type
== long_integer_type_node
8006 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
8007 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
8009 gimple_stmt_iterator gsi2
= *gsi
;
8010 bool warned_step
= false;
8012 for (i
= 0; i
< fd
->ordered
; i
++)
8014 off
= TREE_PURPOSE (deps
);
8015 if (!integer_zerop (off
))
8017 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
8018 || fd
->loops
[i
].cond_code
== GT_EXPR
);
8019 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
8020 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8021 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
8022 "lexically later iteration");
8025 deps
= TREE_CHAIN (deps
);
8027 /* If all offsets corresponding to the collapsed loops are zero,
8028 this depend clause can be ignored. FIXME: but there is still a
8029 flush needed. We need to emit one __sync_synchronize () for it
8030 though (perhaps conditionally)? Solve this together with the
8031 conservative dependence folding optimization.
8032 if (i >= fd->collapse)
8035 deps
= OMP_CLAUSE_DECL (c
);
8037 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
8038 edge e2
= split_block_after_labels (e1
->dest
);
8040 *gsi
= gsi_after_labels (e1
->dest
);
8041 for (i
= 0; i
< fd
->ordered
; i
++)
8043 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
8044 if (POINTER_TYPE_P (itype
))
8047 deps
= TREE_CHAIN (deps
);
8048 off
= TREE_PURPOSE (deps
);
8049 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
8051 if (integer_zerop (off
))
8052 t
= boolean_true_node
;
8056 tree co
= fold_convert_loc (loc
, itype
, off
);
8057 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
8059 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8060 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
8061 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
8062 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
8065 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8066 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8067 fd
->loops
[i
].v
, co
);
8069 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8070 fd
->loops
[i
].v
, co
);
8071 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
8073 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8074 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
8077 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
8080 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8081 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
8084 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
8088 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
8092 off
= fold_convert_loc (loc
, itype
, off
);
8094 if (fd
->loops
[i
].cond_code
== LT_EXPR
8095 ? !integer_onep (fd
->loops
[i
].step
)
8096 : !integer_minus_onep (fd
->loops
[i
].step
))
8098 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8099 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
8100 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8103 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
8104 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
8105 build_int_cst (itype
, 0));
8106 if (integer_zerop (t
) && !warned_step
)
8108 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
8109 "in the iteration space");
8112 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
8116 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8122 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8123 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8124 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
8126 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8127 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
8128 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8131 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
8132 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8133 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
8134 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
8135 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8138 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
8140 if (i
< fd
->collapse
- 1)
8142 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
8147 off
= unshare_expr (off
);
8148 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
8149 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
8150 true, GSI_SAME_STMT
);
8153 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
8154 gimple_set_location (g
, loc
);
8155 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
8157 *gsi
= gsi_last_bb (e1
->src
);
8158 cond
= unshare_expr (cond
);
8159 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
8160 GSI_CONTINUE_LINKING
);
8161 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
8162 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
8163 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8164 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8165 e1
->flags
= EDGE_TRUE_VALUE
;
8166 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
8168 *gsi
= gsi_after_labels (e2
->dest
);
8171 /* Expand all #pragma omp ordered depend(source) and
8172 #pragma omp ordered depend(sink:...) constructs in the current
8173 #pragma omp for ordered(n) region. */
8176 expand_omp_ordered_source_sink (struct omp_region
*region
,
8177 struct omp_for_data
*fd
, tree
*counts
,
8178 basic_block cont_bb
)
8180 struct omp_region
*inner
;
8182 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
8183 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
8184 counts
[i
] = NULL_TREE
;
8185 else if (i
>= fd
->collapse
&& !cont_bb
)
8186 counts
[i
] = build_zero_cst (fd
->iter_type
);
8187 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
8188 && integer_onep (fd
->loops
[i
].step
))
8189 counts
[i
] = NULL_TREE
;
8191 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
8193 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
8194 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
8195 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
8197 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
8198 if (inner
->type
== GIMPLE_OMP_ORDERED
)
8200 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
8201 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
8202 location_t loc
= gimple_location (ord_stmt
);
8204 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8205 c
; c
= OMP_CLAUSE_CHAIN (c
))
8206 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
8209 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
8210 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8211 c
; c
= OMP_CLAUSE_CHAIN (c
))
8212 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8213 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
8214 gsi_remove (&gsi
, true);
8218 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8222 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
8223 basic_block cont_bb
, basic_block body_bb
,
8224 bool ordered_lastprivate
)
8226 if (fd
->ordered
== fd
->collapse
)
8231 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8232 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8234 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8235 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
8236 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
8237 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8238 size_int (i
- fd
->collapse
+ 1),
8239 NULL_TREE
, NULL_TREE
);
8240 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8245 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
8247 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
8248 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8249 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8250 fold_convert (type
, fd
->loops
[i
].n1
));
8252 expand_omp_build_assign (&gsi
, counts
[i
],
8253 build_zero_cst (fd
->iter_type
));
8254 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8255 size_int (i
- fd
->collapse
+ 1),
8256 NULL_TREE
, NULL_TREE
);
8257 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8258 if (!gsi_end_p (gsi
))
8261 gsi
= gsi_last_bb (body_bb
);
8262 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
8263 basic_block new_body
= e1
->dest
;
8264 if (body_bb
== cont_bb
)
8267 basic_block new_header
;
8268 if (EDGE_COUNT (cont_bb
->preds
) > 0)
8270 gsi
= gsi_last_bb (cont_bb
);
8271 if (POINTER_TYPE_P (type
))
8272 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8273 fold_convert (sizetype
,
8274 fd
->loops
[i
].step
));
8276 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8277 fold_convert (type
, fd
->loops
[i
].step
));
8278 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8281 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8282 build_int_cst (fd
->iter_type
, 1));
8283 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8288 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8289 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8290 t
= fold_convert (fd
->iter_type
, t
);
8291 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8292 true, GSI_SAME_STMT
);
8294 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8295 size_int (i
- fd
->collapse
+ 1),
8296 NULL_TREE
, NULL_TREE
);
8297 expand_omp_build_assign (&gsi
, aref
, t
);
8299 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8300 new_header
= e2
->dest
;
8303 new_header
= cont_bb
;
8304 gsi
= gsi_after_labels (new_header
);
8305 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8306 true, GSI_SAME_STMT
);
8308 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8309 true, NULL_TREE
, true, GSI_SAME_STMT
);
8310 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8311 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8312 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8315 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8316 e3
->flags
= EDGE_FALSE_VALUE
;
8317 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8318 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8319 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8321 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8322 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8326 struct loop
*loop
= alloc_loop ();
8327 loop
->header
= new_header
;
8328 loop
->latch
= e2
->src
;
8329 add_loop (loop
, body_bb
->loop_father
);
8333 /* If there are any lastprivate clauses and it is possible some loops
8334 might have zero iterations, ensure all the decls are initialized,
8335 otherwise we could crash evaluating C++ class iterators with lastprivate
8337 bool need_inits
= false;
8338 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8341 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8342 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8343 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8344 fold_convert (type
, fd
->loops
[i
].n1
));
8348 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8349 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8351 fold_convert (type
, fd
->loops
[i
].n1
),
8352 fold_convert (type
, fd
->loops
[i
].n2
));
8353 if (!integer_onep (this_cond
))
8361 /* A subroutine of expand_omp_for. Generate code for a parallel
8362 loop with any schedule. Given parameters:
8364 for (V = N1; V cond N2; V += STEP) BODY;
8366 where COND is "<" or ">", we generate pseudocode
8368 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8369 if (more) goto L0; else goto L3;
8376 if (V cond iend) goto L1; else goto L2;
8378 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8381 If this is a combined omp parallel loop, instead of the call to
8382 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8383 If this is gimple_omp_for_combined_p loop, then instead of assigning
8384 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8385 inner GIMPLE_OMP_FOR and V += STEP; and
8386 if (V cond iend) goto L1; else goto L2; are removed.
8388 For collapsed loops, given parameters:
8390 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8391 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8392 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8395 we generate pseudocode
8397 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8402 count3 = (adj + N32 - N31) / STEP3;
8403 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8408 count2 = (adj + N22 - N21) / STEP2;
8409 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8414 count1 = (adj + N12 - N11) / STEP1;
8415 count = count1 * count2 * count3;
8420 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8421 if (more) goto L0; else goto L3;
8425 V3 = N31 + (T % count3) * STEP3;
8427 V2 = N21 + (T % count2) * STEP2;
8429 V1 = N11 + T * STEP1;
8434 if (V < iend) goto L10; else goto L2;
8437 if (V3 cond3 N32) goto L1; else goto L11;
8441 if (V2 cond2 N22) goto L1; else goto L12;
8447 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8453 expand_omp_for_generic (struct omp_region
*region
,
8454 struct omp_for_data
*fd
,
8455 enum built_in_function start_fn
,
8456 enum built_in_function next_fn
,
8459 tree type
, istart0
, iend0
, iend
;
8460 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8461 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8462 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8463 gimple_stmt_iterator gsi
;
8464 gassign
*assign_stmt
;
8465 bool in_combined_parallel
= is_combined_parallel (region
);
8466 bool broken_loop
= region
->cont
== NULL
;
8468 tree
*counts
= NULL
;
8470 bool ordered_lastprivate
= false;
8472 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8473 gcc_assert (fd
->iter_type
== long_integer_type_node
8474 || !in_combined_parallel
);
8476 entry_bb
= region
->entry
;
8477 cont_bb
= region
->cont
;
8479 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8480 gcc_assert (broken_loop
8481 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8482 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8483 l1_bb
= single_succ (l0_bb
);
8486 l2_bb
= create_empty_bb (cont_bb
);
8487 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8488 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8490 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8494 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8495 exit_bb
= region
->exit
;
8497 gsi
= gsi_last_bb (entry_bb
);
8499 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8501 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8502 OMP_CLAUSE_LASTPRIVATE
))
8503 ordered_lastprivate
= false;
8504 if (fd
->collapse
> 1 || fd
->ordered
)
8506 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8507 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8509 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8510 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8511 zero_iter1_bb
, first_zero_iter1
,
8512 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8516 /* Some counts[i] vars might be uninitialized if
8517 some loop has zero iterations. But the body shouldn't
8518 be executed in that case, so just avoid uninit warnings. */
8519 for (i
= first_zero_iter1
;
8520 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8521 if (SSA_VAR_P (counts
[i
]))
8522 TREE_NO_WARNING (counts
[i
]) = 1;
8524 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8526 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8527 gsi
= gsi_last_bb (entry_bb
);
8528 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8529 get_immediate_dominator (CDI_DOMINATORS
,
8534 /* Some counts[i] vars might be uninitialized if
8535 some loop has zero iterations. But the body shouldn't
8536 be executed in that case, so just avoid uninit warnings. */
8537 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8538 if (SSA_VAR_P (counts
[i
]))
8539 TREE_NO_WARNING (counts
[i
]) = 1;
8541 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8545 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8547 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8548 gsi
= gsi_last_bb (entry_bb
);
8549 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8550 get_immediate_dominator
8551 (CDI_DOMINATORS
, zero_iter2_bb
));
8554 if (fd
->collapse
== 1)
8556 counts
[0] = fd
->loop
.n2
;
8557 fd
->loop
= fd
->loops
[0];
8561 type
= TREE_TYPE (fd
->loop
.v
);
8562 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8563 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8564 TREE_ADDRESSABLE (istart0
) = 1;
8565 TREE_ADDRESSABLE (iend0
) = 1;
8567 /* See if we need to bias by LLONG_MIN. */
8568 if (fd
->iter_type
== long_long_unsigned_type_node
8569 && TREE_CODE (type
) == INTEGER_TYPE
8570 && !TYPE_UNSIGNED (type
)
8571 && fd
->ordered
== 0)
8575 if (fd
->loop
.cond_code
== LT_EXPR
)
8578 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8582 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8585 if (TREE_CODE (n1
) != INTEGER_CST
8586 || TREE_CODE (n2
) != INTEGER_CST
8587 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8588 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8591 gimple_stmt_iterator gsif
= gsi
;
8594 tree arr
= NULL_TREE
;
8595 if (in_combined_parallel
)
8597 gcc_assert (fd
->ordered
== 0);
8598 /* In a combined parallel loop, emit a call to
8599 GOMP_loop_foo_next. */
8600 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8601 build_fold_addr_expr (istart0
),
8602 build_fold_addr_expr (iend0
));
8606 tree t0
, t1
, t2
, t3
, t4
;
8607 /* If this is not a combined parallel loop, emit a call to
8608 GOMP_loop_foo_start in ENTRY_BB. */
8609 t4
= build_fold_addr_expr (iend0
);
8610 t3
= build_fold_addr_expr (istart0
);
8613 t0
= build_int_cst (unsigned_type_node
,
8614 fd
->ordered
- fd
->collapse
+ 1);
8615 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8617 - fd
->collapse
+ 1),
8619 DECL_NAMELESS (arr
) = 1;
8620 TREE_ADDRESSABLE (arr
) = 1;
8621 TREE_STATIC (arr
) = 1;
8622 vec
<constructor_elt
, va_gc
> *v
;
8623 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8626 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8629 if (idx
== 0 && fd
->collapse
> 1)
8632 c
= counts
[idx
+ fd
->collapse
- 1];
8633 tree purpose
= size_int (idx
);
8634 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8635 if (TREE_CODE (c
) != INTEGER_CST
)
8636 TREE_STATIC (arr
) = 0;
8639 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8640 if (!TREE_STATIC (arr
))
8641 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8642 void_type_node
, arr
),
8643 true, NULL_TREE
, true, GSI_SAME_STMT
);
8644 t1
= build_fold_addr_expr (arr
);
8649 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8652 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8655 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8656 OMP_CLAUSE__LOOPTEMP_
);
8657 gcc_assert (innerc
);
8658 t0
= OMP_CLAUSE_DECL (innerc
);
8659 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8660 OMP_CLAUSE__LOOPTEMP_
);
8661 gcc_assert (innerc
);
8662 t1
= OMP_CLAUSE_DECL (innerc
);
8664 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8665 && TYPE_PRECISION (TREE_TYPE (t0
))
8666 != TYPE_PRECISION (fd
->iter_type
))
8668 /* Avoid casting pointers to integer of a different size. */
8669 tree itype
= signed_type_for (type
);
8670 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8671 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8675 t1
= fold_convert (fd
->iter_type
, t1
);
8676 t0
= fold_convert (fd
->iter_type
, t0
);
8680 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8681 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8684 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8688 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8689 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8691 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8692 5, t0
, t1
, t
, t3
, t4
);
8694 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8695 6, t0
, t1
, t2
, t
, t3
, t4
);
8697 else if (fd
->ordered
)
8698 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8701 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8702 5, t0
, t1
, t2
, t3
, t4
);
8710 /* The GOMP_loop_ull_*start functions have additional boolean
8711 argument, true for < loops and false for > loops.
8712 In Fortran, the C bool type can be different from
8713 boolean_type_node. */
8714 bfn_decl
= builtin_decl_explicit (start_fn
);
8715 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8716 t5
= build_int_cst (c_bool_type
,
8717 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8720 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8721 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8722 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8723 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8726 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8727 6, t5
, t0
, t1
, t2
, t3
, t4
);
8730 if (TREE_TYPE (t
) != boolean_type_node
)
8731 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8732 t
, build_int_cst (TREE_TYPE (t
), 0));
8733 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8734 true, GSI_SAME_STMT
);
8735 if (arr
&& !TREE_STATIC (arr
))
8737 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8738 TREE_THIS_VOLATILE (clobber
) = 1;
8739 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8742 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8744 /* Remove the GIMPLE_OMP_FOR statement. */
8745 gsi_remove (&gsi
, true);
8747 if (gsi_end_p (gsif
))
8748 gsif
= gsi_after_labels (gsi_bb (gsif
));
8751 /* Iteration setup for sequential loop goes in L0_BB. */
8752 tree startvar
= fd
->loop
.v
;
8753 tree endvar
= NULL_TREE
;
8755 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8757 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8758 && gimple_omp_for_kind (inner_stmt
)
8759 == GF_OMP_FOR_KIND_SIMD
);
8760 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8761 OMP_CLAUSE__LOOPTEMP_
);
8762 gcc_assert (innerc
);
8763 startvar
= OMP_CLAUSE_DECL (innerc
);
8764 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8765 OMP_CLAUSE__LOOPTEMP_
);
8766 gcc_assert (innerc
);
8767 endvar
= OMP_CLAUSE_DECL (innerc
);
8770 gsi
= gsi_start_bb (l0_bb
);
8772 if (fd
->ordered
&& fd
->collapse
== 1)
8773 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8774 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8776 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8777 if (fd
->ordered
&& fd
->collapse
== 1)
8779 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8780 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8781 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8784 t
= fold_convert (TREE_TYPE (startvar
), t
);
8785 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8791 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8792 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8793 t
= fold_convert (TREE_TYPE (startvar
), t
);
8795 t
= force_gimple_operand_gsi (&gsi
, t
,
8797 && TREE_ADDRESSABLE (startvar
),
8798 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8799 assign_stmt
= gimple_build_assign (startvar
, t
);
8800 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8803 if (fd
->ordered
&& fd
->collapse
== 1)
8804 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8805 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8807 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8808 if (fd
->ordered
&& fd
->collapse
== 1)
8810 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8811 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8812 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8815 t
= fold_convert (TREE_TYPE (startvar
), t
);
8816 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8822 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8823 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8824 t
= fold_convert (TREE_TYPE (startvar
), t
);
8826 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8827 false, GSI_CONTINUE_LINKING
);
8830 assign_stmt
= gimple_build_assign (endvar
, iend
);
8831 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8832 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8833 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8835 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8836 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8838 /* Handle linear clause adjustments. */
8839 tree itercnt
= NULL_TREE
;
8840 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8841 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8842 c
; c
= OMP_CLAUSE_CHAIN (c
))
8843 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8844 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8846 tree d
= OMP_CLAUSE_DECL (c
);
8847 bool is_ref
= is_reference (d
);
8848 tree t
= d
, a
, dest
;
8850 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8851 tree type
= TREE_TYPE (t
);
8852 if (POINTER_TYPE_P (type
))
8854 dest
= unshare_expr (t
);
8855 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8856 expand_omp_build_assign (&gsif
, v
, t
);
8857 if (itercnt
== NULL_TREE
)
8860 tree n1
= fd
->loop
.n1
;
8861 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8864 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8866 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8868 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8870 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8871 itercnt
, fd
->loop
.step
);
8872 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8874 GSI_CONTINUE_LINKING
);
8876 a
= fold_build2 (MULT_EXPR
, type
,
8877 fold_convert (type
, itercnt
),
8878 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8879 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8880 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8881 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8882 false, GSI_CONTINUE_LINKING
);
8883 assign_stmt
= gimple_build_assign (dest
, t
);
8884 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8886 if (fd
->collapse
> 1)
8887 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8891 /* Until now, counts array contained number of iterations or
8892 variable containing it for ith loop. From now on, we need
8893 those counts only for collapsed loops, and only for the 2nd
8894 till the last collapsed one. Move those one element earlier,
8895 we'll use counts[fd->collapse - 1] for the first source/sink
8896 iteration counter and so on and counts[fd->ordered]
8897 as the array holding the current counter values for
8899 if (fd
->collapse
> 1)
8900 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8904 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8906 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8908 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8909 fold_convert (type
, fd
->loops
[i
].n1
),
8910 fold_convert (type
, fd
->loops
[i
].n2
));
8911 if (!integer_onep (this_cond
))
8914 if (i
< fd
->ordered
)
8917 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8918 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8919 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8920 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8921 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8922 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8923 make_edge (cont_bb
, l1_bb
, 0);
8924 l2_bb
= create_empty_bb (cont_bb
);
8925 broken_loop
= false;
8928 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8929 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8930 ordered_lastprivate
);
8931 if (counts
[fd
->collapse
- 1])
8933 gcc_assert (fd
->collapse
== 1);
8934 gsi
= gsi_last_bb (l0_bb
);
8935 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8937 gsi
= gsi_last_bb (cont_bb
);
8938 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8939 build_int_cst (fd
->iter_type
, 1));
8940 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8941 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8942 size_zero_node
, NULL_TREE
, NULL_TREE
);
8943 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8944 t
= counts
[fd
->collapse
- 1];
8946 else if (fd
->collapse
> 1)
8950 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8951 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8952 t
= fold_convert (fd
->iter_type
, t
);
8954 gsi
= gsi_last_bb (l0_bb
);
8955 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8956 size_zero_node
, NULL_TREE
, NULL_TREE
);
8957 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8958 false, GSI_CONTINUE_LINKING
);
8959 expand_omp_build_assign (&gsi
, aref
, t
, true);
8964 /* Code to control the increment and predicate for the sequential
8965 loop goes in the CONT_BB. */
8966 gsi
= gsi_last_bb (cont_bb
);
8967 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8968 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8969 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8970 vback
= gimple_omp_continue_control_def (cont_stmt
);
8972 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8974 if (POINTER_TYPE_P (type
))
8975 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8977 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8978 t
= force_gimple_operand_gsi (&gsi
, t
,
8980 && TREE_ADDRESSABLE (vback
),
8981 NULL_TREE
, true, GSI_SAME_STMT
);
8982 assign_stmt
= gimple_build_assign (vback
, t
);
8983 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8985 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8987 if (fd
->collapse
> 1)
8991 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8992 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8993 t
= fold_convert (fd
->iter_type
, t
);
8995 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8996 counts
[fd
->ordered
], size_zero_node
,
8997 NULL_TREE
, NULL_TREE
);
8998 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8999 true, GSI_SAME_STMT
);
9000 expand_omp_build_assign (&gsi
, aref
, t
);
9003 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9004 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
9006 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9007 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9010 /* Remove GIMPLE_OMP_CONTINUE. */
9011 gsi_remove (&gsi
, true);
9013 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9014 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
9016 /* Emit code to get the next parallel iteration in L2_BB. */
9017 gsi
= gsi_start_bb (l2_bb
);
9019 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
9020 build_fold_addr_expr (istart0
),
9021 build_fold_addr_expr (iend0
));
9022 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9023 false, GSI_CONTINUE_LINKING
);
9024 if (TREE_TYPE (t
) != boolean_type_node
)
9025 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
9026 t
, build_int_cst (TREE_TYPE (t
), 0));
9027 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9028 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
9031 /* Add the loop cleanup function. */
9032 gsi
= gsi_last_bb (exit_bb
);
9033 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9034 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
9035 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9036 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
9038 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
9039 gcall
*call_stmt
= gimple_build_call (t
, 0);
9040 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9041 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
9042 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
9045 tree arr
= counts
[fd
->ordered
];
9046 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
9047 TREE_THIS_VOLATILE (clobber
) = 1;
9048 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
9051 gsi_remove (&gsi
, true);
9053 /* Connect the new blocks. */
9054 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
9055 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
9061 e
= find_edge (cont_bb
, l3_bb
);
9062 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
9064 phis
= phi_nodes (l3_bb
);
9065 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9067 gimple
*phi
= gsi_stmt (gsi
);
9068 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
9069 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
9073 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
9074 e
= find_edge (cont_bb
, l1_bb
);
9077 e
= BRANCH_EDGE (cont_bb
);
9078 gcc_assert (single_succ (e
->dest
) == l1_bb
);
9080 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9085 else if (fd
->collapse
> 1)
9088 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9091 e
->flags
= EDGE_TRUE_VALUE
;
9094 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9095 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
9099 e
= find_edge (cont_bb
, l2_bb
);
9100 e
->flags
= EDGE_FALLTHRU
;
9102 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9104 if (gimple_in_ssa_p (cfun
))
9106 /* Add phis to the outer loop that connect to the phis in the inner,
9107 original loop, and move the loop entry value of the inner phi to
9108 the loop entry value of the outer phi. */
9110 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
9112 source_location locus
;
9114 gphi
*exit_phi
= psi
.phi ();
9116 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
9117 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
9119 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
9120 edge latch_to_l1
= find_edge (latch
, l1_bb
);
9122 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
9124 tree t
= gimple_phi_result (exit_phi
);
9125 tree new_res
= copy_ssa_name (t
, NULL
);
9126 nphi
= create_phi_node (new_res
, l0_bb
);
9128 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
9129 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
9130 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
9131 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
9132 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
9134 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
9135 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
9137 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
9141 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
9142 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
9143 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
9144 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
9145 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
9146 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
9147 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
9148 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
9150 /* We enter expand_omp_for_generic with a loop. This original loop may
9151 have its own loop struct, or it may be part of an outer loop struct
9152 (which may be the fake loop). */
9153 struct loop
*outer_loop
= entry_bb
->loop_father
;
9154 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
9156 add_bb_to_loop (l2_bb
, outer_loop
);
9158 /* We've added a new loop around the original loop. Allocate the
9159 corresponding loop struct. */
9160 struct loop
*new_loop
= alloc_loop ();
9161 new_loop
->header
= l0_bb
;
9162 new_loop
->latch
= l2_bb
;
9163 add_loop (new_loop
, outer_loop
);
9165 /* Allocate a loop structure for the original loop unless we already
9167 if (!orig_loop_has_loop_struct
9168 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9170 struct loop
*orig_loop
= alloc_loop ();
9171 orig_loop
->header
= l1_bb
;
9172 /* The loop may have multiple latches. */
9173 add_loop (orig_loop
, new_loop
);
9179 /* A subroutine of expand_omp_for. Generate code for a parallel
9180 loop with static schedule and no specified chunk size. Given
9183 for (V = N1; V cond N2; V += STEP) BODY;
9185 where COND is "<" or ">", we generate pseudocode
9187 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9192 if ((__typeof (V)) -1 > 0 && cond is >)
9193 n = -(adj + N2 - N1) / -STEP;
9195 n = (adj + N2 - N1) / STEP;
9198 if (threadid < tt) goto L3; else goto L4;
9203 s0 = q * threadid + tt;
9206 if (s0 >= e0) goto L2; else goto L0;
9212 if (V cond e) goto L1;
9217 expand_omp_for_static_nochunk (struct omp_region
*region
,
9218 struct omp_for_data
*fd
,
9221 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
9222 tree type
, itype
, vmain
, vback
;
9223 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
9224 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
9226 gimple_stmt_iterator gsi
;
9228 bool broken_loop
= region
->cont
== NULL
;
9229 tree
*counts
= NULL
;
9232 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9233 if (POINTER_TYPE_P (type
))
9234 itype
= signed_type_for (type
);
9236 entry_bb
= region
->entry
;
9237 cont_bb
= region
->cont
;
9238 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9239 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9240 gcc_assert (broken_loop
9241 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
9242 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
9243 body_bb
= single_succ (seq_start_bb
);
9246 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9247 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9248 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9250 exit_bb
= region
->exit
;
9252 /* Iteration space partitioning goes in ENTRY_BB. */
9253 gsi
= gsi_last_bb (entry_bb
);
9254 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9256 if (fd
->collapse
> 1)
9258 int first_zero_iter
= -1, dummy
= -1;
9259 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9261 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9262 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9263 fin_bb
, first_zero_iter
,
9264 dummy_bb
, dummy
, l2_dom_bb
);
9267 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9268 t
= integer_one_node
;
9270 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9271 fold_convert (type
, fd
->loop
.n1
),
9272 fold_convert (type
, fd
->loop
.n2
));
9273 if (fd
->collapse
== 1
9274 && TYPE_UNSIGNED (type
)
9275 && (t
== NULL_TREE
|| !integer_onep (t
)))
9277 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9278 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9279 true, GSI_SAME_STMT
);
9280 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9281 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9282 true, GSI_SAME_STMT
);
9283 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9284 NULL_TREE
, NULL_TREE
);
9285 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9286 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9287 expand_omp_regimplify_p
, NULL
, NULL
)
9288 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9289 expand_omp_regimplify_p
, NULL
, NULL
))
9291 gsi
= gsi_for_stmt (cond_stmt
);
9292 gimple_regimplify_operands (cond_stmt
, &gsi
);
9294 ep
= split_block (entry_bb
, cond_stmt
);
9295 ep
->flags
= EDGE_TRUE_VALUE
;
9296 entry_bb
= ep
->dest
;
9297 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9298 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9299 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9300 if (gimple_in_ssa_p (cfun
))
9302 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9303 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9304 !gsi_end_p (gpi
); gsi_next (&gpi
))
9306 gphi
*phi
= gpi
.phi ();
9307 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9308 ep
, UNKNOWN_LOCATION
);
9311 gsi
= gsi_last_bb (entry_bb
);
9314 switch (gimple_omp_for_kind (fd
->for_stmt
))
9316 case GF_OMP_FOR_KIND_FOR
:
9317 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9318 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9320 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9321 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9322 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9327 nthreads
= build_call_expr (nthreads
, 0);
9328 nthreads
= fold_convert (itype
, nthreads
);
9329 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9330 true, GSI_SAME_STMT
);
9331 threadid
= build_call_expr (threadid
, 0);
9332 threadid
= fold_convert (itype
, threadid
);
9333 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9334 true, GSI_SAME_STMT
);
9338 step
= fd
->loop
.step
;
9339 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9341 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9342 OMP_CLAUSE__LOOPTEMP_
);
9343 gcc_assert (innerc
);
9344 n1
= OMP_CLAUSE_DECL (innerc
);
9345 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9346 OMP_CLAUSE__LOOPTEMP_
);
9347 gcc_assert (innerc
);
9348 n2
= OMP_CLAUSE_DECL (innerc
);
9350 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9351 true, NULL_TREE
, true, GSI_SAME_STMT
);
9352 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9353 true, NULL_TREE
, true, GSI_SAME_STMT
);
9354 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9355 true, NULL_TREE
, true, GSI_SAME_STMT
);
9357 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9358 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9359 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9360 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9361 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9362 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9363 fold_build1 (NEGATE_EXPR
, itype
, t
),
9364 fold_build1 (NEGATE_EXPR
, itype
, step
));
9366 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9367 t
= fold_convert (itype
, t
);
9368 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9370 q
= create_tmp_reg (itype
, "q");
9371 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9372 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9373 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9375 tt
= create_tmp_reg (itype
, "tt");
9376 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9377 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9378 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9380 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9381 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9382 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9384 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9385 gsi
= gsi_last_bb (second_bb
);
9386 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9388 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9390 gassign
*assign_stmt
9391 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9392 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9394 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9395 gsi
= gsi_last_bb (third_bb
);
9396 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9398 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9399 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9400 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9402 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9403 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9405 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9406 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9408 /* Remove the GIMPLE_OMP_FOR statement. */
9409 gsi_remove (&gsi
, true);
9411 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9412 gsi
= gsi_start_bb (seq_start_bb
);
9414 tree startvar
= fd
->loop
.v
;
9415 tree endvar
= NULL_TREE
;
9417 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9419 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9420 ? gimple_omp_parallel_clauses (inner_stmt
)
9421 : gimple_omp_for_clauses (inner_stmt
);
9422 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9423 gcc_assert (innerc
);
9424 startvar
= OMP_CLAUSE_DECL (innerc
);
9425 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9426 OMP_CLAUSE__LOOPTEMP_
);
9427 gcc_assert (innerc
);
9428 endvar
= OMP_CLAUSE_DECL (innerc
);
9429 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9430 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9433 for (i
= 1; i
< fd
->collapse
; i
++)
9435 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9436 OMP_CLAUSE__LOOPTEMP_
);
9437 gcc_assert (innerc
);
9439 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9440 OMP_CLAUSE__LOOPTEMP_
);
9443 /* If needed (distribute parallel for with lastprivate),
9444 propagate down the total number of iterations. */
9445 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9447 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9448 GSI_CONTINUE_LINKING
);
9449 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9450 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9454 t
= fold_convert (itype
, s0
);
9455 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9456 if (POINTER_TYPE_P (type
))
9457 t
= fold_build_pointer_plus (n1
, t
);
9459 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9460 t
= fold_convert (TREE_TYPE (startvar
), t
);
9461 t
= force_gimple_operand_gsi (&gsi
, t
,
9463 && TREE_ADDRESSABLE (startvar
),
9464 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9465 assign_stmt
= gimple_build_assign (startvar
, t
);
9466 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9468 t
= fold_convert (itype
, e0
);
9469 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9470 if (POINTER_TYPE_P (type
))
9471 t
= fold_build_pointer_plus (n1
, t
);
9473 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9474 t
= fold_convert (TREE_TYPE (startvar
), t
);
9475 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9476 false, GSI_CONTINUE_LINKING
);
9479 assign_stmt
= gimple_build_assign (endvar
, e
);
9480 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9481 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9482 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9484 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9485 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9487 /* Handle linear clause adjustments. */
9488 tree itercnt
= NULL_TREE
;
9489 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9490 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9491 c
; c
= OMP_CLAUSE_CHAIN (c
))
9492 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9493 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9495 tree d
= OMP_CLAUSE_DECL (c
);
9496 bool is_ref
= is_reference (d
);
9497 tree t
= d
, a
, dest
;
9499 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9500 if (itercnt
== NULL_TREE
)
9502 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9504 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9505 fold_convert (itype
, n1
),
9506 fold_convert (itype
, fd
->loop
.n1
));
9507 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9508 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9509 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9511 GSI_CONTINUE_LINKING
);
9516 tree type
= TREE_TYPE (t
);
9517 if (POINTER_TYPE_P (type
))
9519 a
= fold_build2 (MULT_EXPR
, type
,
9520 fold_convert (type
, itercnt
),
9521 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9522 dest
= unshare_expr (t
);
9523 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9524 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9525 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9526 false, GSI_CONTINUE_LINKING
);
9527 assign_stmt
= gimple_build_assign (dest
, t
);
9528 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9530 if (fd
->collapse
> 1)
9531 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9535 /* The code controlling the sequential loop replaces the
9536 GIMPLE_OMP_CONTINUE. */
9537 gsi
= gsi_last_bb (cont_bb
);
9538 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9539 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9540 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9541 vback
= gimple_omp_continue_control_def (cont_stmt
);
9543 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9545 if (POINTER_TYPE_P (type
))
9546 t
= fold_build_pointer_plus (vmain
, step
);
9548 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9549 t
= force_gimple_operand_gsi (&gsi
, t
,
9551 && TREE_ADDRESSABLE (vback
),
9552 NULL_TREE
, true, GSI_SAME_STMT
);
9553 assign_stmt
= gimple_build_assign (vback
, t
);
9554 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9556 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9557 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9559 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9562 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9563 gsi_remove (&gsi
, true);
9565 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9566 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9569 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9570 gsi
= gsi_last_bb (exit_bb
);
9571 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9573 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9574 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9576 gsi_remove (&gsi
, true);
9578 /* Connect all the blocks. */
9579 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9580 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9581 ep
= find_edge (entry_bb
, second_bb
);
9582 ep
->flags
= EDGE_TRUE_VALUE
;
9583 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9584 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9585 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9589 ep
= find_edge (cont_bb
, body_bb
);
9592 ep
= BRANCH_EDGE (cont_bb
);
9593 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9595 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9600 else if (fd
->collapse
> 1)
9603 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9606 ep
->flags
= EDGE_TRUE_VALUE
;
9607 find_edge (cont_bb
, fin_bb
)->flags
9608 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9611 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9612 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9613 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9615 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9616 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9617 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9618 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9620 struct loop
*loop
= body_bb
->loop_father
;
9621 if (loop
!= entry_bb
->loop_father
)
9623 gcc_assert (loop
->header
== body_bb
);
9624 gcc_assert (broken_loop
9625 || loop
->latch
== region
->cont
9626 || single_pred (loop
->latch
) == region
->cont
);
9630 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9632 loop
= alloc_loop ();
9633 loop
->header
= body_bb
;
9634 if (collapse_bb
== NULL
)
9635 loop
->latch
= cont_bb
;
9636 add_loop (loop
, body_bb
->loop_father
);
9640 /* Return phi in E->DEST with ARG on edge E. */
9643 find_phi_with_arg_on_edge (tree arg
, edge e
)
9645 basic_block bb
= e
->dest
;
9647 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9651 gphi
*phi
= gpi
.phi ();
9652 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9659 /* A subroutine of expand_omp_for. Generate code for a parallel
9660 loop with static schedule and a specified chunk size. Given
9663 for (V = N1; V cond N2; V += STEP) BODY;
9665 where COND is "<" or ">", we generate pseudocode
9667 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9672 if ((__typeof (V)) -1 > 0 && cond is >)
9673 n = -(adj + N2 - N1) / -STEP;
9675 n = (adj + N2 - N1) / STEP;
9677 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9678 here so that V is defined
9679 if the loop is not entered
9681 s0 = (trip * nthreads + threadid) * CHUNK;
9682 e0 = min(s0 + CHUNK, n);
9683 if (s0 < n) goto L1; else goto L4;
9690 if (V cond e) goto L2; else goto L3;
9698 expand_omp_for_static_chunk (struct omp_region
*region
,
9699 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9701 tree n
, s0
, e0
, e
, t
;
9702 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9703 tree type
, itype
, vmain
, vback
, vextra
;
9704 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9705 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9706 gimple_stmt_iterator gsi
;
9708 bool broken_loop
= region
->cont
== NULL
;
9709 tree
*counts
= NULL
;
9712 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9713 if (POINTER_TYPE_P (type
))
9714 itype
= signed_type_for (type
);
9716 entry_bb
= region
->entry
;
9717 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9719 iter_part_bb
= se
->dest
;
9720 cont_bb
= region
->cont
;
9721 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9722 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9723 gcc_assert (broken_loop
9724 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9725 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9726 body_bb
= single_succ (seq_start_bb
);
9729 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9730 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9731 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9732 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9734 exit_bb
= region
->exit
;
9736 /* Trip and adjustment setup goes in ENTRY_BB. */
9737 gsi
= gsi_last_bb (entry_bb
);
9738 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9740 if (fd
->collapse
> 1)
9742 int first_zero_iter
= -1, dummy
= -1;
9743 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9745 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9746 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9747 fin_bb
, first_zero_iter
,
9748 dummy_bb
, dummy
, l2_dom_bb
);
9751 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9752 t
= integer_one_node
;
9754 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9755 fold_convert (type
, fd
->loop
.n1
),
9756 fold_convert (type
, fd
->loop
.n2
));
9757 if (fd
->collapse
== 1
9758 && TYPE_UNSIGNED (type
)
9759 && (t
== NULL_TREE
|| !integer_onep (t
)))
9761 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9762 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9763 true, GSI_SAME_STMT
);
9764 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9765 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9766 true, GSI_SAME_STMT
);
9767 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9768 NULL_TREE
, NULL_TREE
);
9769 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9770 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9771 expand_omp_regimplify_p
, NULL
, NULL
)
9772 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9773 expand_omp_regimplify_p
, NULL
, NULL
))
9775 gsi
= gsi_for_stmt (cond_stmt
);
9776 gimple_regimplify_operands (cond_stmt
, &gsi
);
9778 se
= split_block (entry_bb
, cond_stmt
);
9779 se
->flags
= EDGE_TRUE_VALUE
;
9780 entry_bb
= se
->dest
;
9781 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9782 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9783 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9784 if (gimple_in_ssa_p (cfun
))
9786 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9787 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9788 !gsi_end_p (gpi
); gsi_next (&gpi
))
9790 gphi
*phi
= gpi
.phi ();
9791 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9792 se
, UNKNOWN_LOCATION
);
9795 gsi
= gsi_last_bb (entry_bb
);
9798 switch (gimple_omp_for_kind (fd
->for_stmt
))
9800 case GF_OMP_FOR_KIND_FOR
:
9801 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9802 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9804 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9805 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9806 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9811 nthreads
= build_call_expr (nthreads
, 0);
9812 nthreads
= fold_convert (itype
, nthreads
);
9813 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9814 true, GSI_SAME_STMT
);
9815 threadid
= build_call_expr (threadid
, 0);
9816 threadid
= fold_convert (itype
, threadid
);
9817 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9818 true, GSI_SAME_STMT
);
9822 step
= fd
->loop
.step
;
9823 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9825 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9826 OMP_CLAUSE__LOOPTEMP_
);
9827 gcc_assert (innerc
);
9828 n1
= OMP_CLAUSE_DECL (innerc
);
9829 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9830 OMP_CLAUSE__LOOPTEMP_
);
9831 gcc_assert (innerc
);
9832 n2
= OMP_CLAUSE_DECL (innerc
);
9834 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9835 true, NULL_TREE
, true, GSI_SAME_STMT
);
9836 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9837 true, NULL_TREE
, true, GSI_SAME_STMT
);
9838 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9839 true, NULL_TREE
, true, GSI_SAME_STMT
);
9840 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9841 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9843 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9846 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9847 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9848 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9849 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9850 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9851 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9852 fold_build1 (NEGATE_EXPR
, itype
, t
),
9853 fold_build1 (NEGATE_EXPR
, itype
, step
));
9855 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9856 t
= fold_convert (itype
, t
);
9857 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9858 true, GSI_SAME_STMT
);
9860 trip_var
= create_tmp_reg (itype
, ".trip");
9861 if (gimple_in_ssa_p (cfun
))
9863 trip_init
= make_ssa_name (trip_var
);
9864 trip_main
= make_ssa_name (trip_var
);
9865 trip_back
= make_ssa_name (trip_var
);
9869 trip_init
= trip_var
;
9870 trip_main
= trip_var
;
9871 trip_back
= trip_var
;
9874 gassign
*assign_stmt
9875 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9876 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9878 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9879 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9880 if (POINTER_TYPE_P (type
))
9881 t
= fold_build_pointer_plus (n1
, t
);
9883 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9884 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9885 true, GSI_SAME_STMT
);
9887 /* Remove the GIMPLE_OMP_FOR. */
9888 gsi_remove (&gsi
, true);
9890 gimple_stmt_iterator gsif
= gsi
;
9892 /* Iteration space partitioning goes in ITER_PART_BB. */
9893 gsi
= gsi_last_bb (iter_part_bb
);
9895 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9896 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9897 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9898 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9899 false, GSI_CONTINUE_LINKING
);
9901 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9902 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9903 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9904 false, GSI_CONTINUE_LINKING
);
9906 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9907 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9909 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9910 gsi
= gsi_start_bb (seq_start_bb
);
9912 tree startvar
= fd
->loop
.v
;
9913 tree endvar
= NULL_TREE
;
9915 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9917 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9918 ? gimple_omp_parallel_clauses (inner_stmt
)
9919 : gimple_omp_for_clauses (inner_stmt
);
9920 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9921 gcc_assert (innerc
);
9922 startvar
= OMP_CLAUSE_DECL (innerc
);
9923 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9924 OMP_CLAUSE__LOOPTEMP_
);
9925 gcc_assert (innerc
);
9926 endvar
= OMP_CLAUSE_DECL (innerc
);
9927 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9928 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9931 for (i
= 1; i
< fd
->collapse
; i
++)
9933 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9934 OMP_CLAUSE__LOOPTEMP_
);
9935 gcc_assert (innerc
);
9937 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9938 OMP_CLAUSE__LOOPTEMP_
);
9941 /* If needed (distribute parallel for with lastprivate),
9942 propagate down the total number of iterations. */
9943 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9945 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9946 GSI_CONTINUE_LINKING
);
9947 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9948 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9953 t
= fold_convert (itype
, s0
);
9954 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9955 if (POINTER_TYPE_P (type
))
9956 t
= fold_build_pointer_plus (n1
, t
);
9958 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9959 t
= fold_convert (TREE_TYPE (startvar
), t
);
9960 t
= force_gimple_operand_gsi (&gsi
, t
,
9962 && TREE_ADDRESSABLE (startvar
),
9963 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9964 assign_stmt
= gimple_build_assign (startvar
, t
);
9965 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9967 t
= fold_convert (itype
, e0
);
9968 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9969 if (POINTER_TYPE_P (type
))
9970 t
= fold_build_pointer_plus (n1
, t
);
9972 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9973 t
= fold_convert (TREE_TYPE (startvar
), t
);
9974 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9975 false, GSI_CONTINUE_LINKING
);
9978 assign_stmt
= gimple_build_assign (endvar
, e
);
9979 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9980 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9981 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9983 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9984 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9986 /* Handle linear clause adjustments. */
9987 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9988 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9989 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9990 c
; c
= OMP_CLAUSE_CHAIN (c
))
9991 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9992 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9994 tree d
= OMP_CLAUSE_DECL (c
);
9995 bool is_ref
= is_reference (d
);
9996 tree t
= d
, a
, dest
;
9998 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9999 tree type
= TREE_TYPE (t
);
10000 if (POINTER_TYPE_P (type
))
10002 dest
= unshare_expr (t
);
10003 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
10004 expand_omp_build_assign (&gsif
, v
, t
);
10005 if (itercnt
== NULL_TREE
)
10007 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10010 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
10011 fold_convert (itype
, fd
->loop
.n1
));
10012 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
10013 itercntbias
, step
);
10015 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
10018 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
10019 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
10021 GSI_CONTINUE_LINKING
);
10026 a
= fold_build2 (MULT_EXPR
, type
,
10027 fold_convert (type
, itercnt
),
10028 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
10029 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
10030 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
10031 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10032 false, GSI_CONTINUE_LINKING
);
10033 assign_stmt
= gimple_build_assign (dest
, t
);
10034 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10036 if (fd
->collapse
> 1)
10037 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10041 /* The code controlling the sequential loop goes in CONT_BB,
10042 replacing the GIMPLE_OMP_CONTINUE. */
10043 gsi
= gsi_last_bb (cont_bb
);
10044 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10045 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10046 vback
= gimple_omp_continue_control_def (cont_stmt
);
10048 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10050 if (POINTER_TYPE_P (type
))
10051 t
= fold_build_pointer_plus (vmain
, step
);
10053 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10054 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
10055 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10056 true, GSI_SAME_STMT
);
10057 assign_stmt
= gimple_build_assign (vback
, t
);
10058 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10060 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
10061 t
= build2 (EQ_EXPR
, boolean_type_node
,
10062 build_int_cst (itype
, 0),
10063 build_int_cst (itype
, 1));
10065 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10066 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10068 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10071 /* Remove GIMPLE_OMP_CONTINUE. */
10072 gsi_remove (&gsi
, true);
10074 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10075 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10077 /* Trip update code goes into TRIP_UPDATE_BB. */
10078 gsi
= gsi_start_bb (trip_update_bb
);
10080 t
= build_int_cst (itype
, 1);
10081 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
10082 assign_stmt
= gimple_build_assign (trip_back
, t
);
10083 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10086 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10087 gsi
= gsi_last_bb (exit_bb
);
10088 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
10090 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
10091 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
10093 gsi_remove (&gsi
, true);
10095 /* Connect the new blocks. */
10096 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
10097 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
10101 se
= find_edge (cont_bb
, body_bb
);
10104 se
= BRANCH_EDGE (cont_bb
);
10105 gcc_assert (single_succ (se
->dest
) == body_bb
);
10107 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10112 else if (fd
->collapse
> 1)
10115 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10118 se
->flags
= EDGE_TRUE_VALUE
;
10119 find_edge (cont_bb
, trip_update_bb
)->flags
10120 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10122 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
10125 if (gimple_in_ssa_p (cfun
))
10133 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
10135 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10136 remove arguments of the phi nodes in fin_bb. We need to create
10137 appropriate phi nodes in iter_part_bb instead. */
10138 se
= find_edge (iter_part_bb
, fin_bb
);
10139 re
= single_succ_edge (trip_update_bb
);
10140 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
10141 ene
= single_succ_edge (entry_bb
);
10143 psi
= gsi_start_phis (fin_bb
);
10144 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
10145 gsi_next (&psi
), ++i
)
10148 source_location locus
;
10151 t
= gimple_phi_result (phi
);
10152 gcc_assert (t
== redirect_edge_var_map_result (vm
));
10154 if (!single_pred_p (fin_bb
))
10155 t
= copy_ssa_name (t
, phi
);
10157 nphi
= create_phi_node (t
, iter_part_bb
);
10159 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
10160 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
10162 /* A special case -- fd->loop.v is not yet computed in
10163 iter_part_bb, we need to use vextra instead. */
10164 if (t
== fd
->loop
.v
)
10166 add_phi_arg (nphi
, t
, ene
, locus
);
10167 locus
= redirect_edge_var_map_location (vm
);
10168 tree back_arg
= redirect_edge_var_map_def (vm
);
10169 add_phi_arg (nphi
, back_arg
, re
, locus
);
10170 edge ce
= find_edge (cont_bb
, body_bb
);
10173 ce
= BRANCH_EDGE (cont_bb
);
10174 gcc_assert (single_succ (ce
->dest
) == body_bb
);
10175 ce
= single_succ_edge (ce
->dest
);
10177 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
10178 gcc_assert (inner_loop_phi
!= NULL
);
10179 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
10180 find_edge (seq_start_bb
, body_bb
), locus
);
10182 if (!single_pred_p (fin_bb
))
10183 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
10185 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
10186 redirect_edge_var_map_clear (re
);
10187 if (single_pred_p (fin_bb
))
10190 psi
= gsi_start_phis (fin_bb
);
10191 if (gsi_end_p (psi
))
10193 remove_phi_node (&psi
, false);
10196 /* Make phi node for trip. */
10197 phi
= create_phi_node (trip_main
, iter_part_bb
);
10198 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
10200 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
10205 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
10206 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
10207 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
10208 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10209 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10210 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
10211 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
10212 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10213 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10217 struct loop
*loop
= body_bb
->loop_father
;
10218 struct loop
*trip_loop
= alloc_loop ();
10219 trip_loop
->header
= iter_part_bb
;
10220 trip_loop
->latch
= trip_update_bb
;
10221 add_loop (trip_loop
, iter_part_bb
->loop_father
);
10223 if (loop
!= entry_bb
->loop_father
)
10225 gcc_assert (loop
->header
== body_bb
);
10226 gcc_assert (loop
->latch
== region
->cont
10227 || single_pred (loop
->latch
) == region
->cont
);
10228 trip_loop
->inner
= loop
;
10232 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10234 loop
= alloc_loop ();
10235 loop
->header
= body_bb
;
10236 if (collapse_bb
== NULL
)
10237 loop
->latch
= cont_bb
;
10238 add_loop (loop
, trip_loop
);
10243 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10245 for (V = N1; V cond N2; V += STEP) BODY;
10247 where COND is "<" or ">" or "!=", we generate pseudocode
10249 for (ind_var = low; ind_var < high; ind_var++)
10251 V = n1 + (ind_var * STEP)
10256 In the above pseudocode, low and high are function parameters of the
10257 child function. In the function below, we are inserting a temp.
10258 variable that will be making a call to two OMP functions that will not be
10259 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10260 with _Cilk_for). These functions are replaced with low and high
10261 by the function that handles taskreg. */
10265 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10267 bool broken_loop
= region
->cont
== NULL
;
10268 basic_block entry_bb
= region
->entry
;
10269 basic_block cont_bb
= region
->cont
;
10271 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10272 gcc_assert (broken_loop
10273 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10274 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10275 basic_block l1_bb
, l2_bb
;
10279 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10280 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10281 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10282 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10286 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10287 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10288 l2_bb
= single_succ (l1_bb
);
10290 basic_block exit_bb
= region
->exit
;
10291 basic_block l2_dom_bb
= NULL
;
10293 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10295 /* Below statements until the "tree high_val = ..." are pseudo statements
10296 used to pass information to be used by expand_omp_taskreg.
10297 low_val and high_val will be replaced by the __low and __high
10298 parameter from the child function.
10300 The call_exprs part is a place-holder, it is mainly used
10301 to distinctly identify to the top-level part that this is
10302 where we should put low and high (reasoning given in header
10306 = gimple_omp_parallel_child_fn (
10307 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10308 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10309 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10311 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10313 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10316 gcc_assert (low_val
&& high_val
);
10318 tree type
= TREE_TYPE (low_val
);
10319 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10320 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10322 /* Not needed in SSA form right now. */
10323 gcc_assert (!gimple_in_ssa_p (cfun
));
10324 if (l2_dom_bb
== NULL
)
10328 tree n2
= high_val
;
10330 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10332 /* Replace the GIMPLE_OMP_FOR statement. */
10333 gsi_replace (&gsi
, stmt
, true);
10337 /* Code to control the increment goes in the CONT_BB. */
10338 gsi
= gsi_last_bb (cont_bb
);
10339 stmt
= gsi_stmt (gsi
);
10340 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10341 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10342 build_one_cst (type
));
10344 /* Replace GIMPLE_OMP_CONTINUE. */
10345 gsi_replace (&gsi
, stmt
, true);
10348 /* Emit the condition in L1_BB. */
10349 gsi
= gsi_after_labels (l1_bb
);
10350 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10351 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10353 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10354 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10355 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10357 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10358 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10359 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10360 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10362 /* The condition is always '<' since the runtime will fill in the low
10363 and high values. */
10364 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10365 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10367 /* Remove GIMPLE_OMP_RETURN. */
10368 gsi
= gsi_last_bb (exit_bb
);
10369 gsi_remove (&gsi
, true);
10371 /* Connect the new blocks. */
10372 remove_edge (FALLTHRU_EDGE (entry_bb
));
10377 remove_edge (BRANCH_EDGE (entry_bb
));
10378 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10380 e
= BRANCH_EDGE (l1_bb
);
10381 ne
= FALLTHRU_EDGE (l1_bb
);
10382 e
->flags
= EDGE_TRUE_VALUE
;
10386 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10388 ne
= single_succ_edge (l1_bb
);
10389 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10392 ne
->flags
= EDGE_FALSE_VALUE
;
10393 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10394 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10396 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10397 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10398 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10402 struct loop
*loop
= alloc_loop ();
10403 loop
->header
= l1_bb
;
10404 loop
->latch
= cont_bb
;
10405 add_loop (loop
, l1_bb
->loop_father
);
10406 loop
->safelen
= INT_MAX
;
10409 /* Pick the correct library function based on the precision of the
10410 induction variable type. */
10411 tree lib_fun
= NULL_TREE
;
10412 if (TYPE_PRECISION (type
) == 32)
10413 lib_fun
= cilk_for_32_fndecl
;
10414 else if (TYPE_PRECISION (type
) == 64)
10415 lib_fun
= cilk_for_64_fndecl
;
10417 gcc_unreachable ();
10419 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10421 /* WS_ARGS contains the library function flavor to call:
10422 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10423 user-defined grain value. If the user does not define one, then zero
10424 is passed in by the parser. */
10425 vec_alloc (region
->ws_args
, 2);
10426 region
->ws_args
->quick_push (lib_fun
);
10427 region
->ws_args
->quick_push (fd
->chunk_size
);
10430 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10431 loop. Given parameters:
10433 for (V = N1; V cond N2; V += STEP) BODY;
10435 where COND is "<" or ">", we generate pseudocode
10443 if (V cond N2) goto L0; else goto L2;
10446 For collapsed loops, given parameters:
10448 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10449 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10450 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10453 we generate pseudocode
10459 count3 = (adj + N32 - N31) / STEP3;
10464 count2 = (adj + N22 - N21) / STEP2;
10469 count1 = (adj + N12 - N11) / STEP1;
10470 count = count1 * count2 * count3;
10480 V2 += (V3 cond3 N32) ? 0 : STEP2;
10481 V3 = (V3 cond3 N32) ? V3 : N31;
10482 V1 += (V2 cond2 N22) ? 0 : STEP1;
10483 V2 = (V2 cond2 N22) ? V2 : N21;
10485 if (V < count) goto L0; else goto L2;
10491 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10494 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10495 gimple_stmt_iterator gsi
;
10498 bool broken_loop
= region
->cont
== NULL
;
10500 tree
*counts
= NULL
;
10502 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10503 OMP_CLAUSE_SAFELEN
);
10504 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10505 OMP_CLAUSE__SIMDUID_
);
10508 type
= TREE_TYPE (fd
->loop
.v
);
10509 entry_bb
= region
->entry
;
10510 cont_bb
= region
->cont
;
10511 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10512 gcc_assert (broken_loop
10513 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10514 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10517 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10518 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10519 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10520 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10524 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10525 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10526 l2_bb
= single_succ (l1_bb
);
10528 exit_bb
= region
->exit
;
10531 gsi
= gsi_last_bb (entry_bb
);
10533 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10534 /* Not needed in SSA form right now. */
10535 gcc_assert (!gimple_in_ssa_p (cfun
));
10536 if (fd
->collapse
> 1)
10538 int first_zero_iter
= -1, dummy
= -1;
10539 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10541 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10542 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10543 zero_iter_bb
, first_zero_iter
,
10544 dummy_bb
, dummy
, l2_dom_bb
);
10546 if (l2_dom_bb
== NULL
)
10551 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10553 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10554 OMP_CLAUSE__LOOPTEMP_
);
10555 gcc_assert (innerc
);
10556 n1
= OMP_CLAUSE_DECL (innerc
);
10557 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10558 OMP_CLAUSE__LOOPTEMP_
);
10559 gcc_assert (innerc
);
10560 n2
= OMP_CLAUSE_DECL (innerc
);
10561 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10562 fold_convert (type
, n1
));
10563 if (fd
->collapse
> 1)
10566 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10572 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10573 fold_convert (type
, fd
->loop
.n1
));
10574 if (fd
->collapse
> 1)
10575 for (i
= 0; i
< fd
->collapse
; i
++)
10577 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10578 if (POINTER_TYPE_P (itype
))
10579 itype
= signed_type_for (itype
);
10580 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10581 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10585 /* Remove the GIMPLE_OMP_FOR statement. */
10586 gsi_remove (&gsi
, true);
10590 /* Code to control the increment goes in the CONT_BB. */
10591 gsi
= gsi_last_bb (cont_bb
);
10592 stmt
= gsi_stmt (gsi
);
10593 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10595 if (POINTER_TYPE_P (type
))
10596 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10598 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10599 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10601 if (fd
->collapse
> 1)
10603 i
= fd
->collapse
- 1;
10604 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10606 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10607 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10611 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10612 fd
->loops
[i
].step
);
10613 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10614 fd
->loops
[i
].v
, t
);
10616 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10618 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10620 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10621 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10622 if (POINTER_TYPE_P (itype2
))
10623 itype2
= signed_type_for (itype2
);
10624 t
= build3 (COND_EXPR
, itype2
,
10625 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10627 fold_convert (itype
, fd
->loops
[i
].n2
)),
10628 build_int_cst (itype2
, 0),
10629 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10630 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10631 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10633 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10634 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10636 t
= build3 (COND_EXPR
, itype
,
10637 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10639 fold_convert (itype
, fd
->loops
[i
].n2
)),
10641 fold_convert (itype
, fd
->loops
[i
].n1
));
10642 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10646 /* Remove GIMPLE_OMP_CONTINUE. */
10647 gsi_remove (&gsi
, true);
10650 /* Emit the condition in L1_BB. */
10651 gsi
= gsi_start_bb (l1_bb
);
10653 t
= fold_convert (type
, n2
);
10654 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10655 false, GSI_CONTINUE_LINKING
);
10656 tree v
= fd
->loop
.v
;
10657 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10658 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10659 false, GSI_CONTINUE_LINKING
);
10660 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10661 cond_stmt
= gimple_build_cond_empty (t
);
10662 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10663 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10665 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10668 gsi
= gsi_for_stmt (cond_stmt
);
10669 gimple_regimplify_operands (cond_stmt
, &gsi
);
10672 /* Remove GIMPLE_OMP_RETURN. */
10673 gsi
= gsi_last_bb (exit_bb
);
10674 gsi_remove (&gsi
, true);
10676 /* Connect the new blocks. */
10677 remove_edge (FALLTHRU_EDGE (entry_bb
));
10681 remove_edge (BRANCH_EDGE (entry_bb
));
10682 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10684 e
= BRANCH_EDGE (l1_bb
);
10685 ne
= FALLTHRU_EDGE (l1_bb
);
10686 e
->flags
= EDGE_TRUE_VALUE
;
10690 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10692 ne
= single_succ_edge (l1_bb
);
10693 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10696 ne
->flags
= EDGE_FALSE_VALUE
;
10697 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10698 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10700 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10701 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10702 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10706 struct loop
*loop
= alloc_loop ();
10707 loop
->header
= l1_bb
;
10708 loop
->latch
= cont_bb
;
10709 add_loop (loop
, l1_bb
->loop_father
);
10710 if (safelen
== NULL_TREE
)
10711 loop
->safelen
= INT_MAX
;
10714 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10715 if (TREE_CODE (safelen
) != INTEGER_CST
)
10717 else if (!tree_fits_uhwi_p (safelen
)
10718 || tree_to_uhwi (safelen
) > INT_MAX
)
10719 loop
->safelen
= INT_MAX
;
10721 loop
->safelen
= tree_to_uhwi (safelen
);
10722 if (loop
->safelen
== 1)
10727 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10728 cfun
->has_simduid_loops
= true;
10730 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10732 if ((flag_tree_loop_vectorize
10733 || (!global_options_set
.x_flag_tree_loop_vectorize
10734 && !global_options_set
.x_flag_tree_vectorize
))
10735 && flag_tree_loop_optimize
10736 && loop
->safelen
> 1)
10738 loop
->force_vectorize
= true;
10739 cfun
->has_force_vectorize_loops
= true;
10743 cfun
->has_simduid_loops
= true;
10746 /* Taskloop construct is represented after gimplification with
10747 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10748 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10749 which should just compute all the needed loop temporaries
10750 for GIMPLE_OMP_TASK. */
10753 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10754 struct omp_for_data
*fd
,
10755 gimple
*inner_stmt
)
10757 tree type
, bias
= NULL_TREE
;
10758 basic_block entry_bb
, cont_bb
, exit_bb
;
10759 gimple_stmt_iterator gsi
;
10760 gassign
*assign_stmt
;
10761 tree
*counts
= NULL
;
10764 gcc_assert (inner_stmt
);
10765 gcc_assert (region
->cont
);
10766 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10767 && gimple_omp_task_taskloop_p (inner_stmt
));
10768 type
= TREE_TYPE (fd
->loop
.v
);
10770 /* See if we need to bias by LLONG_MIN. */
10771 if (fd
->iter_type
== long_long_unsigned_type_node
10772 && TREE_CODE (type
) == INTEGER_TYPE
10773 && !TYPE_UNSIGNED (type
))
10777 if (fd
->loop
.cond_code
== LT_EXPR
)
10780 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10784 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10787 if (TREE_CODE (n1
) != INTEGER_CST
10788 || TREE_CODE (n2
) != INTEGER_CST
10789 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10790 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10793 entry_bb
= region
->entry
;
10794 cont_bb
= region
->cont
;
10795 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10796 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10797 exit_bb
= region
->exit
;
10799 gsi
= gsi_last_bb (entry_bb
);
10800 gimple
*for_stmt
= gsi_stmt (gsi
);
10801 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10802 if (fd
->collapse
> 1)
10804 int first_zero_iter
= -1, dummy
= -1;
10805 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10807 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10808 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10809 zero_iter_bb
, first_zero_iter
,
10810 dummy_bb
, dummy
, l2_dom_bb
);
10814 /* Some counts[i] vars might be uninitialized if
10815 some loop has zero iterations. But the body shouldn't
10816 be executed in that case, so just avoid uninit warnings. */
10817 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10818 if (SSA_VAR_P (counts
[i
]))
10819 TREE_NO_WARNING (counts
[i
]) = 1;
10821 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10822 entry_bb
= e
->dest
;
10823 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10824 gsi
= gsi_last_bb (entry_bb
);
10825 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10826 get_immediate_dominator (CDI_DOMINATORS
,
10834 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10835 && TYPE_PRECISION (TREE_TYPE (t0
))
10836 != TYPE_PRECISION (fd
->iter_type
))
10838 /* Avoid casting pointers to integer of a different size. */
10839 tree itype
= signed_type_for (type
);
10840 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10841 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10845 t1
= fold_convert (fd
->iter_type
, t1
);
10846 t0
= fold_convert (fd
->iter_type
, t0
);
10850 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10851 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10854 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10855 OMP_CLAUSE__LOOPTEMP_
);
10856 gcc_assert (innerc
);
10857 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10858 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10859 gcc_assert (innerc
);
10860 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10861 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10863 gcc_assert (innerc
);
10864 for (i
= 1; i
< fd
->collapse
; i
++)
10866 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10867 OMP_CLAUSE__LOOPTEMP_
);
10868 gcc_assert (innerc
);
10870 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10871 OMP_CLAUSE__LOOPTEMP_
);
10874 /* If needed (inner taskloop has lastprivate clause), propagate
10875 down the total number of iterations. */
10876 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10878 GSI_CONTINUE_LINKING
);
10879 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10880 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10884 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10885 GSI_CONTINUE_LINKING
);
10886 assign_stmt
= gimple_build_assign (startvar
, t0
);
10887 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10889 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10890 GSI_CONTINUE_LINKING
);
10891 assign_stmt
= gimple_build_assign (endvar
, t1
);
10892 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10893 if (fd
->collapse
> 1)
10894 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10896 /* Remove the GIMPLE_OMP_FOR statement. */
10897 gsi
= gsi_for_stmt (for_stmt
);
10898 gsi_remove (&gsi
, true);
10900 gsi
= gsi_last_bb (cont_bb
);
10901 gsi_remove (&gsi
, true);
10903 gsi
= gsi_last_bb (exit_bb
);
10904 gsi_remove (&gsi
, true);
10906 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10907 remove_edge (BRANCH_EDGE (entry_bb
));
10908 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10909 remove_edge (BRANCH_EDGE (cont_bb
));
10910 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10911 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10912 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10915 /* Taskloop construct is represented after gimplification with
10916 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10917 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10918 GOMP_taskloop{,_ull} function arranges for each task to be given just
10919 a single range of iterations. */
10922 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10923 struct omp_for_data
*fd
,
10924 gimple
*inner_stmt
)
10926 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10927 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10928 basic_block fin_bb
;
10929 gimple_stmt_iterator gsi
;
10931 bool broken_loop
= region
->cont
== NULL
;
10932 tree
*counts
= NULL
;
10935 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10936 if (POINTER_TYPE_P (type
))
10937 itype
= signed_type_for (type
);
10939 /* See if we need to bias by LLONG_MIN. */
10940 if (fd
->iter_type
== long_long_unsigned_type_node
10941 && TREE_CODE (type
) == INTEGER_TYPE
10942 && !TYPE_UNSIGNED (type
))
10946 if (fd
->loop
.cond_code
== LT_EXPR
)
10949 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10953 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10956 if (TREE_CODE (n1
) != INTEGER_CST
10957 || TREE_CODE (n2
) != INTEGER_CST
10958 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10959 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10962 entry_bb
= region
->entry
;
10963 cont_bb
= region
->cont
;
10964 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10965 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10966 gcc_assert (broken_loop
10967 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10968 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10971 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10972 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10974 exit_bb
= region
->exit
;
10976 /* Iteration space partitioning goes in ENTRY_BB. */
10977 gsi
= gsi_last_bb (entry_bb
);
10978 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10980 if (fd
->collapse
> 1)
10982 int first_zero_iter
= -1, dummy
= -1;
10983 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10985 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10986 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10987 fin_bb
, first_zero_iter
,
10988 dummy_bb
, dummy
, l2_dom_bb
);
10992 t
= integer_one_node
;
10994 step
= fd
->loop
.step
;
10995 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10996 OMP_CLAUSE__LOOPTEMP_
);
10997 gcc_assert (innerc
);
10998 n1
= OMP_CLAUSE_DECL (innerc
);
10999 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
11000 gcc_assert (innerc
);
11001 n2
= OMP_CLAUSE_DECL (innerc
);
11004 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
11005 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
11007 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
11008 true, NULL_TREE
, true, GSI_SAME_STMT
);
11009 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
11010 true, NULL_TREE
, true, GSI_SAME_STMT
);
11011 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
11012 true, NULL_TREE
, true, GSI_SAME_STMT
);
11014 tree startvar
= fd
->loop
.v
;
11015 tree endvar
= NULL_TREE
;
11017 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11019 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
11020 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
11021 gcc_assert (innerc
);
11022 startvar
= OMP_CLAUSE_DECL (innerc
);
11023 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
11024 OMP_CLAUSE__LOOPTEMP_
);
11025 gcc_assert (innerc
);
11026 endvar
= OMP_CLAUSE_DECL (innerc
);
11028 t
= fold_convert (TREE_TYPE (startvar
), n1
);
11029 t
= force_gimple_operand_gsi (&gsi
, t
,
11031 && TREE_ADDRESSABLE (startvar
),
11032 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
11033 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
11034 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11036 t
= fold_convert (TREE_TYPE (startvar
), n2
);
11037 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
11038 false, GSI_CONTINUE_LINKING
);
11041 assign_stmt
= gimple_build_assign (endvar
, e
);
11042 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11043 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
11044 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
11046 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
11047 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11049 if (fd
->collapse
> 1)
11050 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
11054 /* The code controlling the sequential loop replaces the
11055 GIMPLE_OMP_CONTINUE. */
11056 gsi
= gsi_last_bb (cont_bb
);
11057 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11058 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
11059 vmain
= gimple_omp_continue_control_use (cont_stmt
);
11060 vback
= gimple_omp_continue_control_def (cont_stmt
);
11062 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
11064 if (POINTER_TYPE_P (type
))
11065 t
= fold_build_pointer_plus (vmain
, step
);
11067 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
11068 t
= force_gimple_operand_gsi (&gsi
, t
,
11070 && TREE_ADDRESSABLE (vback
),
11071 NULL_TREE
, true, GSI_SAME_STMT
);
11072 assign_stmt
= gimple_build_assign (vback
, t
);
11073 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
11075 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
11076 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
11078 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
11081 /* Remove the GIMPLE_OMP_CONTINUE statement. */
11082 gsi_remove (&gsi
, true);
11084 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
11085 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
11088 /* Remove the GIMPLE_OMP_FOR statement. */
11089 gsi
= gsi_for_stmt (fd
->for_stmt
);
11090 gsi_remove (&gsi
, true);
11092 /* Remove the GIMPLE_OMP_RETURN statement. */
11093 gsi
= gsi_last_bb (exit_bb
);
11094 gsi_remove (&gsi
, true);
11096 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
11098 remove_edge (BRANCH_EDGE (entry_bb
));
11101 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
11102 region
->outer
->cont
= NULL
;
11105 /* Connect all the blocks. */
11108 ep
= find_edge (cont_bb
, body_bb
);
11109 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11114 else if (fd
->collapse
> 1)
11117 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
11120 ep
->flags
= EDGE_TRUE_VALUE
;
11121 find_edge (cont_bb
, fin_bb
)->flags
11122 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
11125 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
11126 recompute_dominator (CDI_DOMINATORS
, body_bb
));
11128 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
11129 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
11131 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
11133 struct loop
*loop
= alloc_loop ();
11134 loop
->header
= body_bb
;
11135 if (collapse_bb
== NULL
)
11136 loop
->latch
= cont_bb
;
11137 add_loop (loop
, body_bb
->loop_father
);
11141 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11142 partitioned loop. The lowering here is abstracted, in that the
11143 loop parameters are passed through internal functions, which are
11144 further lowered by oacc_device_lower, once we get to the target
11145 compiler. The loop is of the form:
11147 for (V = B; V LTGT E; V += S) {BODY}
11149 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11150 (constant 0 for no chunking) and we will have a GWV partitioning
11151 mask, specifying dimensions over which the loop is to be
11152 partitioned (see note below). We generate code that looks like:
11154 <entry_bb> [incoming FALL->body, BRANCH->exit]
11155 typedef signedintify (typeof (V)) T; // underlying signed integral type
11158 T DIR = LTGT == '<' ? +1 : -1;
11159 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11160 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11162 <head_bb> [created by splitting end of entry_bb]
11163 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11164 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11165 if (!(offset LTGT bound)) goto bottom_bb;
11167 <body_bb> [incoming]
11171 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11173 if (offset LTGT bound) goto body_bb; [*]
11175 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11177 if (chunk < chunk_max) goto head_bb;
11179 <exit_bb> [incoming]
11180 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11182 [*] Needed if V live at end of loop
11184 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11185 transition, and will be specified by a more general mechanism shortly.
11189 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
11191 tree v
= fd
->loop
.v
;
11192 enum tree_code cond_code
= fd
->loop
.cond_code
;
11193 enum tree_code plus_code
= PLUS_EXPR
;
11195 tree chunk_size
= integer_minus_one_node
;
11196 tree gwv
= integer_zero_node
;
11197 tree iter_type
= TREE_TYPE (v
);
11198 tree diff_type
= iter_type
;
11199 tree plus_type
= iter_type
;
11200 struct oacc_collapse
*counts
= NULL
;
11202 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
11203 == GF_OMP_FOR_KIND_OACC_LOOP
);
11204 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
11205 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
11207 if (POINTER_TYPE_P (iter_type
))
11209 plus_code
= POINTER_PLUS_EXPR
;
11210 plus_type
= sizetype
;
11212 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
11213 diff_type
= signed_type_for (diff_type
);
11215 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
11216 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
11217 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
11218 basic_block bottom_bb
= NULL
;
11220 /* entry_bb has two sucessors; the branch edge is to the exit
11221 block, fallthrough edge to body. */
11222 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
11223 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
11225 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11226 body_bb, or to a block whose only successor is the body_bb. Its
11227 fallthrough successor is the final block (same as the branch
11228 successor of the entry_bb). */
11231 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
11232 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
11234 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
11235 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
11238 gcc_assert (!gimple_in_ssa_p (cfun
));
11240 /* The exit block only has entry_bb and cont_bb as predecessors. */
11241 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
11244 tree chunk_max
= NULL_TREE
;
11245 tree bound
, offset
;
11246 tree step
= create_tmp_var (diff_type
, ".step");
11247 bool up
= cond_code
== LT_EXPR
;
11248 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
11249 bool chunking
= !gimple_in_ssa_p (cfun
);;
11252 /* SSA instances. */
11253 tree offset_incr
= NULL_TREE
;
11254 tree offset_init
= NULL_TREE
;
11256 gimple_stmt_iterator gsi
;
11262 edge split
, be
, fte
;
11264 /* Split the end of entry_bb to create head_bb. */
11265 split
= split_block (entry_bb
, last_stmt (entry_bb
));
11266 basic_block head_bb
= split
->dest
;
11267 entry_bb
= split
->src
;
11269 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11270 gsi
= gsi_last_bb (entry_bb
);
11271 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
11272 loc
= gimple_location (for_stmt
);
11274 if (gimple_in_ssa_p (cfun
))
11276 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11277 gcc_assert (integer_zerop (fd
->loop
.n1
));
11278 /* The SSA parallelizer does gang parallelism. */
11279 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11282 if (fd
->collapse
> 1)
11284 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11285 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11286 TREE_TYPE (fd
->loop
.n2
));
11288 if (SSA_VAR_P (fd
->loop
.n2
))
11290 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11291 true, GSI_SAME_STMT
);
11292 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11293 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11298 tree b
= fd
->loop
.n1
;
11299 tree e
= fd
->loop
.n2
;
11300 tree s
= fd
->loop
.step
;
11302 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11303 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11305 /* Convert the step, avoiding possible unsigned->signed overflow. */
11306 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11308 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11309 s
= fold_convert (diff_type
, s
);
11311 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11312 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11315 chunk_size
= integer_zero_node
;
11316 expr
= fold_convert (diff_type
, chunk_size
);
11317 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11318 NULL_TREE
, true, GSI_SAME_STMT
);
11319 /* Determine the range, avoiding possible unsigned->signed overflow. */
11320 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11321 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11322 fold_convert (plus_type
, negating
? b
: e
),
11323 fold_convert (plus_type
, negating
? e
: b
));
11324 expr
= fold_convert (diff_type
, expr
);
11326 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11327 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11328 NULL_TREE
, true, GSI_SAME_STMT
);
11330 chunk_no
= build_int_cst (diff_type
, 0);
11333 gcc_assert (!gimple_in_ssa_p (cfun
));
11336 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11337 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11339 ass
= gimple_build_assign (chunk_no
, expr
);
11340 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11342 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11343 build_int_cst (integer_type_node
,
11344 IFN_GOACC_LOOP_CHUNKS
),
11345 dir
, range
, s
, chunk_size
, gwv
);
11346 gimple_call_set_lhs (call
, chunk_max
);
11347 gimple_set_location (call
, loc
);
11348 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11351 chunk_size
= chunk_no
;
11353 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11354 build_int_cst (integer_type_node
,
11355 IFN_GOACC_LOOP_STEP
),
11356 dir
, range
, s
, chunk_size
, gwv
);
11357 gimple_call_set_lhs (call
, step
);
11358 gimple_set_location (call
, loc
);
11359 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11361 /* Remove the GIMPLE_OMP_FOR. */
11362 gsi_remove (&gsi
, true);
11364 /* Fixup edges from head_bb */
11365 be
= BRANCH_EDGE (head_bb
);
11366 fte
= FALLTHRU_EDGE (head_bb
);
11367 be
->flags
|= EDGE_FALSE_VALUE
;
11368 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11370 basic_block body_bb
= fte
->dest
;
11372 if (gimple_in_ssa_p (cfun
))
11374 gsi
= gsi_last_bb (cont_bb
);
11375 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11377 offset
= gimple_omp_continue_control_use (cont_stmt
);
11378 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11382 offset
= create_tmp_var (diff_type
, ".offset");
11383 offset_init
= offset_incr
= offset
;
11385 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11387 /* Loop offset & bound go into head_bb. */
11388 gsi
= gsi_start_bb (head_bb
);
11390 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11391 build_int_cst (integer_type_node
,
11392 IFN_GOACC_LOOP_OFFSET
),
11394 chunk_size
, gwv
, chunk_no
);
11395 gimple_call_set_lhs (call
, offset_init
);
11396 gimple_set_location (call
, loc
);
11397 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11399 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11400 build_int_cst (integer_type_node
,
11401 IFN_GOACC_LOOP_BOUND
),
11403 chunk_size
, gwv
, offset_init
);
11404 gimple_call_set_lhs (call
, bound
);
11405 gimple_set_location (call
, loc
);
11406 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11408 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11409 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11410 GSI_CONTINUE_LINKING
);
11412 /* V assignment goes into body_bb. */
11413 if (!gimple_in_ssa_p (cfun
))
11415 gsi
= gsi_start_bb (body_bb
);
11417 expr
= build2 (plus_code
, iter_type
, b
,
11418 fold_convert (plus_type
, offset
));
11419 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11420 true, GSI_SAME_STMT
);
11421 ass
= gimple_build_assign (v
, expr
);
11422 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11423 if (fd
->collapse
> 1)
11424 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11427 /* Loop increment goes into cont_bb. If this is not a loop, we
11428 will have spawned threads as if it was, and each one will
11429 execute one iteration. The specification is not explicit about
11430 whether such constructs are ill-formed or not, and they can
11431 occur, especially when noreturn routines are involved. */
11434 gsi
= gsi_last_bb (cont_bb
);
11435 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11436 loc
= gimple_location (cont_stmt
);
11438 /* Increment offset. */
11439 if (gimple_in_ssa_p (cfun
))
11440 expr
= build2 (plus_code
, iter_type
, offset
,
11441 fold_convert (plus_type
, step
));
11443 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11444 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11445 true, GSI_SAME_STMT
);
11446 ass
= gimple_build_assign (offset_incr
, expr
);
11447 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11448 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11449 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11451 /* Remove the GIMPLE_OMP_CONTINUE. */
11452 gsi_remove (&gsi
, true);
11454 /* Fixup edges from cont_bb */
11455 be
= BRANCH_EDGE (cont_bb
);
11456 fte
= FALLTHRU_EDGE (cont_bb
);
11457 be
->flags
|= EDGE_TRUE_VALUE
;
11458 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11462 /* Split the beginning of exit_bb to make bottom_bb. We
11463 need to insert a nop at the start, because splitting is
11464 after a stmt, not before. */
11465 gsi
= gsi_start_bb (exit_bb
);
11466 stmt
= gimple_build_nop ();
11467 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11468 split
= split_block (exit_bb
, stmt
);
11469 bottom_bb
= split
->src
;
11470 exit_bb
= split
->dest
;
11471 gsi
= gsi_last_bb (bottom_bb
);
11473 /* Chunk increment and test goes into bottom_bb. */
11474 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11475 build_int_cst (diff_type
, 1));
11476 ass
= gimple_build_assign (chunk_no
, expr
);
11477 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11479 /* Chunk test at end of bottom_bb. */
11480 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11481 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11482 GSI_CONTINUE_LINKING
);
11484 /* Fixup edges from bottom_bb. */
11485 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11486 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11490 gsi
= gsi_last_bb (exit_bb
);
11491 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11492 loc
= gimple_location (gsi_stmt (gsi
));
11494 if (!gimple_in_ssa_p (cfun
))
11496 /* Insert the final value of V, in case it is live. This is the
11497 value for the only thread that survives past the join. */
11498 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11499 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11500 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11501 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11502 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11503 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11504 true, GSI_SAME_STMT
);
11505 ass
= gimple_build_assign (v
, expr
);
11506 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11509 /* Remove the OMP_RETURN. */
11510 gsi_remove (&gsi
, true);
11514 /* We now have one or two nested loops. Update the loop
11516 struct loop
*parent
= entry_bb
->loop_father
;
11517 struct loop
*body
= body_bb
->loop_father
;
11521 struct loop
*chunk_loop
= alloc_loop ();
11522 chunk_loop
->header
= head_bb
;
11523 chunk_loop
->latch
= bottom_bb
;
11524 add_loop (chunk_loop
, parent
);
11525 parent
= chunk_loop
;
11527 else if (parent
!= body
)
11529 gcc_assert (body
->header
== body_bb
);
11530 gcc_assert (body
->latch
== cont_bb
11531 || single_pred (body
->latch
) == cont_bb
);
11537 struct loop
*body_loop
= alloc_loop ();
11538 body_loop
->header
= body_bb
;
11539 body_loop
->latch
= cont_bb
;
11540 add_loop (body_loop
, parent
);
11545 /* Expand the OMP loop defined by REGION. */
11548 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11550 struct omp_for_data fd
;
11551 struct omp_for_data_loop
*loops
;
11554 = (struct omp_for_data_loop
*)
11555 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11556 * sizeof (struct omp_for_data_loop
));
11557 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11559 region
->sched_kind
= fd
.sched_kind
;
11560 region
->sched_modifiers
= fd
.sched_modifiers
;
11562 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11563 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11564 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11567 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11568 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11569 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11572 /* If there isn't a continue then this is a degerate case where
11573 the introduction of abnormal edges during lowering will prevent
11574 original loops from being detected. Fix that up. */
11575 loops_state_set (LOOPS_NEED_FIXUP
);
11577 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11578 expand_omp_simd (region
, &fd
);
11579 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11580 expand_cilk_for (region
, &fd
);
11581 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11583 gcc_assert (!inner_stmt
);
11584 expand_oacc_for (region
, &fd
);
11586 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11588 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11589 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11591 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11593 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11594 && !fd
.have_ordered
)
11596 if (fd
.chunk_size
== NULL
)
11597 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11599 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11603 int fn_index
, start_ix
, next_ix
;
11605 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11606 == GF_OMP_FOR_KIND_FOR
);
11607 if (fd
.chunk_size
== NULL
11608 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11609 fd
.chunk_size
= integer_zero_node
;
11610 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11611 switch (fd
.sched_kind
)
11613 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11616 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11617 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11618 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11620 && !fd
.have_ordered
)
11622 fn_index
= 3 + fd
.sched_kind
;
11627 fn_index
= fd
.sched_kind
;
11631 fn_index
+= fd
.have_ordered
* 6;
11633 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11635 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11636 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11637 if (fd
.iter_type
== long_long_unsigned_type_node
)
11639 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11640 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11641 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11642 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11644 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11645 (enum built_in_function
) next_ix
, inner_stmt
);
11648 if (gimple_in_ssa_p (cfun
))
11649 update_ssa (TODO_update_ssa_only_virtuals
);
11653 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11655 v = GOMP_sections_start (n);
11672 v = GOMP_sections_next ();
11677 If this is a combined parallel sections, replace the call to
11678 GOMP_sections_start with call to GOMP_sections_next. */
11681 expand_omp_sections (struct omp_region
*region
)
11683 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11685 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11686 gimple_stmt_iterator si
, switch_si
;
11687 gomp_sections
*sections_stmt
;
11689 gomp_continue
*cont
;
11692 struct omp_region
*inner
;
11694 bool exit_reachable
= region
->cont
!= NULL
;
11696 gcc_assert (region
->exit
!= NULL
);
11697 entry_bb
= region
->entry
;
11698 l0_bb
= single_succ (entry_bb
);
11699 l1_bb
= region
->cont
;
11700 l2_bb
= region
->exit
;
11701 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11702 l2
= gimple_block_label (l2_bb
);
11705 /* This can happen if there are reductions. */
11706 len
= EDGE_COUNT (l0_bb
->succs
);
11707 gcc_assert (len
> 0);
11708 e
= EDGE_SUCC (l0_bb
, len
- 1);
11709 si
= gsi_last_bb (e
->dest
);
11712 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11713 l2
= gimple_block_label (e
->dest
);
11715 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11717 si
= gsi_last_bb (e
->dest
);
11719 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11721 l2
= gimple_block_label (e
->dest
);
11726 if (exit_reachable
)
11727 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11729 default_bb
= create_empty_bb (l0_bb
);
11731 /* We will build a switch() with enough cases for all the
11732 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11733 and a default case to abort if something goes wrong. */
11734 len
= EDGE_COUNT (l0_bb
->succs
);
11736 /* Use vec::quick_push on label_vec throughout, since we know the size
11738 auto_vec
<tree
> label_vec (len
);
11740 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11741 GIMPLE_OMP_SECTIONS statement. */
11742 si
= gsi_last_bb (entry_bb
);
11743 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11744 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11745 vin
= gimple_omp_sections_control (sections_stmt
);
11746 if (!is_combined_parallel (region
))
11748 /* If we are not inside a combined parallel+sections region,
11749 call GOMP_sections_start. */
11750 t
= build_int_cst (unsigned_type_node
, len
- 1);
11751 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11752 stmt
= gimple_build_call (u
, 1, t
);
11756 /* Otherwise, call GOMP_sections_next. */
11757 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11758 stmt
= gimple_build_call (u
, 0);
11760 gimple_call_set_lhs (stmt
, vin
);
11761 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11762 gsi_remove (&si
, true);
11764 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11766 switch_si
= gsi_last_bb (l0_bb
);
11767 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11768 if (exit_reachable
)
11770 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11771 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11772 vmain
= gimple_omp_continue_control_use (cont
);
11773 vnext
= gimple_omp_continue_control_def (cont
);
11781 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11782 label_vec
.quick_push (t
);
11785 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11786 for (inner
= region
->inner
, casei
= 1;
11788 inner
= inner
->next
, i
++, casei
++)
11790 basic_block s_entry_bb
, s_exit_bb
;
11792 /* Skip optional reduction region. */
11793 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11800 s_entry_bb
= inner
->entry
;
11801 s_exit_bb
= inner
->exit
;
11803 t
= gimple_block_label (s_entry_bb
);
11804 u
= build_int_cst (unsigned_type_node
, casei
);
11805 u
= build_case_label (u
, NULL
, t
);
11806 label_vec
.quick_push (u
);
11808 si
= gsi_last_bb (s_entry_bb
);
11809 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11810 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11811 gsi_remove (&si
, true);
11812 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11814 if (s_exit_bb
== NULL
)
11817 si
= gsi_last_bb (s_exit_bb
);
11818 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11819 gsi_remove (&si
, true);
11821 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11824 /* Error handling code goes in DEFAULT_BB. */
11825 t
= gimple_block_label (default_bb
);
11826 u
= build_case_label (NULL
, NULL
, t
);
11827 make_edge (l0_bb
, default_bb
, 0);
11828 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11830 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11831 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11832 gsi_remove (&switch_si
, true);
11834 si
= gsi_start_bb (default_bb
);
11835 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11836 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11838 if (exit_reachable
)
11842 /* Code to get the next section goes in L1_BB. */
11843 si
= gsi_last_bb (l1_bb
);
11844 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11846 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11847 stmt
= gimple_build_call (bfn_decl
, 0);
11848 gimple_call_set_lhs (stmt
, vnext
);
11849 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11850 gsi_remove (&si
, true);
11852 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11855 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11856 si
= gsi_last_bb (l2_bb
);
11857 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11858 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11859 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11860 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11862 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11863 stmt
= gimple_build_call (t
, 0);
11864 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11865 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11866 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11867 gsi_remove (&si
, true);
11869 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11873 /* Expand code for an OpenMP single directive. We've already expanded
11874 much of the code, here we simply place the GOMP_barrier call. */
11877 expand_omp_single (struct omp_region
*region
)
11879 basic_block entry_bb
, exit_bb
;
11880 gimple_stmt_iterator si
;
11882 entry_bb
= region
->entry
;
11883 exit_bb
= region
->exit
;
11885 si
= gsi_last_bb (entry_bb
);
11886 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11887 gsi_remove (&si
, true);
11888 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11890 si
= gsi_last_bb (exit_bb
);
11891 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11893 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11894 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11896 gsi_remove (&si
, true);
11897 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11901 /* Generic expansion for OpenMP synchronization directives: master,
11902 ordered and critical. All we need to do here is remove the entry
11903 and exit markers for REGION. */
11906 expand_omp_synch (struct omp_region
*region
)
11908 basic_block entry_bb
, exit_bb
;
11909 gimple_stmt_iterator si
;
11911 entry_bb
= region
->entry
;
11912 exit_bb
= region
->exit
;
11914 si
= gsi_last_bb (entry_bb
);
11915 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11916 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11917 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11918 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11919 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11920 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11921 gsi_remove (&si
, true);
11922 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11926 si
= gsi_last_bb (exit_bb
);
11927 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11928 gsi_remove (&si
, true);
11929 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11933 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11934 operation as a normal volatile load. */
11937 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11938 tree loaded_val
, int index
)
11940 enum built_in_function tmpbase
;
11941 gimple_stmt_iterator gsi
;
11942 basic_block store_bb
;
11945 tree decl
, call
, type
, itype
;
11947 gsi
= gsi_last_bb (load_bb
);
11948 stmt
= gsi_stmt (gsi
);
11949 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11950 loc
= gimple_location (stmt
);
11952 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11953 is smaller than word size, then expand_atomic_load assumes that the load
11954 is atomic. We could avoid the builtin entirely in this case. */
11956 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11957 decl
= builtin_decl_explicit (tmpbase
);
11958 if (decl
== NULL_TREE
)
11961 type
= TREE_TYPE (loaded_val
);
11962 itype
= TREE_TYPE (TREE_TYPE (decl
));
11964 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11965 build_int_cst (NULL
,
11966 gimple_omp_atomic_seq_cst_p (stmt
)
11968 : MEMMODEL_RELAXED
));
11969 if (!useless_type_conversion_p (type
, itype
))
11970 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11971 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11973 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11974 gsi_remove (&gsi
, true);
11976 store_bb
= single_succ (load_bb
);
11977 gsi
= gsi_last_bb (store_bb
);
11978 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11979 gsi_remove (&gsi
, true);
11981 if (gimple_in_ssa_p (cfun
))
11982 update_ssa (TODO_update_ssa_no_phi
);
11987 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11988 operation as a normal volatile store. */
11991 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11992 tree loaded_val
, tree stored_val
, int index
)
11994 enum built_in_function tmpbase
;
11995 gimple_stmt_iterator gsi
;
11996 basic_block store_bb
= single_succ (load_bb
);
11999 tree decl
, call
, type
, itype
;
12000 machine_mode imode
;
12003 gsi
= gsi_last_bb (load_bb
);
12004 stmt
= gsi_stmt (gsi
);
12005 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
12007 /* If the load value is needed, then this isn't a store but an exchange. */
12008 exchange
= gimple_omp_atomic_need_value_p (stmt
);
12010 gsi
= gsi_last_bb (store_bb
);
12011 stmt
= gsi_stmt (gsi
);
12012 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
12013 loc
= gimple_location (stmt
);
12015 /* ??? If the target does not implement atomic_store_optab[mode], and mode
12016 is smaller than word size, then expand_atomic_store assumes that the store
12017 is atomic. We could avoid the builtin entirely in this case. */
12019 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
12020 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
12021 decl
= builtin_decl_explicit (tmpbase
);
12022 if (decl
== NULL_TREE
)
12025 type
= TREE_TYPE (stored_val
);
12027 /* Dig out the type of the function's second argument. */
12028 itype
= TREE_TYPE (decl
);
12029 itype
= TYPE_ARG_TYPES (itype
);
12030 itype
= TREE_CHAIN (itype
);
12031 itype
= TREE_VALUE (itype
);
12032 imode
= TYPE_MODE (itype
);
12034 if (exchange
&& !can_atomic_exchange_p (imode
, true))
12037 if (!useless_type_conversion_p (itype
, type
))
12038 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
12039 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
12040 build_int_cst (NULL
,
12041 gimple_omp_atomic_seq_cst_p (stmt
)
12043 : MEMMODEL_RELAXED
));
12046 if (!useless_type_conversion_p (type
, itype
))
12047 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
12048 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
12051 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12052 gsi_remove (&gsi
, true);
12054 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
12055 gsi
= gsi_last_bb (load_bb
);
12056 gsi_remove (&gsi
, true);
12058 if (gimple_in_ssa_p (cfun
))
12059 update_ssa (TODO_update_ssa_no_phi
);
12064 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
12065 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
12066 size of the data type, and thus usable to find the index of the builtin
12067 decl. Returns false if the expression is not of the proper form. */
12070 expand_omp_atomic_fetch_op (basic_block load_bb
,
12071 tree addr
, tree loaded_val
,
12072 tree stored_val
, int index
)
12074 enum built_in_function oldbase
, newbase
, tmpbase
;
12075 tree decl
, itype
, call
;
12077 basic_block store_bb
= single_succ (load_bb
);
12078 gimple_stmt_iterator gsi
;
12081 enum tree_code code
;
12082 bool need_old
, need_new
;
12083 machine_mode imode
;
12086 /* We expect to find the following sequences:
12089 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12092 val = tmp OP something; (or: something OP tmp)
12093 GIMPLE_OMP_STORE (val)
12095 ???FIXME: Allow a more flexible sequence.
12096 Perhaps use data flow to pick the statements.
12100 gsi
= gsi_after_labels (store_bb
);
12101 stmt
= gsi_stmt (gsi
);
12102 loc
= gimple_location (stmt
);
12103 if (!is_gimple_assign (stmt
))
12106 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
12108 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
12109 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
12110 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
12111 gcc_checking_assert (!need_old
|| !need_new
);
12113 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
12116 /* Check for one of the supported fetch-op operations. */
12117 code
= gimple_assign_rhs_code (stmt
);
12121 case POINTER_PLUS_EXPR
:
12122 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
12123 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
12126 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
12127 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
12130 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
12131 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
12134 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
12135 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
12138 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
12139 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
12145 /* Make sure the expression is of the proper form. */
12146 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
12147 rhs
= gimple_assign_rhs2 (stmt
);
12148 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
12149 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
12150 rhs
= gimple_assign_rhs1 (stmt
);
12154 tmpbase
= ((enum built_in_function
)
12155 ((need_new
? newbase
: oldbase
) + index
+ 1));
12156 decl
= builtin_decl_explicit (tmpbase
);
12157 if (decl
== NULL_TREE
)
12159 itype
= TREE_TYPE (TREE_TYPE (decl
));
12160 imode
= TYPE_MODE (itype
);
12162 /* We could test all of the various optabs involved, but the fact of the
12163 matter is that (with the exception of i486 vs i586 and xadd) all targets
12164 that support any atomic operaton optab also implements compare-and-swap.
12165 Let optabs.c take care of expanding any compare-and-swap loop. */
12166 if (!can_compare_and_swap_p (imode
, true))
12169 gsi
= gsi_last_bb (load_bb
);
12170 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12172 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12173 It only requires that the operation happen atomically. Thus we can
12174 use the RELAXED memory model. */
12175 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
12176 fold_convert_loc (loc
, itype
, rhs
),
12177 build_int_cst (NULL
,
12178 seq_cst
? MEMMODEL_SEQ_CST
12179 : MEMMODEL_RELAXED
));
12181 if (need_old
|| need_new
)
12183 lhs
= need_old
? loaded_val
: stored_val
;
12184 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
12185 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
12188 call
= fold_convert_loc (loc
, void_type_node
, call
);
12189 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12190 gsi_remove (&gsi
, true);
12192 gsi
= gsi_last_bb (store_bb
);
12193 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
12194 gsi_remove (&gsi
, true);
12195 gsi
= gsi_last_bb (store_bb
);
12196 stmt
= gsi_stmt (gsi
);
12197 gsi_remove (&gsi
, true);
12199 if (gimple_in_ssa_p (cfun
))
12201 release_defs (stmt
);
12202 update_ssa (TODO_update_ssa_no_phi
);
12208 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12212 newval = rhs; // with oldval replacing *addr in rhs
12213 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12214 if (oldval != newval)
12217 INDEX is log2 of the size of the data type, and thus usable to find the
12218 index of the builtin decl. */
12221 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
12222 tree addr
, tree loaded_val
, tree stored_val
,
12225 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
12226 tree type
, itype
, cmpxchg
, iaddr
;
12227 gimple_stmt_iterator si
;
12228 basic_block loop_header
= single_succ (load_bb
);
12229 gimple
*phi
, *stmt
;
12231 enum built_in_function fncode
;
12233 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12234 order to use the RELAXED memory model effectively. */
12235 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12237 cmpxchg
= builtin_decl_explicit (fncode
);
12238 if (cmpxchg
== NULL_TREE
)
12240 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12241 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
12243 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
12246 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12247 si
= gsi_last_bb (load_bb
);
12248 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12250 /* For floating-point values, we'll need to view-convert them to integers
12251 so that we can perform the atomic compare and swap. Simplify the
12252 following code by always setting up the "i"ntegral variables. */
12253 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
12257 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
12260 = force_gimple_operand_gsi (&si
,
12261 fold_convert (TREE_TYPE (iaddr
), addr
),
12262 false, NULL_TREE
, true, GSI_SAME_STMT
);
12263 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
12264 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12265 loadedi
= create_tmp_var (itype
);
12266 if (gimple_in_ssa_p (cfun
))
12267 loadedi
= make_ssa_name (loadedi
);
12272 loadedi
= loaded_val
;
12275 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
12276 tree loaddecl
= builtin_decl_explicit (fncode
);
12279 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12280 build_call_expr (loaddecl
, 2, iaddr
,
12281 build_int_cst (NULL_TREE
,
12282 MEMMODEL_RELAXED
)));
12284 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12285 build_int_cst (TREE_TYPE (iaddr
), 0));
12288 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12291 /* Move the value to the LOADEDI temporary. */
12292 if (gimple_in_ssa_p (cfun
))
12294 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12295 phi
= create_phi_node (loadedi
, loop_header
);
12296 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12300 gsi_insert_before (&si
,
12301 gimple_build_assign (loadedi
, initial
),
12303 if (loadedi
!= loaded_val
)
12305 gimple_stmt_iterator gsi2
;
12308 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12309 gsi2
= gsi_start_bb (loop_header
);
12310 if (gimple_in_ssa_p (cfun
))
12313 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12314 true, GSI_SAME_STMT
);
12315 stmt
= gimple_build_assign (loaded_val
, x
);
12316 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12320 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12321 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12322 true, GSI_SAME_STMT
);
12325 gsi_remove (&si
, true);
12327 si
= gsi_last_bb (store_bb
);
12328 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12331 storedi
= stored_val
;
12334 force_gimple_operand_gsi (&si
,
12335 build1 (VIEW_CONVERT_EXPR
, itype
,
12336 stored_val
), true, NULL_TREE
, true,
12339 /* Build the compare&swap statement. */
12340 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12341 new_storedi
= force_gimple_operand_gsi (&si
,
12342 fold_convert (TREE_TYPE (loadedi
),
12345 true, GSI_SAME_STMT
);
12347 if (gimple_in_ssa_p (cfun
))
12348 old_vali
= loadedi
;
12351 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12352 stmt
= gimple_build_assign (old_vali
, loadedi
);
12353 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12355 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12356 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12359 /* Note that we always perform the comparison as an integer, even for
12360 floating point. This allows the atomic operation to properly
12361 succeed even with NaNs and -0.0. */
12362 stmt
= gimple_build_cond_empty
12363 (build2 (NE_EXPR
, boolean_type_node
,
12364 new_storedi
, old_vali
));
12365 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12368 e
= single_succ_edge (store_bb
);
12369 e
->flags
&= ~EDGE_FALLTHRU
;
12370 e
->flags
|= EDGE_FALSE_VALUE
;
12372 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12374 /* Copy the new value to loadedi (we already did that before the condition
12375 if we are not in SSA). */
12376 if (gimple_in_ssa_p (cfun
))
12378 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12379 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12382 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12383 gsi_remove (&si
, true);
12385 struct loop
*loop
= alloc_loop ();
12386 loop
->header
= loop_header
;
12387 loop
->latch
= store_bb
;
12388 add_loop (loop
, loop_header
->loop_father
);
12390 if (gimple_in_ssa_p (cfun
))
12391 update_ssa (TODO_update_ssa_no_phi
);
12396 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12398 GOMP_atomic_start ();
12400 GOMP_atomic_end ();
12402 The result is not globally atomic, but works so long as all parallel
12403 references are within #pragma omp atomic directives. According to
12404 responses received from omp@openmp.org, appears to be within spec.
12405 Which makes sense, since that's how several other compilers handle
12406 this situation as well.
12407 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12408 expanding. STORED_VAL is the operand of the matching
12409 GIMPLE_OMP_ATOMIC_STORE.
12412 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12413 loaded_val = *addr;
12416 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12417 *addr = stored_val;
12421 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12422 tree addr
, tree loaded_val
, tree stored_val
)
12424 gimple_stmt_iterator si
;
12428 si
= gsi_last_bb (load_bb
);
12429 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12431 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12432 t
= build_call_expr (t
, 0);
12433 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12435 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12436 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12437 gsi_remove (&si
, true);
12439 si
= gsi_last_bb (store_bb
);
12440 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12442 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12444 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12446 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12447 t
= build_call_expr (t
, 0);
12448 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12449 gsi_remove (&si
, true);
12451 if (gimple_in_ssa_p (cfun
))
12452 update_ssa (TODO_update_ssa_no_phi
);
12456 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12457 using expand_omp_atomic_fetch_op. If it failed, we try to
12458 call expand_omp_atomic_pipeline, and if it fails too, the
12459 ultimate fallback is wrapping the operation in a mutex
12460 (expand_omp_atomic_mutex). REGION is the atomic region built
12461 by build_omp_regions_1(). */
12464 expand_omp_atomic (struct omp_region
*region
)
12466 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12467 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12468 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12469 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12470 tree addr
= gimple_omp_atomic_load_rhs (load
);
12471 tree stored_val
= gimple_omp_atomic_store_val (store
);
12472 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12473 HOST_WIDE_INT index
;
12475 /* Make sure the type is one of the supported sizes. */
12476 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12477 index
= exact_log2 (index
);
12478 if (index
>= 0 && index
<= 4)
12480 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12482 /* __sync builtins require strict data alignment. */
12483 if (exact_log2 (align
) >= index
)
12486 if (loaded_val
== stored_val
12487 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12488 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12489 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12490 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12493 /* Atomic store. */
12494 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12495 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12496 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12497 && store_bb
== single_succ (load_bb
)
12498 && first_stmt (store_bb
) == store
12499 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12500 stored_val
, index
))
12503 /* When possible, use specialized atomic update functions. */
12504 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12505 && store_bb
== single_succ (load_bb
)
12506 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12507 loaded_val
, stored_val
, index
))
12510 /* If we don't have specialized __sync builtins, try and implement
12511 as a compare and swap loop. */
12512 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12513 loaded_val
, stored_val
, index
))
12518 /* The ultimate fallback is wrapping the operation in a mutex. */
12519 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12523 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12524 macro on gomp-constants.h. We do not check for overflow. */
12527 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12531 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12534 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12535 device
, build_int_cst (unsigned_type_node
,
12536 GOMP_LAUNCH_DEVICE_SHIFT
));
12537 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12542 /* Look for compute grid dimension clauses and convert to an attribute
12543 attached to FN. This permits the target-side code to (a) massage
12544 the dimensions, (b) emit that data and (c) optimize. Non-constant
12545 dimensions are pushed onto ARGS.
12547 The attribute value is a TREE_LIST. A set of dimensions is
12548 represented as a list of INTEGER_CST. Those that are runtime
12549 exprs are represented as an INTEGER_CST of zero.
12551 TOOO. Normally the attribute will just contain a single such list. If
12552 however it contains a list of lists, this will represent the use of
12553 device_type. Each member of the outer list is an assoc list of
12554 dimensions, keyed by the device type. The first entry will be the
12555 default. Well, that's the plan. */
12557 #define OACC_FN_ATTRIB "oacc function"
12559 /* Replace any existing oacc fn attribute with updated dimensions. */
12562 replace_oacc_fn_attrib (tree fn
, tree dims
)
12564 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12565 tree attribs
= DECL_ATTRIBUTES (fn
);
12567 /* If we happen to be present as the first attrib, drop it. */
12568 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12569 attribs
= TREE_CHAIN (attribs
);
12570 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12573 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12574 function attribute. Push any that are non-constant onto the ARGS
12575 list, along with an appropriate GOMP_LAUNCH_DIM tag. IS_KERNEL is
12576 true, if these are for a kernels region offload function. */
12579 set_oacc_fn_attrib (tree fn
, tree clauses
, bool is_kernel
, vec
<tree
> *args
)
12581 /* Must match GOMP_DIM ordering. */
12582 static const omp_clause_code ids
[]
12583 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12584 OMP_CLAUSE_VECTOR_LENGTH
};
12586 tree dims
[GOMP_DIM_MAX
];
12587 tree attr
= NULL_TREE
;
12588 unsigned non_const
= 0;
12590 for (ix
= GOMP_DIM_MAX
; ix
--;)
12592 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12593 tree dim
= NULL_TREE
;
12596 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12598 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12600 dim
= integer_zero_node
;
12601 non_const
|= GOMP_DIM_MASK (ix
);
12603 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12604 /* Note kernelness with TREE_PUBLIC. */
12606 TREE_PUBLIC (attr
) = 1;
12609 replace_oacc_fn_attrib (fn
, attr
);
12613 /* Push a dynamic argument set. */
12614 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12615 NULL_TREE
, non_const
));
12616 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12617 if (non_const
& GOMP_DIM_MASK (ix
))
12618 args
->safe_push (dims
[ix
]);
12622 /* Process the routine's dimension clauess to generate an attribute
12623 value. Issue diagnostics as appropriate. We default to SEQ
12624 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12625 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12626 can have a loop partitioned on it. non-zero indicates
12627 yes, zero indicates no. By construction once a non-zero has been
12628 reached, further inner dimensions must also be non-zero. We set
12629 TREE_VALUE to zero for the dimensions that may be partitioned and
12630 1 for the other ones -- if a loop is (erroneously) spawned at
12631 an outer level, we don't want to try and partition it. */
12634 build_oacc_routine_dims (tree clauses
)
12636 /* Must match GOMP_DIM ordering. */
12637 static const omp_clause_code ids
[] =
12638 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12642 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12643 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12644 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12647 error_at (OMP_CLAUSE_LOCATION (clauses
),
12648 "multiple loop axes specified for routine");
12653 /* Default to SEQ. */
12655 level
= GOMP_DIM_MAX
;
12657 tree dims
= NULL_TREE
;
12659 for (ix
= GOMP_DIM_MAX
; ix
--;)
12660 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12661 build_int_cst (integer_type_node
, ix
< level
), dims
);
12666 /* Retrieve the oacc function attrib and return it. Non-oacc
12667 functions will return NULL. */
12670 get_oacc_fn_attrib (tree fn
)
12672 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12675 /* Return true if this oacc fn attrib is for a kernels offload
12676 region. We use the TREE_PUBLIC flag of each dimension -- only
12677 need to check the first one. */
12680 oacc_fn_attrib_kernels_p (tree attr
)
12682 return TREE_PUBLIC (TREE_VALUE (attr
));
12685 /* Return level at which oacc routine may spawn a partitioned loop, or
12686 -1 if it is not a routine (i.e. is an offload fn). */
12689 oacc_fn_attrib_level (tree attr
)
12691 tree pos
= TREE_VALUE (attr
);
12693 if (!TREE_PURPOSE (pos
))
12697 for (ix
= 0; ix
!= GOMP_DIM_MAX
;
12698 ix
++, pos
= TREE_CHAIN (pos
))
12699 if (!integer_zerop (TREE_PURPOSE (pos
)))
12705 /* Extract an oacc execution dimension from FN. FN must be an
12706 offloaded function or routine that has already had its execution
12707 dimensions lowered to the target-specific values. */
12710 get_oacc_fn_dim_size (tree fn
, int axis
)
12712 tree attrs
= get_oacc_fn_attrib (fn
);
12714 gcc_assert (axis
< GOMP_DIM_MAX
);
12716 tree dims
= TREE_VALUE (attrs
);
12718 dims
= TREE_CHAIN (dims
);
12720 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12725 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12726 IFN_GOACC_DIM_SIZE call. */
12729 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12731 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12732 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12733 tree arg
= gimple_call_arg (stmt
, 0);
12734 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12736 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12740 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12744 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
12745 basic_block region_exit
)
12747 struct loop
*outer
= region_entry
->loop_father
;
12748 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
12750 /* Don't parallelize the kernels region if it contains more than one outer
12752 unsigned int nr_outer_loops
= 0;
12753 struct loop
*single_outer
= NULL
;
12754 for (struct loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
12756 gcc_assert (loop_outer (loop
) == outer
);
12758 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
12761 if (region_exit
!= NULL
12762 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
12766 single_outer
= loop
;
12768 if (nr_outer_loops
!= 1)
12771 for (struct loop
*loop
= single_outer
->inner
; loop
!= NULL
; loop
= loop
->inner
)
12775 /* Mark the loops in the region. */
12776 for (struct loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
12777 loop
->in_oacc_kernels_region
= true;
12780 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12782 struct GTY(()) grid_launch_attributes_trees
12784 tree kernel_dim_array_type
;
12785 tree kernel_lattrs_dimnum_decl
;
12786 tree kernel_lattrs_grid_decl
;
12787 tree kernel_lattrs_group_decl
;
12788 tree kernel_launch_attributes_type
;
12791 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
12793 /* Create types used to pass kernel launch attributes to target. */
12796 grid_create_kernel_launch_attr_types (void)
12798 if (grid_attr_trees
)
12800 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
12802 tree dim_arr_index_type
12803 = build_index_type (build_int_cst (integer_type_node
, 2));
12804 grid_attr_trees
->kernel_dim_array_type
12805 = build_array_type (uint32_type_node
, dim_arr_index_type
);
12807 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
12808 grid_attr_trees
->kernel_lattrs_dimnum_decl
12809 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
12811 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
12813 grid_attr_trees
->kernel_lattrs_grid_decl
12814 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
12815 grid_attr_trees
->kernel_dim_array_type
);
12816 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
12817 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
12818 grid_attr_trees
->kernel_lattrs_group_decl
12819 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
12820 grid_attr_trees
->kernel_dim_array_type
);
12821 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
12822 = grid_attr_trees
->kernel_lattrs_grid_decl
;
12823 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
12824 "__gomp_kernel_launch_attributes",
12825 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
12828 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12829 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12830 of type uint32_type_node. */
12833 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
12834 tree fld_decl
, int index
, tree value
)
12836 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
12837 build3 (COMPONENT_REF
,
12838 grid_attr_trees
->kernel_dim_array_type
,
12839 range_var
, fld_decl
, NULL_TREE
),
12840 build_int_cst (integer_type_node
, index
),
12841 NULL_TREE
, NULL_TREE
);
12842 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
12845 /* Return a tree representation of a pointer to a structure with grid and
12846 work-group size information. Statements filling that information will be
12847 inserted before GSI, TGT_STMT is the target statement which has the
12848 necessary information in it. */
12851 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
12852 gomp_target
*tgt_stmt
)
12854 grid_create_kernel_launch_attr_types ();
12855 tree u32_one
= build_one_cst (uint32_type_node
);
12856 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
12857 "__kernel_launch_attrs");
12859 unsigned max_dim
= 0;
12860 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
12862 clause
= OMP_CLAUSE_CHAIN (clause
))
12864 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
12867 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
12868 max_dim
= MAX (dim
, max_dim
);
12870 grid_insert_store_range_dim (gsi
, lattrs
,
12871 grid_attr_trees
->kernel_lattrs_grid_decl
,
12872 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
12873 grid_insert_store_range_dim (gsi
, lattrs
,
12874 grid_attr_trees
->kernel_lattrs_group_decl
,
12875 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
12878 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
12879 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
12880 /* At this moment we cannot gridify a loop with a collapse clause. */
12881 /* TODO: Adjust when we support bigger collapse. */
12882 gcc_assert (max_dim
== 0);
12883 gsi_insert_before (gsi
, gimple_build_assign (dimref
, u32_one
), GSI_SAME_STMT
);
12884 TREE_ADDRESSABLE (lattrs
) = 1;
12885 return build_fold_addr_expr (lattrs
);
12888 /* Build target argument identifier from the DEVICE identifier, value
12889 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12892 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
12894 tree t
= build_int_cst (integer_type_node
, device
);
12895 if (subseqent_param
)
12896 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12897 build_int_cst (integer_type_node
,
12898 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
12899 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12900 build_int_cst (integer_type_node
, id
));
12904 /* Like above but return it in type that can be directly stored as an element
12905 of the argument array. */
12908 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
12910 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
12911 return fold_convert (ptr_type_node
, t
);
12914 /* Return a target argument consisting of DEVICE identifier, value identifier
12915 ID, and the actual VALUE. */
12918 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
12921 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
12922 fold_convert (integer_type_node
, value
),
12923 build_int_cst (unsigned_type_node
,
12924 GOMP_TARGET_ARG_VALUE_SHIFT
));
12925 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12926 get_target_argument_identifier_1 (device
, false, id
));
12927 t
= fold_convert (ptr_type_node
, t
);
12928 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
12931 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12932 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12933 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12937 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
12938 int id
, tree value
, vec
<tree
> *args
)
12940 if (tree_fits_shwi_p (value
)
12941 && tree_to_shwi (value
) > -(1 << 15)
12942 && tree_to_shwi (value
) < (1 << 15))
12943 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
12946 args
->quick_push (get_target_argument_identifier (device
, true, id
));
12947 value
= fold_convert (ptr_type_node
, value
);
12948 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
12950 args
->quick_push (value
);
12954 /* Create an array of arguments that is then passed to GOMP_target. */
12957 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
12959 auto_vec
<tree
, 6> args
;
12960 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
12961 tree t
, c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12963 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
12965 t
= integer_minus_one_node
;
12966 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12967 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
12969 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12971 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
12973 t
= integer_minus_one_node
;
12974 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12975 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
12978 /* Add HSA-specific grid sizes, if available. */
12979 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
12980 OMP_CLAUSE__GRIDDIM_
))
12982 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true,
12983 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
);
12984 args
.quick_push (t
);
12985 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
12988 /* Produce more, perhaps device specific, arguments here. */
12990 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
12991 args
.length () + 1),
12992 ".omp_target_args");
12993 for (unsigned i
= 0; i
< args
.length (); i
++)
12995 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12996 build_int_cst (integer_type_node
, i
),
12997 NULL_TREE
, NULL_TREE
);
12998 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
13001 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
13002 build_int_cst (integer_type_node
, args
.length ()),
13003 NULL_TREE
, NULL_TREE
);
13004 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
13006 TREE_ADDRESSABLE (argarray
) = 1;
13007 return build_fold_addr_expr (argarray
);
13010 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
13013 expand_omp_target (struct omp_region
*region
)
13015 basic_block entry_bb
, exit_bb
, new_bb
;
13016 struct function
*child_cfun
;
13017 tree child_fn
, block
, t
;
13018 gimple_stmt_iterator gsi
;
13019 gomp_target
*entry_stmt
;
13022 bool offloaded
, data_region
;
13024 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
13025 new_bb
= region
->entry
;
13027 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
13028 switch (gimple_omp_target_kind (entry_stmt
))
13030 case GF_OMP_TARGET_KIND_REGION
:
13031 case GF_OMP_TARGET_KIND_UPDATE
:
13032 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13033 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13034 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13035 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13036 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13037 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13038 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13039 data_region
= false;
13041 case GF_OMP_TARGET_KIND_DATA
:
13042 case GF_OMP_TARGET_KIND_OACC_DATA
:
13043 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13044 data_region
= true;
13047 gcc_unreachable ();
13050 child_fn
= NULL_TREE
;
13054 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
13055 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
13058 /* Supported by expand_omp_taskreg, but not here. */
13059 if (child_cfun
!= NULL
)
13060 gcc_checking_assert (!child_cfun
->cfg
);
13061 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
13063 entry_bb
= region
->entry
;
13064 exit_bb
= region
->exit
;
13066 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
13067 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
13071 unsigned srcidx
, dstidx
, num
;
13073 /* If the offloading region needs data sent from the parent
13074 function, then the very first statement (except possible
13075 tree profile counter updates) of the offloading body
13076 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
13077 &.OMP_DATA_O is passed as an argument to the child function,
13078 we need to replace it with the argument as seen by the child
13081 In most cases, this will end up being the identity assignment
13082 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
13083 a function call that has been inlined, the original PARM_DECL
13084 .OMP_DATA_I may have been converted into a different local
13085 variable. In which case, we need to keep the assignment. */
13086 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
13089 basic_block entry_succ_bb
= single_succ (entry_bb
);
13090 gimple_stmt_iterator gsi
;
13092 gimple
*tgtcopy_stmt
= NULL
;
13093 tree sender
= TREE_VEC_ELT (data_arg
, 0);
13095 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
13097 gcc_assert (!gsi_end_p (gsi
));
13098 stmt
= gsi_stmt (gsi
);
13099 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
13102 if (gimple_num_ops (stmt
) == 2)
13104 tree arg
= gimple_assign_rhs1 (stmt
);
13106 /* We're ignoring the subcode because we're
13107 effectively doing a STRIP_NOPS. */
13109 if (TREE_CODE (arg
) == ADDR_EXPR
13110 && TREE_OPERAND (arg
, 0) == sender
)
13112 tgtcopy_stmt
= stmt
;
13118 gcc_assert (tgtcopy_stmt
!= NULL
);
13119 arg
= DECL_ARGUMENTS (child_fn
);
13121 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
13122 gsi_remove (&gsi
, true);
13125 /* Declare local variables needed in CHILD_CFUN. */
13126 block
= DECL_INITIAL (child_fn
);
13127 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
13128 /* The gimplifier could record temporaries in the offloading block
13129 rather than in containing function's local_decls chain,
13130 which would mean cgraph missed finalizing them. Do it now. */
13131 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
13132 if (TREE_CODE (t
) == VAR_DECL
13134 && !DECL_EXTERNAL (t
))
13135 varpool_node::finalize_decl (t
);
13136 DECL_SAVED_TREE (child_fn
) = NULL
;
13137 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13138 gimple_set_body (child_fn
, NULL
);
13139 TREE_USED (block
) = 1;
13141 /* Reset DECL_CONTEXT on function arguments. */
13142 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13143 DECL_CONTEXT (t
) = child_fn
;
13145 /* Split ENTRY_BB at GIMPLE_*,
13146 so that it can be moved to the child function. */
13147 gsi
= gsi_last_bb (entry_bb
);
13148 stmt
= gsi_stmt (gsi
);
13150 && gimple_code (stmt
) == gimple_code (entry_stmt
));
13151 e
= split_block (entry_bb
, stmt
);
13152 gsi_remove (&gsi
, true);
13153 entry_bb
= e
->dest
;
13154 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
13156 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13159 gsi
= gsi_last_bb (exit_bb
);
13160 gcc_assert (!gsi_end_p (gsi
)
13161 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13162 stmt
= gimple_build_return (NULL
);
13163 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
13164 gsi_remove (&gsi
, true);
13167 /* Move the offloading region into CHILD_CFUN. */
13169 block
= gimple_block (entry_stmt
);
13171 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
13173 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
13174 /* When the OMP expansion process cannot guarantee an up-to-date
13175 loop tree arrange for the child function to fixup loops. */
13176 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13177 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
13179 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13180 num
= vec_safe_length (child_cfun
->local_decls
);
13181 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
13183 t
= (*child_cfun
->local_decls
)[srcidx
];
13184 if (DECL_CONTEXT (t
) == cfun
->decl
)
13186 if (srcidx
!= dstidx
)
13187 (*child_cfun
->local_decls
)[dstidx
] = t
;
13191 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
13193 /* Inform the callgraph about the new function. */
13194 child_cfun
->curr_properties
= cfun
->curr_properties
;
13195 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
13196 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
13197 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
13198 node
->parallelized_function
= 1;
13199 cgraph_node::add_new_function (child_fn
, true);
13201 /* Add the new function to the offload table. */
13202 if (ENABLE_OFFLOADING
)
13203 vec_safe_push (offload_funcs
, child_fn
);
13205 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
13206 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
13208 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13209 fixed in a following pass. */
13210 push_cfun (child_cfun
);
13212 assign_assembler_name_if_neeeded (child_fn
);
13213 cgraph_edge::rebuild_edges ();
13215 /* Some EH regions might become dead, see PR34608. If
13216 pass_cleanup_cfg isn't the first pass to happen with the
13217 new child, these dead EH edges might cause problems.
13218 Clean them up now. */
13219 if (flag_exceptions
)
13222 bool changed
= false;
13224 FOR_EACH_BB_FN (bb
, cfun
)
13225 changed
|= gimple_purge_dead_eh_edges (bb
);
13227 cleanup_tree_cfg ();
13229 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13230 verify_loop_structure ();
13233 if (dump_file
&& !gimple_in_ssa_p (cfun
))
13235 omp_any_child_fn_dumped
= true;
13236 dump_function_header (dump_file
, child_fn
, dump_flags
);
13237 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
13241 /* Emit a library call to launch the offloading region, or do data
13243 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
13244 enum built_in_function start_ix
;
13245 location_t clause_loc
;
13246 unsigned int flags_i
= 0;
13247 bool oacc_kernels_p
= false;
13249 switch (gimple_omp_target_kind (entry_stmt
))
13251 case GF_OMP_TARGET_KIND_REGION
:
13252 start_ix
= BUILT_IN_GOMP_TARGET
;
13254 case GF_OMP_TARGET_KIND_DATA
:
13255 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
13257 case GF_OMP_TARGET_KIND_UPDATE
:
13258 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
13260 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13261 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13263 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13264 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13265 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
13267 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13268 oacc_kernels_p
= true;
13270 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13271 start_ix
= BUILT_IN_GOACC_PARALLEL
;
13273 case GF_OMP_TARGET_KIND_OACC_DATA
:
13274 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13275 start_ix
= BUILT_IN_GOACC_DATA_START
;
13277 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13278 start_ix
= BUILT_IN_GOACC_UPDATE
;
13280 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13281 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
13283 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13284 start_ix
= BUILT_IN_GOACC_DECLARE
;
13287 gcc_unreachable ();
13290 clauses
= gimple_omp_target_clauses (entry_stmt
);
13292 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13293 library choose) and there is no conditional. */
13295 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
13297 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
13299 cond
= OMP_CLAUSE_IF_EXPR (c
);
13301 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
13304 /* Even if we pass it to all library function calls, it is currently only
13305 defined/used for the OpenMP target ones. */
13306 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
13307 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
13308 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
13309 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
13311 device
= OMP_CLAUSE_DEVICE_ID (c
);
13312 clause_loc
= OMP_CLAUSE_LOCATION (c
);
13315 clause_loc
= gimple_location (entry_stmt
);
13317 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
13319 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
13321 /* Ensure 'device' is of the correct type. */
13322 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
13324 /* If we found the clause 'if (cond)', build
13325 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13328 cond
= gimple_boolify (cond
);
13330 basic_block cond_bb
, then_bb
, else_bb
;
13334 tmp_var
= create_tmp_var (TREE_TYPE (device
));
13336 e
= split_block_after_labels (new_bb
);
13339 gsi
= gsi_last_bb (new_bb
);
13341 e
= split_block (new_bb
, gsi_stmt (gsi
));
13347 then_bb
= create_empty_bb (cond_bb
);
13348 else_bb
= create_empty_bb (then_bb
);
13349 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
13350 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
13352 stmt
= gimple_build_cond_empty (cond
);
13353 gsi
= gsi_last_bb (cond_bb
);
13354 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13356 gsi
= gsi_start_bb (then_bb
);
13357 stmt
= gimple_build_assign (tmp_var
, device
);
13358 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13360 gsi
= gsi_start_bb (else_bb
);
13361 stmt
= gimple_build_assign (tmp_var
,
13362 build_int_cst (integer_type_node
,
13363 GOMP_DEVICE_HOST_FALLBACK
));
13364 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13366 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
13367 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
13368 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
13369 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
13370 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
13371 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
13374 gsi
= gsi_last_bb (new_bb
);
13378 gsi
= gsi_last_bb (new_bb
);
13379 device
= force_gimple_operand_gsi (&gsi
, device
, true, NULL_TREE
,
13380 true, GSI_SAME_STMT
);
13383 t
= gimple_omp_target_data_arg (entry_stmt
);
13386 t1
= size_zero_node
;
13387 t2
= build_zero_cst (ptr_type_node
);
13393 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
13394 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
13395 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
13396 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
13397 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
13401 bool tagging
= false;
13402 /* The maximum number used by any start_ix, without varargs. */
13403 auto_vec
<tree
, 11> args
;
13404 args
.quick_push (device
);
13406 args
.quick_push (build_fold_addr_expr (child_fn
));
13407 args
.quick_push (t1
);
13408 args
.quick_push (t2
);
13409 args
.quick_push (t3
);
13410 args
.quick_push (t4
);
13413 case BUILT_IN_GOACC_DATA_START
:
13414 case BUILT_IN_GOACC_DECLARE
:
13415 case BUILT_IN_GOMP_TARGET_DATA
:
13417 case BUILT_IN_GOMP_TARGET
:
13418 case BUILT_IN_GOMP_TARGET_UPDATE
:
13419 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
13420 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
13421 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
13423 depend
= OMP_CLAUSE_DECL (c
);
13425 depend
= build_int_cst (ptr_type_node
, 0);
13426 args
.quick_push (depend
);
13427 if (start_ix
== BUILT_IN_GOMP_TARGET
)
13428 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
13430 case BUILT_IN_GOACC_PARALLEL
:
13432 set_oacc_fn_attrib (child_fn
, clauses
, oacc_kernels_p
, &args
);
13436 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
13437 case BUILT_IN_GOACC_UPDATE
:
13439 tree t_async
= NULL_TREE
;
13441 /* If present, use the value specified by the respective
13442 clause, making sure that is of the correct type. */
13443 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
13445 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13447 OMP_CLAUSE_ASYNC_EXPR (c
));
13449 /* Default values for t_async. */
13450 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
13452 build_int_cst (integer_type_node
,
13454 if (tagging
&& t_async
)
13456 unsigned HOST_WIDE_INT i_async
= GOMP_LAUNCH_OP_MAX
;
13458 if (TREE_CODE (t_async
) == INTEGER_CST
)
13460 /* See if we can pack the async arg in to the tag's
13462 i_async
= TREE_INT_CST_LOW (t_async
);
13463 if (i_async
< GOMP_LAUNCH_OP_MAX
)
13464 t_async
= NULL_TREE
;
13466 i_async
= GOMP_LAUNCH_OP_MAX
;
13468 args
.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC
, NULL_TREE
,
13472 args
.safe_push (t_async
);
13474 /* Save the argument index, and ... */
13475 unsigned t_wait_idx
= args
.length ();
13476 unsigned num_waits
= 0;
13477 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
13479 /* ... push a placeholder. */
13480 args
.safe_push (integer_zero_node
);
13482 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
13483 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
13485 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13487 OMP_CLAUSE_WAIT_EXPR (c
)));
13491 if (!tagging
|| num_waits
)
13495 /* Now that we know the number, update the placeholder. */
13497 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
13499 len
= build_int_cst (integer_type_node
, num_waits
);
13500 len
= fold_convert_loc (gimple_location (entry_stmt
),
13501 unsigned_type_node
, len
);
13502 args
[t_wait_idx
] = len
;
13507 gcc_unreachable ();
13510 /* Push terminal marker - zero. */
13511 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
13513 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
13514 gimple_set_location (g
, gimple_location (entry_stmt
));
13515 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13518 g
= gsi_stmt (gsi
);
13519 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
13520 gsi_remove (&gsi
, true);
13522 if (data_region
&& region
->exit
)
13524 gsi
= gsi_last_bb (region
->exit
);
13525 g
= gsi_stmt (gsi
);
13526 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
13527 gsi_remove (&gsi
, true);
13531 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13532 variable derived from the thread number. */
13535 grid_expand_omp_for_loop (struct omp_region
*kfor
)
13539 gimple_stmt_iterator gsi
;
13541 struct omp_for_data fd
;
13543 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
13544 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
13545 == GF_OMP_FOR_KIND_GRID_LOOP
);
13546 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
13548 gcc_assert (gimple_omp_for_collapse (for_stmt
) == 1);
13549 gcc_assert (kfor
->cont
);
13550 extract_omp_for_data (for_stmt
, &fd
, NULL
);
13552 itype
= type
= TREE_TYPE (fd
.loop
.v
);
13553 if (POINTER_TYPE_P (type
))
13554 itype
= signed_type_for (type
);
13556 gsi
= gsi_start_bb (body_bb
);
13559 step
= fd
.loop
.step
;
13560 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
13561 true, NULL_TREE
, true, GSI_SAME_STMT
);
13562 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
13563 true, NULL_TREE
, true, GSI_SAME_STMT
);
13564 threadid
= build_call_expr (builtin_decl_explicit
13565 (BUILT_IN_OMP_GET_THREAD_NUM
), 0);
13566 threadid
= fold_convert (itype
, threadid
);
13567 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
13568 true, GSI_SAME_STMT
);
13570 tree startvar
= fd
.loop
.v
;
13571 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
13572 if (POINTER_TYPE_P (type
))
13573 t
= fold_build_pointer_plus (n1
, t
);
13575 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
13576 t
= fold_convert (type
, t
);
13577 t
= force_gimple_operand_gsi (&gsi
, t
,
13579 && TREE_ADDRESSABLE (startvar
),
13580 NULL_TREE
, true, GSI_SAME_STMT
);
13581 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
13582 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
13584 /* Remove the omp for statement */
13585 gsi
= gsi_last_bb (kfor
->entry
);
13586 gsi_remove (&gsi
, true);
13588 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13589 gsi
= gsi_last_bb (kfor
->cont
);
13590 gcc_assert (!gsi_end_p (gsi
)
13591 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
13592 gsi_remove (&gsi
, true);
13594 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13595 gsi
= gsi_last_bb (kfor
->exit
);
13596 gcc_assert (!gsi_end_p (gsi
)
13597 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13598 gsi_remove (&gsi
, true);
13600 /* Fixup the much simpler CFG. */
13601 remove_edge (find_edge (kfor
->cont
, body_bb
));
13603 if (kfor
->cont
!= body_bb
)
13604 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
13605 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
13608 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13611 struct grid_arg_decl_map
13617 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13618 pertaining to kernel function. */
13621 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
13623 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13624 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
13627 if (t
== adm
->old_arg
)
13628 *tp
= adm
->new_arg
;
13629 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
13633 static void expand_omp (struct omp_region
*region
);
13635 /* If TARGET region contains a kernel body for loop, remove its region from the
13636 TARGET and expand it in GPGPU kernel fashion. */
13639 grid_expand_target_grid_body (struct omp_region
*target
)
13641 if (!hsa_gen_requested_p ())
13644 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
13645 struct omp_region
**pp
;
13647 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
13648 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
13651 struct omp_region
*gpukernel
= *pp
;
13653 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
13656 /* HSA cannot handle OACC stuff. */
13657 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
13659 gcc_checking_assert (orig_child_fndecl
);
13660 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13661 OMP_CLAUSE__GRIDDIM_
));
13662 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
13664 hsa_register_kernel (n
);
13668 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13669 OMP_CLAUSE__GRIDDIM_
));
13670 tree inside_block
= gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
13671 *pp
= gpukernel
->next
;
13672 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
13673 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
13676 struct omp_region
*kfor
= *pp
;
13678 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor
)->entry
))
13679 == GF_OMP_FOR_KIND_GRID_LOOP
);
13682 expand_omp (kfor
->inner
);
13683 if (gpukernel
->inner
)
13684 expand_omp (gpukernel
->inner
);
13686 tree kern_fndecl
= copy_node (orig_child_fndecl
);
13687 DECL_NAME (kern_fndecl
) = clone_function_name (kern_fndecl
, "kernel");
13688 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
13689 tree tgtblock
= gimple_block (tgt_stmt
);
13690 tree fniniblock
= make_node (BLOCK
);
13691 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = tgtblock
;
13692 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
13693 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
13694 BLOCK_SUPERCONTEXT (fniniblock
) = kern_fndecl
;
13695 DECL_INITIAL (kern_fndecl
) = fniniblock
;
13696 push_struct_function (kern_fndecl
);
13697 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
13698 init_tree_ssa (cfun
);
13701 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
13702 gcc_assert (!DECL_CHAIN (old_parm_decl
));
13703 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
13704 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
13705 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
13706 gcc_assert (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (kern_fndecl
))));
13707 DECL_RESULT (kern_fndecl
) = copy_node (DECL_RESULT (kern_fndecl
));
13708 DECL_CONTEXT (DECL_RESULT (kern_fndecl
)) = kern_fndecl
;
13709 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
13710 kern_cfun
->curr_properties
= cfun
->curr_properties
;
13712 remove_edge (BRANCH_EDGE (kfor
->entry
));
13713 grid_expand_omp_for_loop (kfor
);
13715 /* Remove the omp for statement */
13716 gimple_stmt_iterator gsi
= gsi_last_bb (gpukernel
->entry
);
13717 gsi_remove (&gsi
, true);
13718 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13720 gsi
= gsi_last_bb (gpukernel
->exit
);
13721 gcc_assert (!gsi_end_p (gsi
)
13722 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13723 gimple
*ret_stmt
= gimple_build_return (NULL
);
13724 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
13725 gsi_remove (&gsi
, true);
13727 /* Statements in the first BB in the target construct have been produced by
13728 target lowering and must be copied inside the GPUKERNEL, with the two
13729 exceptions of the first OMP statement and the OMP_DATA assignment
13731 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
13732 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
13733 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
13734 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
13735 !gsi_end_p (tsi
); gsi_next (&tsi
))
13737 gimple
*stmt
= gsi_stmt (tsi
);
13738 if (is_gimple_omp (stmt
))
13741 && is_gimple_assign (stmt
)
13742 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
13743 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
13745 gimple
*copy
= gimple_copy (stmt
);
13746 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
13747 gimple_set_block (copy
, fniniblock
);
13750 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
13751 gpukernel
->exit
, inside_block
);
13753 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
13754 kcn
->mark_force_output ();
13755 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
13757 hsa_register_kernel (kcn
, orig_child
);
13759 cgraph_node::add_new_function (kern_fndecl
, true);
13760 push_cfun (kern_cfun
);
13761 cgraph_edge::rebuild_edges ();
13763 /* Re-map any mention of the PARM_DECL of the original function to the
13764 PARM_DECL of the new one.
13766 TODO: It would be great if lowering produced references into the GPU
13767 kernel decl straight away and we did not have to do this. */
13768 struct grid_arg_decl_map adm
;
13769 adm
.old_arg
= old_parm_decl
;
13770 adm
.new_arg
= new_parm_decl
;
13772 FOR_EACH_BB_FN (bb
, kern_cfun
)
13774 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13776 gimple
*stmt
= gsi_stmt (gsi
);
13777 struct walk_stmt_info wi
;
13778 memset (&wi
, 0, sizeof (wi
));
13780 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
13788 /* Expand the parallel region tree rooted at REGION. Expansion
13789 proceeds in depth-first order. Innermost regions are expanded
13790 first. This way, parallel regions that require a new function to
13791 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13792 internal dependencies in their body. */
13795 expand_omp (struct omp_region
*region
)
13797 omp_any_child_fn_dumped
= false;
13800 location_t saved_location
;
13801 gimple
*inner_stmt
= NULL
;
13803 /* First, determine whether this is a combined parallel+workshare
13805 if (region
->type
== GIMPLE_OMP_PARALLEL
)
13806 determine_parallel_type (region
);
13807 else if (region
->type
== GIMPLE_OMP_TARGET
)
13808 grid_expand_target_grid_body (region
);
13810 if (region
->type
== GIMPLE_OMP_FOR
13811 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
13812 inner_stmt
= last_stmt (region
->inner
->entry
);
13815 expand_omp (region
->inner
);
13817 saved_location
= input_location
;
13818 if (gimple_has_location (last_stmt (region
->entry
)))
13819 input_location
= gimple_location (last_stmt (region
->entry
));
13821 switch (region
->type
)
13823 case GIMPLE_OMP_PARALLEL
:
13824 case GIMPLE_OMP_TASK
:
13825 expand_omp_taskreg (region
);
13828 case GIMPLE_OMP_FOR
:
13829 expand_omp_for (region
, inner_stmt
);
13832 case GIMPLE_OMP_SECTIONS
:
13833 expand_omp_sections (region
);
13836 case GIMPLE_OMP_SECTION
:
13837 /* Individual omp sections are handled together with their
13838 parent GIMPLE_OMP_SECTIONS region. */
13841 case GIMPLE_OMP_SINGLE
:
13842 expand_omp_single (region
);
13845 case GIMPLE_OMP_ORDERED
:
13847 gomp_ordered
*ord_stmt
13848 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13849 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13850 OMP_CLAUSE_DEPEND
))
13852 /* We'll expand these when expanding corresponding
13853 worksharing region with ordered(n) clause. */
13854 gcc_assert (region
->outer
13855 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13856 region
->ord_stmt
= ord_stmt
;
13861 case GIMPLE_OMP_MASTER
:
13862 case GIMPLE_OMP_TASKGROUP
:
13863 case GIMPLE_OMP_CRITICAL
:
13864 case GIMPLE_OMP_TEAMS
:
13865 expand_omp_synch (region
);
13868 case GIMPLE_OMP_ATOMIC_LOAD
:
13869 expand_omp_atomic (region
);
13872 case GIMPLE_OMP_TARGET
:
13873 expand_omp_target (region
);
13877 gcc_unreachable ();
13880 input_location
= saved_location
;
13881 region
= region
->next
;
13883 if (omp_any_child_fn_dumped
)
13886 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13887 omp_any_child_fn_dumped
= false;
13892 /* Helper for build_omp_regions. Scan the dominator tree starting at
13893 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13894 true, the function ends once a single tree is built (otherwise, whole
13895 forest of OMP constructs may be built). */
13898 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13901 gimple_stmt_iterator gsi
;
13905 gsi
= gsi_last_bb (bb
);
13906 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13908 struct omp_region
*region
;
13909 enum gimple_code code
;
13911 stmt
= gsi_stmt (gsi
);
13912 code
= gimple_code (stmt
);
13913 if (code
== GIMPLE_OMP_RETURN
)
13915 /* STMT is the return point out of region PARENT. Mark it
13916 as the exit point and make PARENT the immediately
13917 enclosing region. */
13918 gcc_assert (parent
);
13921 parent
= parent
->outer
;
13923 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13925 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13926 GIMPLE_OMP_RETURN, but matches with
13927 GIMPLE_OMP_ATOMIC_LOAD. */
13928 gcc_assert (parent
);
13929 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13932 parent
= parent
->outer
;
13934 else if (code
== GIMPLE_OMP_CONTINUE
)
13936 gcc_assert (parent
);
13939 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13941 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13942 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13946 region
= new_omp_region (bb
, code
, parent
);
13948 if (code
== GIMPLE_OMP_TARGET
)
13950 switch (gimple_omp_target_kind (stmt
))
13952 case GF_OMP_TARGET_KIND_REGION
:
13953 case GF_OMP_TARGET_KIND_DATA
:
13954 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13955 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13956 case GF_OMP_TARGET_KIND_OACC_DATA
:
13957 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13959 case GF_OMP_TARGET_KIND_UPDATE
:
13960 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13961 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13962 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13963 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13964 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13965 /* ..., other than for those stand-alone directives... */
13969 gcc_unreachable ();
13972 else if (code
== GIMPLE_OMP_ORDERED
13973 && find_omp_clause (gimple_omp_ordered_clauses
13974 (as_a
<gomp_ordered
*> (stmt
)),
13975 OMP_CLAUSE_DEPEND
))
13976 /* #pragma omp ordered depend is also just a stand-alone
13979 /* ..., this directive becomes the parent for a new region. */
13985 if (single_tree
&& !parent
)
13988 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13990 son
= next_dom_son (CDI_DOMINATORS
, son
))
13991 build_omp_regions_1 (son
, parent
, single_tree
);
13994 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13995 root_omp_region. */
13998 build_omp_regions_root (basic_block root
)
14000 gcc_assert (root_omp_region
== NULL
);
14001 build_omp_regions_1 (root
, NULL
, true);
14002 gcc_assert (root_omp_region
!= NULL
);
14005 /* Expands omp construct (and its subconstructs) starting in HEAD. */
14008 omp_expand_local (basic_block head
)
14010 build_omp_regions_root (head
);
14011 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
14013 fprintf (dump_file
, "\nOMP region tree\n\n");
14014 dump_omp_region (dump_file
, root_omp_region
, 0);
14015 fprintf (dump_file
, "\n");
14018 remove_exit_barriers (root_omp_region
);
14019 expand_omp (root_omp_region
);
14021 free_omp_regions ();
14024 /* Scan the CFG and build a tree of OMP regions. Return the root of
14025 the OMP region tree. */
14028 build_omp_regions (void)
14030 gcc_assert (root_omp_region
== NULL
);
14031 calculate_dominance_info (CDI_DOMINATORS
);
14032 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
14035 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
14037 static unsigned int
14038 execute_expand_omp (void)
14040 build_omp_regions ();
14042 if (!root_omp_region
)
14047 fprintf (dump_file
, "\nOMP region tree\n\n");
14048 dump_omp_region (dump_file
, root_omp_region
, 0);
14049 fprintf (dump_file
, "\n");
14052 remove_exit_barriers (root_omp_region
);
14054 expand_omp (root_omp_region
);
14056 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
14057 verify_loop_structure ();
14058 cleanup_tree_cfg ();
14060 free_omp_regions ();
14065 /* OMP expansion -- the default pass, run before creation of SSA form. */
14069 const pass_data pass_data_expand_omp
=
14071 GIMPLE_PASS
, /* type */
14072 "ompexp", /* name */
14073 OPTGROUP_NONE
, /* optinfo_flags */
14074 TV_NONE
, /* tv_id */
14075 PROP_gimple_any
, /* properties_required */
14076 PROP_gimple_eomp
, /* properties_provided */
14077 0, /* properties_destroyed */
14078 0, /* todo_flags_start */
14079 0, /* todo_flags_finish */
14082 class pass_expand_omp
: public gimple_opt_pass
14085 pass_expand_omp (gcc::context
*ctxt
)
14086 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
14089 /* opt_pass methods: */
14090 virtual unsigned int execute (function
*)
14092 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
14093 || flag_openmp_simd
!= 0)
14094 && !seen_error ());
14096 /* This pass always runs, to provide PROP_gimple_eomp.
14097 But often, there is nothing to do. */
14101 return execute_expand_omp ();
14104 }; // class pass_expand_omp
14106 } // anon namespace
14109 make_pass_expand_omp (gcc::context
*ctxt
)
14111 return new pass_expand_omp (ctxt
);
14116 const pass_data pass_data_expand_omp_ssa
=
14118 GIMPLE_PASS
, /* type */
14119 "ompexpssa", /* name */
14120 OPTGROUP_NONE
, /* optinfo_flags */
14121 TV_NONE
, /* tv_id */
14122 PROP_cfg
| PROP_ssa
, /* properties_required */
14123 PROP_gimple_eomp
, /* properties_provided */
14124 0, /* properties_destroyed */
14125 0, /* todo_flags_start */
14126 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
14129 class pass_expand_omp_ssa
: public gimple_opt_pass
14132 pass_expand_omp_ssa (gcc::context
*ctxt
)
14133 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
14136 /* opt_pass methods: */
14137 virtual bool gate (function
*fun
)
14139 return !(fun
->curr_properties
& PROP_gimple_eomp
);
14141 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
14142 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
14144 }; // class pass_expand_omp_ssa
14146 } // anon namespace
14149 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
14151 return new pass_expand_omp_ssa (ctxt
);
14154 /* Routines to lower OMP directives into OMP-GIMPLE. */
14156 /* If ctx is a worksharing context inside of a cancellable parallel
14157 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14158 and conditional branch to parallel's cancel_label to handle
14159 cancellation in the implicit barrier. */
14162 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
14164 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
14165 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
14166 if (gimple_omp_return_nowait_p (omp_return
))
14169 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
14170 && ctx
->outer
->cancellable
)
14172 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
14173 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
14174 tree lhs
= create_tmp_var (c_bool_type
);
14175 gimple_omp_return_set_lhs (omp_return
, lhs
);
14176 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
14177 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
14178 fold_convert (c_bool_type
,
14179 boolean_false_node
),
14180 ctx
->outer
->cancel_label
, fallthru_label
);
14181 gimple_seq_add_stmt (body
, g
);
14182 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
14186 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14187 CTX is the enclosing OMP context for the current statement. */
14190 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14192 tree block
, control
;
14193 gimple_stmt_iterator tgsi
;
14194 gomp_sections
*stmt
;
14196 gbind
*new_stmt
, *bind
;
14197 gimple_seq ilist
, dlist
, olist
, new_body
;
14199 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
14201 push_gimplify_context ();
14205 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
14206 &ilist
, &dlist
, ctx
, NULL
);
14208 new_body
= gimple_omp_body (stmt
);
14209 gimple_omp_set_body (stmt
, NULL
);
14210 tgsi
= gsi_start (new_body
);
14211 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
14216 sec_start
= gsi_stmt (tgsi
);
14217 sctx
= maybe_lookup_ctx (sec_start
);
14220 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
14221 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
14222 GSI_CONTINUE_LINKING
);
14223 gimple_omp_set_body (sec_start
, NULL
);
14225 if (gsi_one_before_end_p (tgsi
))
14227 gimple_seq l
= NULL
;
14228 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
14230 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
14231 gimple_omp_section_set_last (sec_start
);
14234 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
14235 GSI_CONTINUE_LINKING
);
14238 block
= make_node (BLOCK
);
14239 bind
= gimple_build_bind (NULL
, new_body
, block
);
14242 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
14244 block
= make_node (BLOCK
);
14245 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14246 gsi_replace (gsi_p
, new_stmt
, true);
14248 pop_gimplify_context (new_stmt
);
14249 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14250 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14251 if (BLOCK_VARS (block
))
14252 TREE_USED (block
) = 1;
14255 gimple_seq_add_seq (&new_body
, ilist
);
14256 gimple_seq_add_stmt (&new_body
, stmt
);
14257 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
14258 gimple_seq_add_stmt (&new_body
, bind
);
14260 control
= create_tmp_var (unsigned_type_node
, ".section");
14261 t
= gimple_build_omp_continue (control
, control
);
14262 gimple_omp_sections_set_control (stmt
, control
);
14263 gimple_seq_add_stmt (&new_body
, t
);
14265 gimple_seq_add_seq (&new_body
, olist
);
14266 if (ctx
->cancellable
)
14267 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14268 gimple_seq_add_seq (&new_body
, dlist
);
14270 new_body
= maybe_catch_exception (new_body
);
14272 t
= gimple_build_omp_return
14273 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
14274 OMP_CLAUSE_NOWAIT
));
14275 gimple_seq_add_stmt (&new_body
, t
);
14276 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
14278 gimple_bind_set_body (new_stmt
, new_body
);
14282 /* A subroutine of lower_omp_single. Expand the simple form of
14283 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14285 if (GOMP_single_start ())
14287 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14289 FIXME. It may be better to delay expanding the logic of this until
14290 pass_expand_omp. The expanded logic may make the job more difficult
14291 to a synchronization analysis pass. */
14294 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
14296 location_t loc
= gimple_location (single_stmt
);
14297 tree tlabel
= create_artificial_label (loc
);
14298 tree flabel
= create_artificial_label (loc
);
14299 gimple
*call
, *cond
;
14302 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
14303 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
14304 call
= gimple_build_call (decl
, 0);
14305 gimple_call_set_lhs (call
, lhs
);
14306 gimple_seq_add_stmt (pre_p
, call
);
14308 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
14309 fold_convert_loc (loc
, TREE_TYPE (lhs
),
14310 boolean_true_node
),
14312 gimple_seq_add_stmt (pre_p
, cond
);
14313 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
14314 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14315 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
14319 /* A subroutine of lower_omp_single. Expand the simple form of
14320 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14322 #pragma omp single copyprivate (a, b, c)
14324 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14327 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14333 GOMP_single_copy_end (©out);
14344 FIXME. It may be better to delay expanding the logic of this until
14345 pass_expand_omp. The expanded logic may make the job more difficult
14346 to a synchronization analysis pass. */
14349 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
14352 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
14353 gimple_seq copyin_seq
;
14354 location_t loc
= gimple_location (single_stmt
);
14356 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
14358 ptr_type
= build_pointer_type (ctx
->record_type
);
14359 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
14361 l0
= create_artificial_label (loc
);
14362 l1
= create_artificial_label (loc
);
14363 l2
= create_artificial_label (loc
);
14365 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
14366 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
14367 t
= fold_convert_loc (loc
, ptr_type
, t
);
14368 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
14370 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
14371 build_int_cst (ptr_type
, 0));
14372 t
= build3 (COND_EXPR
, void_type_node
, t
,
14373 build_and_jump (&l0
), build_and_jump (&l1
));
14374 gimplify_and_add (t
, pre_p
);
14376 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
14378 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14381 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
14384 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14385 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
14386 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
14387 gimplify_and_add (t
, pre_p
);
14389 t
= build_and_jump (&l2
);
14390 gimplify_and_add (t
, pre_p
);
14392 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
14394 gimple_seq_add_seq (pre_p
, copyin_seq
);
14396 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
14400 /* Expand code for an OpenMP single directive. */
14403 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14407 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
14409 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
14411 push_gimplify_context ();
14413 block
= make_node (BLOCK
);
14414 bind
= gimple_build_bind (NULL
, NULL
, block
);
14415 gsi_replace (gsi_p
, bind
, true);
14418 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
14419 &bind_body
, &dlist
, ctx
, NULL
);
14420 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
14422 gimple_seq_add_stmt (&bind_body
, single_stmt
);
14424 if (ctx
->record_type
)
14425 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
14427 lower_omp_single_simple (single_stmt
, &bind_body
);
14429 gimple_omp_set_body (single_stmt
, NULL
);
14431 gimple_seq_add_seq (&bind_body
, dlist
);
14433 bind_body
= maybe_catch_exception (bind_body
);
14435 t
= gimple_build_omp_return
14436 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
14437 OMP_CLAUSE_NOWAIT
));
14438 gimple_seq_add_stmt (&bind_body_tail
, t
);
14439 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
14440 if (ctx
->record_type
)
14442 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
14443 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14444 TREE_THIS_VOLATILE (clobber
) = 1;
14445 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
14446 clobber
), GSI_SAME_STMT
);
14448 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
14449 gimple_bind_set_body (bind
, bind_body
);
14451 pop_gimplify_context (bind
);
14453 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14454 BLOCK_VARS (block
) = ctx
->block_vars
;
14455 if (BLOCK_VARS (block
))
14456 TREE_USED (block
) = 1;
14460 /* Expand code for an OpenMP master directive. */
14463 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14465 tree block
, lab
= NULL
, x
, bfn_decl
;
14466 gimple
*stmt
= gsi_stmt (*gsi_p
);
14468 location_t loc
= gimple_location (stmt
);
14471 push_gimplify_context ();
14473 block
= make_node (BLOCK
);
14474 bind
= gimple_build_bind (NULL
, NULL
, block
);
14475 gsi_replace (gsi_p
, bind
, true);
14476 gimple_bind_add_stmt (bind
, stmt
);
14478 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
14479 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
14480 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
14481 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
14483 gimplify_and_add (x
, &tseq
);
14484 gimple_bind_add_seq (bind
, tseq
);
14486 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14487 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14488 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14489 gimple_omp_set_body (stmt
, NULL
);
14491 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
14493 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14495 pop_gimplify_context (bind
);
14497 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14498 BLOCK_VARS (block
) = ctx
->block_vars
;
14502 /* Expand code for an OpenMP taskgroup directive. */
14505 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14507 gimple
*stmt
= gsi_stmt (*gsi_p
);
14510 tree block
= make_node (BLOCK
);
14512 bind
= gimple_build_bind (NULL
, NULL
, block
);
14513 gsi_replace (gsi_p
, bind
, true);
14514 gimple_bind_add_stmt (bind
, stmt
);
14516 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
14518 gimple_bind_add_stmt (bind
, x
);
14520 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14521 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14522 gimple_omp_set_body (stmt
, NULL
);
14524 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14526 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14527 BLOCK_VARS (block
) = ctx
->block_vars
;
14531 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14534 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
14537 struct omp_for_data fd
;
14538 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
14541 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
14542 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
14543 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
14547 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14548 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
14549 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14550 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
14552 /* Merge depend clauses from multiple adjacent
14553 #pragma omp ordered depend(sink:...) constructs
14554 into one #pragma omp ordered depend(sink:...), so that
14555 we can optimize them together. */
14556 gimple_stmt_iterator gsi
= *gsi_p
;
14558 while (!gsi_end_p (gsi
))
14560 gimple
*stmt
= gsi_stmt (gsi
);
14561 if (is_gimple_debug (stmt
)
14562 || gimple_code (stmt
) == GIMPLE_NOP
)
14567 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
14569 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
14570 c
= gimple_omp_ordered_clauses (ord_stmt2
);
14572 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
14573 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14576 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
14578 gsi_remove (&gsi
, true);
14582 /* Canonicalize sink dependence clauses into one folded clause if
14585 The basic algorithm is to create a sink vector whose first
14586 element is the GCD of all the first elements, and whose remaining
14587 elements are the minimum of the subsequent columns.
14589 We ignore dependence vectors whose first element is zero because
14590 such dependencies are known to be executed by the same thread.
14592 We take into account the direction of the loop, so a minimum
14593 becomes a maximum if the loop is iterating forwards. We also
14594 ignore sink clauses where the loop direction is unknown, or where
14595 the offsets are clearly invalid because they are not a multiple
14596 of the loop increment.
14600 #pragma omp for ordered(2)
14601 for (i=0; i < N; ++i)
14602 for (j=0; j < M; ++j)
14604 #pragma omp ordered \
14605 depend(sink:i-8,j-2) \
14606 depend(sink:i,j-1) \ // Completely ignored because i+0.
14607 depend(sink:i-4,j-3) \
14608 depend(sink:i-6,j-4)
14609 #pragma omp ordered depend(source)
14614 depend(sink:-gcd(8,4,6),-min(2,3,4))
14619 /* FIXME: Computing GCD's where the first element is zero is
14620 non-trivial in the presence of collapsed loops. Do this later. */
14621 if (fd
.collapse
> 1)
14624 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
14625 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
14626 tree folded_dep
= NULL_TREE
;
14627 /* TRUE if the first dimension's offset is negative. */
14628 bool neg_offset_p
= false;
14630 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14632 while ((c
= *list_p
) != NULL
)
14634 bool remove
= false;
14636 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
14637 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14638 goto next_ordered_clause
;
14641 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
14642 vec
&& TREE_CODE (vec
) == TREE_LIST
;
14643 vec
= TREE_CHAIN (vec
), ++i
)
14645 gcc_assert (i
< len
);
14647 /* extract_omp_for_data has canonicalized the condition. */
14648 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
14649 || fd
.loops
[i
].cond_code
== GT_EXPR
);
14650 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
14651 bool maybe_lexically_later
= true;
14653 /* While the committee makes up its mind, bail if we have any
14654 non-constant steps. */
14655 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
14656 goto lower_omp_ordered_ret
;
14658 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
14659 if (POINTER_TYPE_P (itype
))
14661 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
14662 TYPE_PRECISION (itype
),
14663 TYPE_SIGN (itype
));
14665 /* Ignore invalid offsets that are not multiples of the step. */
14666 if (!wi::multiple_of_p
14667 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
14670 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
14671 "ignoring sink clause with offset that is not "
14672 "a multiple of the loop step");
14674 goto next_ordered_clause
;
14677 /* Calculate the first dimension. The first dimension of
14678 the folded dependency vector is the GCD of the first
14679 elements, while ignoring any first elements whose offset
14683 /* Ignore dependence vectors whose first dimension is 0. */
14687 goto next_ordered_clause
;
14691 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
14693 error_at (OMP_CLAUSE_LOCATION (c
),
14694 "first offset must be in opposite direction "
14695 "of loop iterations");
14696 goto lower_omp_ordered_ret
;
14700 neg_offset_p
= forward
;
14701 /* Initialize the first time around. */
14702 if (folded_dep
== NULL_TREE
)
14705 folded_deps
[0] = offset
;
14708 folded_deps
[0] = wi::gcd (folded_deps
[0],
14712 /* Calculate minimum for the remaining dimensions. */
14715 folded_deps
[len
+ i
- 1] = offset
;
14716 if (folded_dep
== c
)
14717 folded_deps
[i
] = offset
;
14718 else if (maybe_lexically_later
14719 && !wi::eq_p (folded_deps
[i
], offset
))
14721 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
14725 for (j
= 1; j
<= i
; j
++)
14726 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
14729 maybe_lexically_later
= false;
14733 gcc_assert (i
== len
);
14737 next_ordered_clause
:
14739 *list_p
= OMP_CLAUSE_CHAIN (c
);
14741 list_p
= &OMP_CLAUSE_CHAIN (c
);
14747 folded_deps
[0] = -folded_deps
[0];
14749 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
14750 if (POINTER_TYPE_P (itype
))
14753 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14754 = wide_int_to_tree (itype
, folded_deps
[0]);
14755 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14756 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14759 lower_omp_ordered_ret
:
14761 /* Ordered without clauses is #pragma omp threads, while we want
14762 a nop instead if we remove all clauses. */
14763 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14764 gsi_replace (gsi_p
, gimple_build_nop (), true);
14768 /* Expand code for an OpenMP ordered directive. */
14771 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14774 gimple
*stmt
= gsi_stmt (*gsi_p
);
14775 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14778 bool simd
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14780 bool threads
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14781 OMP_CLAUSE_THREADS
);
14783 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14784 OMP_CLAUSE_DEPEND
))
14786 /* FIXME: This is needs to be moved to the expansion to verify various
14787 conditions only testable on cfg with dominators computed, and also
14788 all the depend clauses to be merged still might need to be available
14789 for the runtime checks. */
14791 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14795 push_gimplify_context ();
14797 block
= make_node (BLOCK
);
14798 bind
= gimple_build_bind (NULL
, NULL
, block
);
14799 gsi_replace (gsi_p
, bind
, true);
14800 gimple_bind_add_stmt (bind
, stmt
);
14804 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
14805 build_int_cst (NULL_TREE
, threads
));
14806 cfun
->has_simduid_loops
= true;
14809 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14811 gimple_bind_add_stmt (bind
, x
);
14813 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14814 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14815 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14816 gimple_omp_set_body (stmt
, NULL
);
14819 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
14820 build_int_cst (NULL_TREE
, threads
));
14822 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14824 gimple_bind_add_stmt (bind
, x
);
14826 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14828 pop_gimplify_context (bind
);
14830 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14831 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14835 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14836 substitution of a couple of function calls. But in the NAMED case,
14837 requires that languages coordinate a symbol name. It is therefore
14838 best put here in common code. */
14840 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14843 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14846 tree name
, lock
, unlock
;
14847 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14849 location_t loc
= gimple_location (stmt
);
14852 name
= gimple_omp_critical_name (stmt
);
14857 if (!critical_name_mutexes
)
14858 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14860 tree
*n
= critical_name_mutexes
->get (name
);
14865 decl
= create_tmp_var_raw (ptr_type_node
);
14867 new_str
= ACONCAT ((".gomp_critical_user_",
14868 IDENTIFIER_POINTER (name
), NULL
));
14869 DECL_NAME (decl
) = get_identifier (new_str
);
14870 TREE_PUBLIC (decl
) = 1;
14871 TREE_STATIC (decl
) = 1;
14872 DECL_COMMON (decl
) = 1;
14873 DECL_ARTIFICIAL (decl
) = 1;
14874 DECL_IGNORED_P (decl
) = 1;
14876 varpool_node::finalize_decl (decl
);
14878 critical_name_mutexes
->put (name
, decl
);
14883 /* If '#pragma omp critical' is inside offloaded region or
14884 inside function marked as offloadable, the symbol must be
14885 marked as offloadable too. */
14887 if (cgraph_node::get (current_function_decl
)->offloadable
)
14888 varpool_node::get_create (decl
)->offloadable
= 1;
14890 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14891 if (is_gimple_omp_offloaded (octx
->stmt
))
14893 varpool_node::get_create (decl
)->offloadable
= 1;
14897 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14898 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14900 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14901 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14902 build_fold_addr_expr_loc (loc
, decl
));
14906 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14907 lock
= build_call_expr_loc (loc
, lock
, 0);
14909 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14910 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14913 push_gimplify_context ();
14915 block
= make_node (BLOCK
);
14916 bind
= gimple_build_bind (NULL
, NULL
, block
);
14917 gsi_replace (gsi_p
, bind
, true);
14918 gimple_bind_add_stmt (bind
, stmt
);
14920 tbody
= gimple_bind_body (bind
);
14921 gimplify_and_add (lock
, &tbody
);
14922 gimple_bind_set_body (bind
, tbody
);
14924 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14925 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14926 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14927 gimple_omp_set_body (stmt
, NULL
);
14929 tbody
= gimple_bind_body (bind
);
14930 gimplify_and_add (unlock
, &tbody
);
14931 gimple_bind_set_body (bind
, tbody
);
14933 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14935 pop_gimplify_context (bind
);
14936 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14937 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14941 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14942 for a lastprivate clause. Given a loop control predicate of (V
14943 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14944 is appended to *DLIST, iterator initialization is appended to
14948 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14949 gimple_seq
*dlist
, struct omp_context
*ctx
)
14951 tree clauses
, cond
, vinit
;
14952 enum tree_code cond_code
;
14955 cond_code
= fd
->loop
.cond_code
;
14956 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14958 /* When possible, use a strict equality expression. This can let VRP
14959 type optimizations deduce the value and remove a copy. */
14960 if (tree_fits_shwi_p (fd
->loop
.step
))
14962 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14963 if (step
== 1 || step
== -1)
14964 cond_code
= EQ_EXPR
;
14967 tree n2
= fd
->loop
.n2
;
14968 if (fd
->collapse
> 1
14969 && TREE_CODE (n2
) != INTEGER_CST
14970 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14972 struct omp_context
*taskreg_ctx
= NULL
;
14973 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14975 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14976 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14977 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14979 if (gimple_omp_for_combined_into_p (gfor
))
14981 gcc_assert (ctx
->outer
->outer
14982 && is_parallel_ctx (ctx
->outer
->outer
));
14983 taskreg_ctx
= ctx
->outer
->outer
;
14987 struct omp_for_data outer_fd
;
14988 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14989 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14992 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14993 taskreg_ctx
= ctx
->outer
->outer
;
14995 else if (is_taskreg_ctx (ctx
->outer
))
14996 taskreg_ctx
= ctx
->outer
;
15001 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
15002 OMP_CLAUSE__LOOPTEMP_
);
15003 gcc_assert (innerc
);
15004 for (i
= 0; i
< fd
->collapse
; i
++)
15006 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
15007 OMP_CLAUSE__LOOPTEMP_
);
15008 gcc_assert (innerc
);
15010 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
15011 OMP_CLAUSE__LOOPTEMP_
);
15013 n2
= fold_convert (TREE_TYPE (n2
),
15014 lookup_decl (OMP_CLAUSE_DECL (innerc
),
15018 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
15020 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
15022 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
15023 if (!gimple_seq_empty_p (stmts
))
15025 gimple_seq_add_seq (&stmts
, *dlist
);
15028 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
15029 vinit
= fd
->loop
.n1
;
15030 if (cond_code
== EQ_EXPR
15031 && tree_fits_shwi_p (fd
->loop
.n2
)
15032 && ! integer_zerop (fd
->loop
.n2
))
15033 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
15035 vinit
= unshare_expr (vinit
);
15037 /* Initialize the iterator variable, so that threads that don't execute
15038 any iterations don't execute the lastprivate clauses by accident. */
15039 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
15044 /* Lower code for an OMP loop directive. */
15047 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15049 tree
*rhs_p
, block
;
15050 struct omp_for_data fd
, *fdp
= NULL
;
15051 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
15053 gimple_seq omp_for_body
, body
, dlist
;
15054 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
15057 push_gimplify_context ();
15059 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
15061 block
= make_node (BLOCK
);
15062 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
15063 /* Replace at gsi right away, so that 'stmt' is no member
15064 of a sequence anymore as we're going to add to a different
15066 gsi_replace (gsi_p
, new_stmt
, true);
15068 /* Move declaration of temporaries in the loop body before we make
15070 omp_for_body
= gimple_omp_body (stmt
);
15071 if (!gimple_seq_empty_p (omp_for_body
)
15072 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
15075 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
15076 tree vars
= gimple_bind_vars (inner_bind
);
15077 gimple_bind_append_vars (new_stmt
, vars
);
15078 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
15079 keep them on the inner_bind and it's block. */
15080 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
15081 if (gimple_bind_block (inner_bind
))
15082 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
15085 if (gimple_omp_for_combined_into_p (stmt
))
15087 extract_omp_for_data (stmt
, &fd
, NULL
);
15090 /* We need two temporaries with fd.loop.v type (istart/iend)
15091 and then (fd.collapse - 1) temporaries with the same
15092 type for count2 ... countN-1 vars if not constant. */
15094 tree type
= fd
.iter_type
;
15095 if (fd
.collapse
> 1
15096 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
15097 count
+= fd
.collapse
- 1;
15099 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
15100 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
15101 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
15102 tree clauses
= *pc
;
15105 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
15106 OMP_CLAUSE__LOOPTEMP_
);
15107 for (i
= 0; i
< count
; i
++)
15112 gcc_assert (outerc
);
15113 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
15114 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
15115 OMP_CLAUSE__LOOPTEMP_
);
15119 temp
= create_tmp_var (type
);
15120 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
15122 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
15123 OMP_CLAUSE_DECL (*pc
) = temp
;
15124 pc
= &OMP_CLAUSE_CHAIN (*pc
);
15129 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15132 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
15134 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
15136 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
15138 /* Lower the header expressions. At this point, we can assume that
15139 the header is of the form:
15141 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15143 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15144 using the .omp_data_s mapping, if needed. */
15145 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
15147 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
15148 if (!is_gimple_min_invariant (*rhs_p
))
15149 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15151 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
15152 if (!is_gimple_min_invariant (*rhs_p
))
15153 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15155 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
15156 if (!is_gimple_min_invariant (*rhs_p
))
15157 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15160 /* Once lowered, extract the bounds and clauses. */
15161 extract_omp_for_data (stmt
, &fd
, NULL
);
15163 if (is_gimple_omp_oacc (ctx
->stmt
)
15164 && !ctx_in_oacc_kernels_region (ctx
))
15165 lower_oacc_head_tail (gimple_location (stmt
),
15166 gimple_omp_for_clauses (stmt
),
15167 &oacc_head
, &oacc_tail
, ctx
);
15169 /* Add OpenACC partitioning and reduction markers just before the loop */
15171 gimple_seq_add_seq (&body
, oacc_head
);
15173 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
15175 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
15176 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15177 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
15178 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
15180 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
15181 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
15182 OMP_CLAUSE_LINEAR_STEP (c
)
15183 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
15187 if (!gimple_omp_for_grid_phony (stmt
))
15188 gimple_seq_add_stmt (&body
, stmt
);
15189 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
15191 if (!gimple_omp_for_grid_phony (stmt
))
15192 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
15195 /* After the loop, add exit clauses. */
15196 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
15198 if (ctx
->cancellable
)
15199 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
15201 gimple_seq_add_seq (&body
, dlist
);
15203 body
= maybe_catch_exception (body
);
15205 if (!gimple_omp_for_grid_phony (stmt
))
15207 /* Region exit marker goes at the end of the loop body. */
15208 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
15209 maybe_add_implicit_barrier_cancel (ctx
, &body
);
15212 /* Add OpenACC joining and reduction markers just after the loop. */
15214 gimple_seq_add_seq (&body
, oacc_tail
);
15216 pop_gimplify_context (new_stmt
);
15218 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
15219 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
15220 if (BLOCK_VARS (block
))
15221 TREE_USED (block
) = 1;
15223 gimple_bind_set_body (new_stmt
, body
);
15224 gimple_omp_set_body (stmt
, NULL
);
15225 gimple_omp_for_set_pre_body (stmt
, NULL
);
15228 /* Callback for walk_stmts. Check if the current statement only contains
15229 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15232 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
15233 bool *handled_ops_p
,
15234 struct walk_stmt_info
*wi
)
15236 int *info
= (int *) wi
->info
;
15237 gimple
*stmt
= gsi_stmt (*gsi_p
);
15239 *handled_ops_p
= true;
15240 switch (gimple_code (stmt
))
15244 case GIMPLE_OMP_FOR
:
15245 case GIMPLE_OMP_SECTIONS
:
15246 *info
= *info
== 0 ? 1 : -1;
15255 struct omp_taskcopy_context
15257 /* This field must be at the beginning, as we do "inheritance": Some
15258 callback functions for tree-inline.c (e.g., omp_copy_decl)
15259 receive a copy_body_data pointer that is up-casted to an
15260 omp_context pointer. */
15266 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
15268 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
15270 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
15271 return create_tmp_var (TREE_TYPE (var
));
15277 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
15279 tree name
, new_fields
= NULL
, type
, f
;
15281 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
15282 name
= DECL_NAME (TYPE_NAME (orig_type
));
15283 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
15284 TYPE_DECL
, name
, type
);
15285 TYPE_NAME (type
) = name
;
15287 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
15289 tree new_f
= copy_node (f
);
15290 DECL_CONTEXT (new_f
) = type
;
15291 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
15292 TREE_CHAIN (new_f
) = new_fields
;
15293 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15294 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15295 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
15297 new_fields
= new_f
;
15298 tcctx
->cb
.decl_map
->put (f
, new_f
);
15300 TYPE_FIELDS (type
) = nreverse (new_fields
);
15301 layout_type (type
);
15305 /* Create task copyfn. */
15308 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
15310 struct function
*child_cfun
;
15311 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
15312 tree record_type
, srecord_type
, bind
, list
;
15313 bool record_needs_remap
= false, srecord_needs_remap
= false;
15315 struct omp_taskcopy_context tcctx
;
15316 location_t loc
= gimple_location (task_stmt
);
15318 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
15319 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
15320 gcc_assert (child_cfun
->cfg
== NULL
);
15321 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
15323 /* Reset DECL_CONTEXT on function arguments. */
15324 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
15325 DECL_CONTEXT (t
) = child_fn
;
15327 /* Populate the function. */
15328 push_gimplify_context ();
15329 push_cfun (child_cfun
);
15331 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
15332 TREE_SIDE_EFFECTS (bind
) = 1;
15334 DECL_SAVED_TREE (child_fn
) = bind
;
15335 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
15337 /* Remap src and dst argument types if needed. */
15338 record_type
= ctx
->record_type
;
15339 srecord_type
= ctx
->srecord_type
;
15340 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
15341 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15343 record_needs_remap
= true;
15346 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
15347 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15349 srecord_needs_remap
= true;
15353 if (record_needs_remap
|| srecord_needs_remap
)
15355 memset (&tcctx
, '\0', sizeof (tcctx
));
15356 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
15357 tcctx
.cb
.dst_fn
= child_fn
;
15358 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
15359 gcc_checking_assert (tcctx
.cb
.src_node
);
15360 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
15361 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
15362 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
15363 tcctx
.cb
.eh_lp_nr
= 0;
15364 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
15365 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
15368 if (record_needs_remap
)
15369 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
15370 if (srecord_needs_remap
)
15371 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
15374 tcctx
.cb
.decl_map
= NULL
;
15376 arg
= DECL_ARGUMENTS (child_fn
);
15377 TREE_TYPE (arg
) = build_pointer_type (record_type
);
15378 sarg
= DECL_CHAIN (arg
);
15379 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
15381 /* First pass: initialize temporaries used in record_type and srecord_type
15382 sizes and field offsets. */
15383 if (tcctx
.cb
.decl_map
)
15384 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15385 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15389 decl
= OMP_CLAUSE_DECL (c
);
15390 p
= tcctx
.cb
.decl_map
->get (decl
);
15393 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15394 sf
= (tree
) n
->value
;
15395 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15396 src
= build_simple_mem_ref_loc (loc
, sarg
);
15397 src
= omp_build_component_ref (src
, sf
);
15398 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
15399 append_to_statement_list (t
, &list
);
15402 /* Second pass: copy shared var pointers and copy construct non-VLA
15403 firstprivate vars. */
15404 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15405 switch (OMP_CLAUSE_CODE (c
))
15407 splay_tree_key key
;
15408 case OMP_CLAUSE_SHARED
:
15409 decl
= OMP_CLAUSE_DECL (c
);
15410 key
= (splay_tree_key
) decl
;
15411 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
15412 key
= (splay_tree_key
) &DECL_UID (decl
);
15413 n
= splay_tree_lookup (ctx
->field_map
, key
);
15416 f
= (tree
) n
->value
;
15417 if (tcctx
.cb
.decl_map
)
15418 f
= *tcctx
.cb
.decl_map
->get (f
);
15419 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
15420 sf
= (tree
) n
->value
;
15421 if (tcctx
.cb
.decl_map
)
15422 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15423 src
= build_simple_mem_ref_loc (loc
, sarg
);
15424 src
= omp_build_component_ref (src
, sf
);
15425 dst
= build_simple_mem_ref_loc (loc
, arg
);
15426 dst
= omp_build_component_ref (dst
, f
);
15427 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15428 append_to_statement_list (t
, &list
);
15430 case OMP_CLAUSE_FIRSTPRIVATE
:
15431 decl
= OMP_CLAUSE_DECL (c
);
15432 if (is_variable_sized (decl
))
15434 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15437 f
= (tree
) n
->value
;
15438 if (tcctx
.cb
.decl_map
)
15439 f
= *tcctx
.cb
.decl_map
->get (f
);
15440 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15443 sf
= (tree
) n
->value
;
15444 if (tcctx
.cb
.decl_map
)
15445 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15446 src
= build_simple_mem_ref_loc (loc
, sarg
);
15447 src
= omp_build_component_ref (src
, sf
);
15448 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
15449 src
= build_simple_mem_ref_loc (loc
, src
);
15453 dst
= build_simple_mem_ref_loc (loc
, arg
);
15454 dst
= omp_build_component_ref (dst
, f
);
15455 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15456 append_to_statement_list (t
, &list
);
15458 case OMP_CLAUSE_PRIVATE
:
15459 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
15461 decl
= OMP_CLAUSE_DECL (c
);
15462 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15463 f
= (tree
) n
->value
;
15464 if (tcctx
.cb
.decl_map
)
15465 f
= *tcctx
.cb
.decl_map
->get (f
);
15466 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15469 sf
= (tree
) n
->value
;
15470 if (tcctx
.cb
.decl_map
)
15471 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15472 src
= build_simple_mem_ref_loc (loc
, sarg
);
15473 src
= omp_build_component_ref (src
, sf
);
15474 if (use_pointer_for_field (decl
, NULL
))
15475 src
= build_simple_mem_ref_loc (loc
, src
);
15479 dst
= build_simple_mem_ref_loc (loc
, arg
);
15480 dst
= omp_build_component_ref (dst
, f
);
15481 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15482 append_to_statement_list (t
, &list
);
15488 /* Last pass: handle VLA firstprivates. */
15489 if (tcctx
.cb
.decl_map
)
15490 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15491 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15495 decl
= OMP_CLAUSE_DECL (c
);
15496 if (!is_variable_sized (decl
))
15498 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15501 f
= (tree
) n
->value
;
15502 f
= *tcctx
.cb
.decl_map
->get (f
);
15503 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
15504 ind
= DECL_VALUE_EXPR (decl
);
15505 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
15506 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
15507 n
= splay_tree_lookup (ctx
->sfield_map
,
15508 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15509 sf
= (tree
) n
->value
;
15510 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15511 src
= build_simple_mem_ref_loc (loc
, sarg
);
15512 src
= omp_build_component_ref (src
, sf
);
15513 src
= build_simple_mem_ref_loc (loc
, src
);
15514 dst
= build_simple_mem_ref_loc (loc
, arg
);
15515 dst
= omp_build_component_ref (dst
, f
);
15516 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15517 append_to_statement_list (t
, &list
);
15518 n
= splay_tree_lookup (ctx
->field_map
,
15519 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15520 df
= (tree
) n
->value
;
15521 df
= *tcctx
.cb
.decl_map
->get (df
);
15522 ptr
= build_simple_mem_ref_loc (loc
, arg
);
15523 ptr
= omp_build_component_ref (ptr
, df
);
15524 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
15525 build_fold_addr_expr_loc (loc
, dst
));
15526 append_to_statement_list (t
, &list
);
15529 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
15530 append_to_statement_list (t
, &list
);
15532 if (tcctx
.cb
.decl_map
)
15533 delete tcctx
.cb
.decl_map
;
15534 pop_gimplify_context (NULL
);
15535 BIND_EXPR_BODY (bind
) = list
;
15540 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
15544 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
15546 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
15547 gcc_assert (clauses
);
15548 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15549 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
15550 switch (OMP_CLAUSE_DEPEND_KIND (c
))
15552 case OMP_CLAUSE_DEPEND_IN
:
15555 case OMP_CLAUSE_DEPEND_OUT
:
15556 case OMP_CLAUSE_DEPEND_INOUT
:
15559 case OMP_CLAUSE_DEPEND_SOURCE
:
15560 case OMP_CLAUSE_DEPEND_SINK
:
15563 gcc_unreachable ();
15565 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
15566 tree array
= create_tmp_var (type
);
15567 TREE_ADDRESSABLE (array
) = 1;
15568 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
15570 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
15571 gimple_seq_add_stmt (iseq
, g
);
15572 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
15574 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
15575 gimple_seq_add_stmt (iseq
, g
);
15576 for (i
= 0; i
< 2; i
++)
15578 if ((i
? n_in
: n_out
) == 0)
15580 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15581 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
15582 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
15584 tree t
= OMP_CLAUSE_DECL (c
);
15585 t
= fold_convert (ptr_type_node
, t
);
15586 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
15587 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
15588 NULL_TREE
, NULL_TREE
);
15589 g
= gimple_build_assign (r
, t
);
15590 gimple_seq_add_stmt (iseq
, g
);
15593 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
15594 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
15595 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
15597 tree clobber
= build_constructor (type
, NULL
);
15598 TREE_THIS_VOLATILE (clobber
) = 1;
15599 g
= gimple_build_assign (array
, clobber
);
15600 gimple_seq_add_stmt (oseq
, g
);
15603 /* Lower the OpenMP parallel or task directive in the current statement
15604 in GSI_P. CTX holds context information for the directive. */
15607 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15611 gimple
*stmt
= gsi_stmt (*gsi_p
);
15612 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
15613 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
15614 location_t loc
= gimple_location (stmt
);
15616 clauses
= gimple_omp_taskreg_clauses (stmt
);
15618 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
15619 par_body
= gimple_bind_body (par_bind
);
15620 child_fn
= ctx
->cb
.dst_fn
;
15621 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15622 && !gimple_omp_parallel_combined_p (stmt
))
15624 struct walk_stmt_info wi
;
15627 memset (&wi
, 0, sizeof (wi
));
15629 wi
.val_only
= true;
15630 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
15632 gimple_omp_parallel_set_combined_p (stmt
, true);
15634 gimple_seq dep_ilist
= NULL
;
15635 gimple_seq dep_olist
= NULL
;
15636 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
15637 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15639 push_gimplify_context ();
15640 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15641 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
15642 &dep_ilist
, &dep_olist
);
15645 if (ctx
->srecord_type
)
15646 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
15648 push_gimplify_context ();
15653 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15654 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
15655 if (phony_construct
&& ctx
->record_type
)
15657 gcc_checking_assert (!ctx
->receiver_decl
);
15658 ctx
->receiver_decl
= create_tmp_var
15659 (build_reference_type (ctx
->record_type
), ".omp_rec");
15661 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
15662 lower_omp (&par_body
, ctx
);
15663 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
15664 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
15666 /* Declare all the variables created by mapping and the variables
15667 declared in the scope of the parallel body. */
15668 record_vars_into (ctx
->block_vars
, child_fn
);
15669 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
15671 if (ctx
->record_type
)
15674 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
15675 : ctx
->record_type
, ".omp_data_o");
15676 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15677 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15678 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
15683 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
15684 lower_send_shared_vars (&ilist
, &olist
, ctx
);
15686 if (ctx
->record_type
)
15688 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
15689 TREE_THIS_VOLATILE (clobber
) = 1;
15690 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15694 /* Once all the expansions are done, sequence all the different
15695 fragments inside gimple_omp_body. */
15699 if (ctx
->record_type
)
15701 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15702 /* fixup_child_record_type might have changed receiver_decl's type. */
15703 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15704 gimple_seq_add_stmt (&new_body
,
15705 gimple_build_assign (ctx
->receiver_decl
, t
));
15708 gimple_seq_add_seq (&new_body
, par_ilist
);
15709 gimple_seq_add_seq (&new_body
, par_body
);
15710 gimple_seq_add_seq (&new_body
, par_rlist
);
15711 if (ctx
->cancellable
)
15712 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
15713 gimple_seq_add_seq (&new_body
, par_olist
);
15714 new_body
= maybe_catch_exception (new_body
);
15715 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
15716 gimple_seq_add_stmt (&new_body
,
15717 gimple_build_omp_continue (integer_zero_node
,
15718 integer_zero_node
));
15719 if (!phony_construct
)
15721 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15722 gimple_omp_set_body (stmt
, new_body
);
15725 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
15726 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15727 gimple_bind_add_seq (bind
, ilist
);
15728 if (!phony_construct
)
15729 gimple_bind_add_stmt (bind
, stmt
);
15731 gimple_bind_add_seq (bind
, new_body
);
15732 gimple_bind_add_seq (bind
, olist
);
15734 pop_gimplify_context (NULL
);
15738 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15739 gimple_bind_add_stmt (dep_bind
, bind
);
15740 gimple_bind_add_seq (dep_bind
, dep_olist
);
15741 pop_gimplify_context (dep_bind
);
15745 /* Lower the GIMPLE_OMP_TARGET in the current statement
15746 in GSI_P. CTX holds context information for the directive. */
15749 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15752 tree child_fn
, t
, c
;
15753 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
15754 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
15755 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
15756 location_t loc
= gimple_location (stmt
);
15757 bool offloaded
, data_region
;
15758 unsigned int map_cnt
= 0;
15760 offloaded
= is_gimple_omp_offloaded (stmt
);
15761 switch (gimple_omp_target_kind (stmt
))
15763 case GF_OMP_TARGET_KIND_REGION
:
15764 case GF_OMP_TARGET_KIND_UPDATE
:
15765 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15766 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15767 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15768 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15769 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15770 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15771 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
15772 data_region
= false;
15774 case GF_OMP_TARGET_KIND_DATA
:
15775 case GF_OMP_TARGET_KIND_OACC_DATA
:
15776 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
15777 data_region
= true;
15780 gcc_unreachable ();
15783 clauses
= gimple_omp_target_clauses (stmt
);
15785 gimple_seq dep_ilist
= NULL
;
15786 gimple_seq dep_olist
= NULL
;
15787 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15789 push_gimplify_context ();
15790 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15791 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15792 &dep_ilist
, &dep_olist
);
15799 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15800 tgt_body
= gimple_bind_body (tgt_bind
);
15802 else if (data_region
)
15803 tgt_body
= gimple_omp_body (stmt
);
15804 child_fn
= ctx
->cb
.dst_fn
;
15806 push_gimplify_context ();
15809 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15810 switch (OMP_CLAUSE_CODE (c
))
15816 case OMP_CLAUSE_MAP
:
15818 /* First check what we're prepared to handle in the following. */
15819 switch (OMP_CLAUSE_MAP_KIND (c
))
15821 case GOMP_MAP_ALLOC
:
15823 case GOMP_MAP_FROM
:
15824 case GOMP_MAP_TOFROM
:
15825 case GOMP_MAP_POINTER
:
15826 case GOMP_MAP_TO_PSET
:
15827 case GOMP_MAP_DELETE
:
15828 case GOMP_MAP_RELEASE
:
15829 case GOMP_MAP_ALWAYS_TO
:
15830 case GOMP_MAP_ALWAYS_FROM
:
15831 case GOMP_MAP_ALWAYS_TOFROM
:
15832 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15833 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
15834 case GOMP_MAP_STRUCT
:
15835 case GOMP_MAP_ALWAYS_POINTER
:
15837 case GOMP_MAP_FORCE_ALLOC
:
15838 case GOMP_MAP_FORCE_TO
:
15839 case GOMP_MAP_FORCE_FROM
:
15840 case GOMP_MAP_FORCE_TOFROM
:
15841 case GOMP_MAP_FORCE_PRESENT
:
15842 case GOMP_MAP_FORCE_DEVICEPTR
:
15843 case GOMP_MAP_DEVICE_RESIDENT
:
15844 case GOMP_MAP_LINK
:
15845 gcc_assert (is_gimple_omp_oacc (stmt
));
15848 gcc_unreachable ();
15852 case OMP_CLAUSE_TO
:
15853 case OMP_CLAUSE_FROM
:
15855 var
= OMP_CLAUSE_DECL (c
);
15858 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15859 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15860 && (OMP_CLAUSE_MAP_KIND (c
)
15861 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15866 if (DECL_SIZE (var
)
15867 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15869 tree var2
= DECL_VALUE_EXPR (var
);
15870 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15871 var2
= TREE_OPERAND (var2
, 0);
15872 gcc_assert (DECL_P (var2
));
15877 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15878 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15879 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15881 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15883 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15884 && varpool_node::get_create (var
)->offloadable
)
15887 tree type
= build_pointer_type (TREE_TYPE (var
));
15888 tree new_var
= lookup_decl (var
, ctx
);
15889 x
= create_tmp_var_raw (type
, get_name (new_var
));
15890 gimple_add_tmp_var (x
);
15891 x
= build_simple_mem_ref (x
);
15892 SET_DECL_VALUE_EXPR (new_var
, x
);
15893 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15898 if (!maybe_lookup_field (var
, ctx
))
15901 /* Don't remap oacc parallel reduction variables, because the
15902 intermediate result must be local to each gang. */
15903 if (offloaded
&& !(OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15904 && OMP_CLAUSE_MAP_IN_REDUCTION (c
)))
15906 x
= build_receiver_ref (var
, true, ctx
);
15907 tree new_var
= lookup_decl (var
, ctx
);
15909 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15910 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15911 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15912 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15913 x
= build_simple_mem_ref (x
);
15914 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15916 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15917 if (is_reference (new_var
))
15919 /* Create a local object to hold the instance
15921 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15922 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15923 tree inst
= create_tmp_var (type
, id
);
15924 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15925 x
= build_fold_addr_expr (inst
);
15927 gimplify_assign (new_var
, x
, &fplist
);
15929 else if (DECL_P (new_var
))
15931 SET_DECL_VALUE_EXPR (new_var
, x
);
15932 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15935 gcc_unreachable ();
15940 case OMP_CLAUSE_FIRSTPRIVATE
:
15941 if (is_oacc_parallel (ctx
))
15942 goto oacc_firstprivate
;
15944 var
= OMP_CLAUSE_DECL (c
);
15945 if (!is_reference (var
)
15946 && !is_gimple_reg_type (TREE_TYPE (var
)))
15948 tree new_var
= lookup_decl (var
, ctx
);
15949 if (is_variable_sized (var
))
15951 tree pvar
= DECL_VALUE_EXPR (var
);
15952 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15953 pvar
= TREE_OPERAND (pvar
, 0);
15954 gcc_assert (DECL_P (pvar
));
15955 tree new_pvar
= lookup_decl (pvar
, ctx
);
15956 x
= build_fold_indirect_ref (new_pvar
);
15957 TREE_THIS_NOTRAP (x
) = 1;
15960 x
= build_receiver_ref (var
, true, ctx
);
15961 SET_DECL_VALUE_EXPR (new_var
, x
);
15962 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15966 case OMP_CLAUSE_PRIVATE
:
15967 if (is_gimple_omp_oacc (ctx
->stmt
))
15969 var
= OMP_CLAUSE_DECL (c
);
15970 if (is_variable_sized (var
))
15972 tree new_var
= lookup_decl (var
, ctx
);
15973 tree pvar
= DECL_VALUE_EXPR (var
);
15974 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15975 pvar
= TREE_OPERAND (pvar
, 0);
15976 gcc_assert (DECL_P (pvar
));
15977 tree new_pvar
= lookup_decl (pvar
, ctx
);
15978 x
= build_fold_indirect_ref (new_pvar
);
15979 TREE_THIS_NOTRAP (x
) = 1;
15980 SET_DECL_VALUE_EXPR (new_var
, x
);
15981 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15985 case OMP_CLAUSE_USE_DEVICE_PTR
:
15986 case OMP_CLAUSE_IS_DEVICE_PTR
:
15987 var
= OMP_CLAUSE_DECL (c
);
15989 if (is_variable_sized (var
))
15991 tree new_var
= lookup_decl (var
, ctx
);
15992 tree pvar
= DECL_VALUE_EXPR (var
);
15993 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15994 pvar
= TREE_OPERAND (pvar
, 0);
15995 gcc_assert (DECL_P (pvar
));
15996 tree new_pvar
= lookup_decl (pvar
, ctx
);
15997 x
= build_fold_indirect_ref (new_pvar
);
15998 TREE_THIS_NOTRAP (x
) = 1;
15999 SET_DECL_VALUE_EXPR (new_var
, x
);
16000 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
16002 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16004 tree new_var
= lookup_decl (var
, ctx
);
16005 tree type
= build_pointer_type (TREE_TYPE (var
));
16006 x
= create_tmp_var_raw (type
, get_name (new_var
));
16007 gimple_add_tmp_var (x
);
16008 x
= build_simple_mem_ref (x
);
16009 SET_DECL_VALUE_EXPR (new_var
, x
);
16010 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
16014 tree new_var
= lookup_decl (var
, ctx
);
16015 x
= create_tmp_var_raw (TREE_TYPE (new_var
), get_name (new_var
));
16016 gimple_add_tmp_var (x
);
16017 SET_DECL_VALUE_EXPR (new_var
, x
);
16018 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
16025 target_nesting_level
++;
16026 lower_omp (&tgt_body
, ctx
);
16027 target_nesting_level
--;
16029 else if (data_region
)
16030 lower_omp (&tgt_body
, ctx
);
16034 /* Declare all the variables created by mapping and the variables
16035 declared in the scope of the target body. */
16036 record_vars_into (ctx
->block_vars
, child_fn
);
16037 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
16042 if (ctx
->record_type
)
16045 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
16046 DECL_NAMELESS (ctx
->sender_decl
) = 1;
16047 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
16048 t
= make_tree_vec (3);
16049 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
16050 TREE_VEC_ELT (t
, 1)
16051 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
16052 ".omp_data_sizes");
16053 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
16054 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
16055 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
16056 tree tkind_type
= short_unsigned_type_node
;
16057 int talign_shift
= 8;
16058 TREE_VEC_ELT (t
, 2)
16059 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
16060 ".omp_data_kinds");
16061 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
16062 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
16063 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
16064 gimple_omp_target_set_data_arg (stmt
, t
);
16066 vec
<constructor_elt
, va_gc
> *vsize
;
16067 vec
<constructor_elt
, va_gc
> *vkind
;
16068 vec_alloc (vsize
, map_cnt
);
16069 vec_alloc (vkind
, map_cnt
);
16070 unsigned int map_idx
= 0;
16072 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16073 switch (OMP_CLAUSE_CODE (c
))
16075 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
16076 unsigned int talign
;
16081 case OMP_CLAUSE_MAP
:
16082 case OMP_CLAUSE_TO
:
16083 case OMP_CLAUSE_FROM
:
16084 oacc_firstprivate_map
:
16086 ovar
= OMP_CLAUSE_DECL (c
);
16087 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16088 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16089 || (OMP_CLAUSE_MAP_KIND (c
)
16090 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16092 if (!DECL_P (ovar
))
16094 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16095 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
16097 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
16098 == get_base_address (ovar
));
16099 nc
= OMP_CLAUSE_CHAIN (c
);
16100 ovar
= OMP_CLAUSE_DECL (nc
);
16104 tree x
= build_sender_ref (ovar
, ctx
);
16106 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
16107 gimplify_assign (x
, v
, &ilist
);
16113 if (DECL_SIZE (ovar
)
16114 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
16116 tree ovar2
= DECL_VALUE_EXPR (ovar
);
16117 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
16118 ovar2
= TREE_OPERAND (ovar2
, 0);
16119 gcc_assert (DECL_P (ovar2
));
16122 if (!maybe_lookup_field (ovar
, ctx
))
16126 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
16127 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
16128 talign
= DECL_ALIGN_UNIT (ovar
);
16131 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16132 x
= build_sender_ref (ovar
, ctx
);
16134 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16135 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
16136 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
16137 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
16139 gcc_assert (offloaded
);
16141 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
16142 mark_addressable (avar
);
16143 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
16144 talign
= DECL_ALIGN_UNIT (avar
);
16145 avar
= build_fold_addr_expr (avar
);
16146 gimplify_assign (x
, avar
, &ilist
);
16148 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16150 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
16151 if (!is_reference (var
))
16153 if (is_gimple_reg (var
)
16154 && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16155 TREE_NO_WARNING (var
) = 1;
16156 var
= build_fold_addr_expr (var
);
16159 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16160 gimplify_assign (x
, var
, &ilist
);
16162 else if (is_gimple_reg (var
))
16164 gcc_assert (offloaded
);
16165 tree avar
= create_tmp_var (TREE_TYPE (var
));
16166 mark_addressable (avar
);
16167 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
16168 if (GOMP_MAP_COPY_TO_P (map_kind
)
16169 || map_kind
== GOMP_MAP_POINTER
16170 || map_kind
== GOMP_MAP_TO_PSET
16171 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16173 /* If we need to initialize a temporary
16174 with VAR because it is not addressable, and
16175 the variable hasn't been initialized yet, then
16176 we'll get a warning for the store to avar.
16177 Don't warn in that case, the mapping might
16179 TREE_NO_WARNING (var
) = 1;
16180 gimplify_assign (avar
, var
, &ilist
);
16182 avar
= build_fold_addr_expr (avar
);
16183 gimplify_assign (x
, avar
, &ilist
);
16184 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
16185 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16186 && !TYPE_READONLY (TREE_TYPE (var
)))
16188 x
= unshare_expr (x
);
16189 x
= build_simple_mem_ref (x
);
16190 gimplify_assign (var
, x
, &olist
);
16195 var
= build_fold_addr_expr (var
);
16196 gimplify_assign (x
, var
, &ilist
);
16200 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16202 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16203 s
= TREE_TYPE (ovar
);
16204 if (TREE_CODE (s
) == REFERENCE_TYPE
)
16206 s
= TYPE_SIZE_UNIT (s
);
16209 s
= OMP_CLAUSE_SIZE (c
);
16210 if (s
== NULL_TREE
)
16211 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16212 s
= fold_convert (size_type_node
, s
);
16213 purpose
= size_int (map_idx
++);
16214 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16215 if (TREE_CODE (s
) != INTEGER_CST
)
16216 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16218 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
16219 switch (OMP_CLAUSE_CODE (c
))
16221 case OMP_CLAUSE_MAP
:
16222 tkind
= OMP_CLAUSE_MAP_KIND (c
);
16223 tkind_zero
= tkind
;
16224 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
16227 case GOMP_MAP_ALLOC
:
16229 case GOMP_MAP_FROM
:
16230 case GOMP_MAP_TOFROM
:
16231 case GOMP_MAP_ALWAYS_TO
:
16232 case GOMP_MAP_ALWAYS_FROM
:
16233 case GOMP_MAP_ALWAYS_TOFROM
:
16234 case GOMP_MAP_RELEASE
:
16235 case GOMP_MAP_FORCE_TO
:
16236 case GOMP_MAP_FORCE_FROM
:
16237 case GOMP_MAP_FORCE_TOFROM
:
16238 case GOMP_MAP_FORCE_PRESENT
:
16239 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
16241 case GOMP_MAP_DELETE
:
16242 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
16246 if (tkind_zero
!= tkind
)
16248 if (integer_zerop (s
))
16249 tkind
= tkind_zero
;
16250 else if (integer_nonzerop (s
))
16251 tkind_zero
= tkind
;
16254 case OMP_CLAUSE_FIRSTPRIVATE
:
16255 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16256 tkind
= GOMP_MAP_TO
;
16257 tkind_zero
= tkind
;
16259 case OMP_CLAUSE_TO
:
16260 tkind
= GOMP_MAP_TO
;
16261 tkind_zero
= tkind
;
16263 case OMP_CLAUSE_FROM
:
16264 tkind
= GOMP_MAP_FROM
;
16265 tkind_zero
= tkind
;
16268 gcc_unreachable ();
16270 gcc_checking_assert (tkind
16271 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16272 gcc_checking_assert (tkind_zero
16273 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16274 talign
= ceil_log2 (talign
);
16275 tkind
|= talign
<< talign_shift
;
16276 tkind_zero
|= talign
<< talign_shift
;
16277 gcc_checking_assert (tkind
16278 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16279 gcc_checking_assert (tkind_zero
16280 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16281 if (tkind
== tkind_zero
)
16282 x
= build_int_cstu (tkind_type
, tkind
);
16285 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
16286 x
= build3 (COND_EXPR
, tkind_type
,
16287 fold_build2 (EQ_EXPR
, boolean_type_node
,
16288 unshare_expr (s
), size_zero_node
),
16289 build_int_cstu (tkind_type
, tkind_zero
),
16290 build_int_cstu (tkind_type
, tkind
));
16292 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
16297 case OMP_CLAUSE_FIRSTPRIVATE
:
16298 if (is_oacc_parallel (ctx
))
16299 goto oacc_firstprivate_map
;
16300 ovar
= OMP_CLAUSE_DECL (c
);
16301 if (is_reference (ovar
))
16302 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16304 talign
= DECL_ALIGN_UNIT (ovar
);
16305 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16306 x
= build_sender_ref (ovar
, ctx
);
16307 tkind
= GOMP_MAP_FIRSTPRIVATE
;
16308 type
= TREE_TYPE (ovar
);
16309 if (is_reference (ovar
))
16310 type
= TREE_TYPE (type
);
16311 if ((INTEGRAL_TYPE_P (type
)
16312 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16313 || TREE_CODE (type
) == POINTER_TYPE
)
16315 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16317 if (is_reference (var
))
16318 t
= build_simple_mem_ref (var
);
16319 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16320 TREE_NO_WARNING (var
) = 1;
16321 if (TREE_CODE (type
) != POINTER_TYPE
)
16322 t
= fold_convert (pointer_sized_int_node
, t
);
16323 t
= fold_convert (TREE_TYPE (x
), t
);
16324 gimplify_assign (x
, t
, &ilist
);
16326 else if (is_reference (var
))
16327 gimplify_assign (x
, var
, &ilist
);
16328 else if (is_gimple_reg (var
))
16330 tree avar
= create_tmp_var (TREE_TYPE (var
));
16331 mark_addressable (avar
);
16332 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16333 TREE_NO_WARNING (var
) = 1;
16334 gimplify_assign (avar
, var
, &ilist
);
16335 avar
= build_fold_addr_expr (avar
);
16336 gimplify_assign (x
, avar
, &ilist
);
16340 var
= build_fold_addr_expr (var
);
16341 gimplify_assign (x
, var
, &ilist
);
16343 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
16345 else if (is_reference (var
))
16346 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16348 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16349 s
= fold_convert (size_type_node
, s
);
16350 purpose
= size_int (map_idx
++);
16351 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16352 if (TREE_CODE (s
) != INTEGER_CST
)
16353 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16355 gcc_checking_assert (tkind
16356 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16357 talign
= ceil_log2 (talign
);
16358 tkind
|= talign
<< talign_shift
;
16359 gcc_checking_assert (tkind
16360 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16361 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16362 build_int_cstu (tkind_type
, tkind
));
16365 case OMP_CLAUSE_USE_DEVICE_PTR
:
16366 case OMP_CLAUSE_IS_DEVICE_PTR
:
16367 ovar
= OMP_CLAUSE_DECL (c
);
16368 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16369 x
= build_sender_ref (ovar
, ctx
);
16370 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16371 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
16373 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16374 type
= TREE_TYPE (ovar
);
16375 if (TREE_CODE (type
) == ARRAY_TYPE
)
16376 var
= build_fold_addr_expr (var
);
16379 if (is_reference (ovar
))
16381 type
= TREE_TYPE (type
);
16382 if (TREE_CODE (type
) != ARRAY_TYPE
)
16383 var
= build_simple_mem_ref (var
);
16384 var
= fold_convert (TREE_TYPE (x
), var
);
16387 gimplify_assign (x
, var
, &ilist
);
16389 purpose
= size_int (map_idx
++);
16390 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16391 gcc_checking_assert (tkind
16392 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16393 gcc_checking_assert (tkind
16394 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16395 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16396 build_int_cstu (tkind_type
, tkind
));
16400 gcc_assert (map_idx
== map_cnt
);
16402 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
16403 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
16404 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
16405 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
16406 for (int i
= 1; i
<= 2; i
++)
16407 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
16409 gimple_seq initlist
= NULL
;
16410 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
16411 TREE_VEC_ELT (t
, i
)),
16412 &initlist
, true, NULL_TREE
);
16413 gimple_seq_add_seq (&ilist
, initlist
);
16415 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
16417 TREE_THIS_VOLATILE (clobber
) = 1;
16418 gimple_seq_add_stmt (&olist
,
16419 gimple_build_assign (TREE_VEC_ELT (t
, i
),
16423 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
16424 TREE_THIS_VOLATILE (clobber
) = 1;
16425 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
16429 /* Once all the expansions are done, sequence all the different
16430 fragments inside gimple_omp_body. */
16435 && ctx
->record_type
)
16437 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
16438 /* fixup_child_record_type might have changed receiver_decl's type. */
16439 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
16440 gimple_seq_add_stmt (&new_body
,
16441 gimple_build_assign (ctx
->receiver_decl
, t
));
16443 gimple_seq_add_seq (&new_body
, fplist
);
16445 if (offloaded
|| data_region
)
16447 tree prev
= NULL_TREE
;
16448 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16449 switch (OMP_CLAUSE_CODE (c
))
16454 case OMP_CLAUSE_FIRSTPRIVATE
:
16455 if (is_gimple_omp_oacc (ctx
->stmt
))
16457 var
= OMP_CLAUSE_DECL (c
);
16458 if (is_reference (var
)
16459 || is_gimple_reg_type (TREE_TYPE (var
)))
16461 tree new_var
= lookup_decl (var
, ctx
);
16463 type
= TREE_TYPE (var
);
16464 if (is_reference (var
))
16465 type
= TREE_TYPE (type
);
16466 if ((INTEGRAL_TYPE_P (type
)
16467 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16468 || TREE_CODE (type
) == POINTER_TYPE
)
16470 x
= build_receiver_ref (var
, false, ctx
);
16471 if (TREE_CODE (type
) != POINTER_TYPE
)
16472 x
= fold_convert (pointer_sized_int_node
, x
);
16473 x
= fold_convert (type
, x
);
16474 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16476 if (is_reference (var
))
16478 tree v
= create_tmp_var_raw (type
, get_name (var
));
16479 gimple_add_tmp_var (v
);
16480 TREE_ADDRESSABLE (v
) = 1;
16481 gimple_seq_add_stmt (&new_body
,
16482 gimple_build_assign (v
, x
));
16483 x
= build_fold_addr_expr (v
);
16485 gimple_seq_add_stmt (&new_body
,
16486 gimple_build_assign (new_var
, x
));
16490 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
16491 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16493 gimple_seq_add_stmt (&new_body
,
16494 gimple_build_assign (new_var
, x
));
16497 else if (is_variable_sized (var
))
16499 tree pvar
= DECL_VALUE_EXPR (var
);
16500 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16501 pvar
= TREE_OPERAND (pvar
, 0);
16502 gcc_assert (DECL_P (pvar
));
16503 tree new_var
= lookup_decl (pvar
, ctx
);
16504 x
= build_receiver_ref (var
, false, ctx
);
16505 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16506 gimple_seq_add_stmt (&new_body
,
16507 gimple_build_assign (new_var
, x
));
16510 case OMP_CLAUSE_PRIVATE
:
16511 if (is_gimple_omp_oacc (ctx
->stmt
))
16513 var
= OMP_CLAUSE_DECL (c
);
16514 if (is_reference (var
))
16516 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16517 tree new_var
= lookup_decl (var
, ctx
);
16518 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16519 if (TREE_CONSTANT (x
))
16521 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
16523 gimple_add_tmp_var (x
);
16524 TREE_ADDRESSABLE (x
) = 1;
16525 x
= build_fold_addr_expr_loc (clause_loc
, x
);
16530 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16531 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16532 gimple_seq_add_stmt (&new_body
,
16533 gimple_build_assign (new_var
, x
));
16536 case OMP_CLAUSE_USE_DEVICE_PTR
:
16537 case OMP_CLAUSE_IS_DEVICE_PTR
:
16538 var
= OMP_CLAUSE_DECL (c
);
16539 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16540 x
= build_sender_ref (var
, ctx
);
16542 x
= build_receiver_ref (var
, false, ctx
);
16543 if (is_variable_sized (var
))
16545 tree pvar
= DECL_VALUE_EXPR (var
);
16546 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16547 pvar
= TREE_OPERAND (pvar
, 0);
16548 gcc_assert (DECL_P (pvar
));
16549 tree new_var
= lookup_decl (pvar
, ctx
);
16550 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16551 gimple_seq_add_stmt (&new_body
,
16552 gimple_build_assign (new_var
, x
));
16554 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16556 tree new_var
= lookup_decl (var
, ctx
);
16557 new_var
= DECL_VALUE_EXPR (new_var
);
16558 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
16559 new_var
= TREE_OPERAND (new_var
, 0);
16560 gcc_assert (DECL_P (new_var
));
16561 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16562 gimple_seq_add_stmt (&new_body
,
16563 gimple_build_assign (new_var
, x
));
16567 tree type
= TREE_TYPE (var
);
16568 tree new_var
= lookup_decl (var
, ctx
);
16569 if (is_reference (var
))
16571 type
= TREE_TYPE (type
);
16572 if (TREE_CODE (type
) != ARRAY_TYPE
)
16574 tree v
= create_tmp_var_raw (type
, get_name (var
));
16575 gimple_add_tmp_var (v
);
16576 TREE_ADDRESSABLE (v
) = 1;
16577 x
= fold_convert (type
, x
);
16578 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16580 gimple_seq_add_stmt (&new_body
,
16581 gimple_build_assign (v
, x
));
16582 x
= build_fold_addr_expr (v
);
16585 new_var
= DECL_VALUE_EXPR (new_var
);
16586 x
= fold_convert (TREE_TYPE (new_var
), x
);
16587 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16588 gimple_seq_add_stmt (&new_body
,
16589 gimple_build_assign (new_var
, x
));
16593 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16594 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16595 are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
16596 or references to VLAs. */
16597 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16598 switch (OMP_CLAUSE_CODE (c
))
16603 case OMP_CLAUSE_MAP
:
16604 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16605 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16607 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16608 HOST_WIDE_INT offset
= 0;
16610 var
= OMP_CLAUSE_DECL (c
);
16612 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
16613 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
16615 && varpool_node::get_create (var
)->offloadable
)
16617 if (TREE_CODE (var
) == INDIRECT_REF
16618 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
16619 var
= TREE_OPERAND (var
, 0);
16620 if (TREE_CODE (var
) == COMPONENT_REF
)
16622 var
= get_addr_base_and_unit_offset (var
, &offset
);
16623 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
16625 else if (DECL_SIZE (var
)
16626 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
16628 tree var2
= DECL_VALUE_EXPR (var
);
16629 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
16630 var2
= TREE_OPERAND (var2
, 0);
16631 gcc_assert (DECL_P (var2
));
16634 tree new_var
= lookup_decl (var
, ctx
), x
;
16635 tree type
= TREE_TYPE (new_var
);
16637 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
16638 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
16641 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
16643 new_var
= build2 (MEM_REF
, type
,
16644 build_fold_addr_expr (new_var
),
16645 build_int_cst (build_pointer_type (type
),
16648 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
16650 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
16651 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
16652 new_var
= build2 (MEM_REF
, type
,
16653 build_fold_addr_expr (new_var
),
16654 build_int_cst (build_pointer_type (type
),
16658 is_ref
= is_reference (var
);
16659 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16661 bool ref_to_array
= false;
16664 type
= TREE_TYPE (type
);
16665 if (TREE_CODE (type
) == ARRAY_TYPE
)
16667 type
= build_pointer_type (type
);
16668 ref_to_array
= true;
16671 else if (TREE_CODE (type
) == ARRAY_TYPE
)
16673 tree decl2
= DECL_VALUE_EXPR (new_var
);
16674 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
16675 decl2
= TREE_OPERAND (decl2
, 0);
16676 gcc_assert (DECL_P (decl2
));
16678 type
= TREE_TYPE (new_var
);
16680 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
16681 x
= fold_convert_loc (clause_loc
, type
, x
);
16682 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
16684 tree bias
= OMP_CLAUSE_SIZE (c
);
16686 bias
= lookup_decl (bias
, ctx
);
16687 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
16688 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
16690 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
16691 TREE_TYPE (x
), x
, bias
);
16694 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16695 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16696 if (is_ref
&& !ref_to_array
)
16698 tree t
= create_tmp_var_raw (type
, get_name (var
));
16699 gimple_add_tmp_var (t
);
16700 TREE_ADDRESSABLE (t
) = 1;
16701 gimple_seq_add_stmt (&new_body
,
16702 gimple_build_assign (t
, x
));
16703 x
= build_fold_addr_expr_loc (clause_loc
, t
);
16705 gimple_seq_add_stmt (&new_body
,
16706 gimple_build_assign (new_var
, x
));
16709 else if (OMP_CLAUSE_CHAIN (c
)
16710 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
16712 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16713 == GOMP_MAP_FIRSTPRIVATE_POINTER
16714 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16715 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16718 case OMP_CLAUSE_PRIVATE
:
16719 var
= OMP_CLAUSE_DECL (c
);
16720 if (is_variable_sized (var
))
16722 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16723 tree new_var
= lookup_decl (var
, ctx
);
16724 tree pvar
= DECL_VALUE_EXPR (var
);
16725 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16726 pvar
= TREE_OPERAND (pvar
, 0);
16727 gcc_assert (DECL_P (pvar
));
16728 tree new_pvar
= lookup_decl (pvar
, ctx
);
16729 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16730 tree al
= size_int (DECL_ALIGN (var
));
16731 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
16732 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16733 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
16734 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16735 gimple_seq_add_stmt (&new_body
,
16736 gimple_build_assign (new_pvar
, x
));
16738 else if (is_reference (var
) && !is_gimple_omp_oacc (ctx
->stmt
))
16740 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16741 tree new_var
= lookup_decl (var
, ctx
);
16742 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16743 if (TREE_CONSTANT (x
))
16748 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16749 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
16750 tree al
= size_int (TYPE_ALIGN (rtype
));
16751 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16754 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16755 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16756 gimple_seq_add_stmt (&new_body
,
16757 gimple_build_assign (new_var
, x
));
16762 gimple_seq fork_seq
= NULL
;
16763 gimple_seq join_seq
= NULL
;
16765 if (is_oacc_parallel (ctx
))
16767 /* If there are reductions on the offloaded region itself, treat
16768 them as a dummy GANG loop. */
16769 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
16771 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
16772 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
16775 gimple_seq_add_seq (&new_body
, fork_seq
);
16776 gimple_seq_add_seq (&new_body
, tgt_body
);
16777 gimple_seq_add_seq (&new_body
, join_seq
);
16780 new_body
= maybe_catch_exception (new_body
);
16782 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
16783 gimple_omp_set_body (stmt
, new_body
);
16786 bind
= gimple_build_bind (NULL
, NULL
,
16787 tgt_bind
? gimple_bind_block (tgt_bind
)
16789 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
16790 gimple_bind_add_seq (bind
, ilist
);
16791 gimple_bind_add_stmt (bind
, stmt
);
16792 gimple_bind_add_seq (bind
, olist
);
16794 pop_gimplify_context (NULL
);
16798 gimple_bind_add_seq (dep_bind
, dep_ilist
);
16799 gimple_bind_add_stmt (dep_bind
, bind
);
16800 gimple_bind_add_seq (dep_bind
, dep_olist
);
16801 pop_gimplify_context (dep_bind
);
16805 /* Expand code for an OpenMP teams directive. */
16808 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16810 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
16811 push_gimplify_context ();
16813 tree block
= make_node (BLOCK
);
16814 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
16815 gsi_replace (gsi_p
, bind
, true);
16816 gimple_seq bind_body
= NULL
;
16817 gimple_seq dlist
= NULL
;
16818 gimple_seq olist
= NULL
;
16820 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16821 OMP_CLAUSE_NUM_TEAMS
);
16822 if (num_teams
== NULL_TREE
)
16823 num_teams
= build_int_cst (unsigned_type_node
, 0);
16826 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16827 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16828 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16830 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16831 OMP_CLAUSE_THREAD_LIMIT
);
16832 if (thread_limit
== NULL_TREE
)
16833 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16836 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16837 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16838 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16842 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16843 &bind_body
, &dlist
, ctx
, NULL
);
16844 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16845 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16846 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16848 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16849 location_t loc
= gimple_location (teams_stmt
);
16850 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16851 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16852 gimple_set_location (call
, loc
);
16853 gimple_seq_add_stmt (&bind_body
, call
);
16856 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16857 gimple_omp_set_body (teams_stmt
, NULL
);
16858 gimple_seq_add_seq (&bind_body
, olist
);
16859 gimple_seq_add_seq (&bind_body
, dlist
);
16860 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16861 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16862 gimple_bind_set_body (bind
, bind_body
);
16864 pop_gimplify_context (bind
);
16866 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16867 BLOCK_VARS (block
) = ctx
->block_vars
;
16868 if (BLOCK_VARS (block
))
16869 TREE_USED (block
) = 1;
16872 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16875 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16877 gimple
*stmt
= gsi_stmt (*gsi_p
);
16878 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
16879 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
16880 gimple_build_omp_return (false));
16884 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16885 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16886 of OMP context, but with task_shared_vars set. */
16889 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16894 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16895 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16898 if (task_shared_vars
16900 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16903 /* If a global variable has been privatized, TREE_CONSTANT on
16904 ADDR_EXPR might be wrong. */
16905 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16906 recompute_tree_invariant_for_addr_expr (t
);
16908 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16912 /* Data to be communicated between lower_omp_regimplify_operands and
16913 lower_omp_regimplify_operands_p. */
16915 struct lower_omp_regimplify_operands_data
16921 /* Helper function for lower_omp_regimplify_operands. Find
16922 omp_member_access_dummy_var vars and adjust temporarily their
16923 DECL_VALUE_EXPRs if needed. */
16926 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16929 tree t
= omp_member_access_dummy_var (*tp
);
16932 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16933 lower_omp_regimplify_operands_data
*ldata
16934 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16935 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16938 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16939 ldata
->decls
->safe_push (*tp
);
16940 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16941 SET_DECL_VALUE_EXPR (*tp
, v
);
16944 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16948 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16949 of omp_member_access_dummy_var vars during regimplification. */
16952 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16953 gimple_stmt_iterator
*gsi_p
)
16955 auto_vec
<tree
, 10> decls
;
16958 struct walk_stmt_info wi
;
16959 memset (&wi
, '\0', sizeof (wi
));
16960 struct lower_omp_regimplify_operands_data data
;
16962 data
.decls
= &decls
;
16964 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16966 gimple_regimplify_operands (stmt
, gsi_p
);
16967 while (!decls
.is_empty ())
16969 tree t
= decls
.pop ();
16970 tree v
= decls
.pop ();
16971 SET_DECL_VALUE_EXPR (t
, v
);
16976 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16978 gimple
*stmt
= gsi_stmt (*gsi_p
);
16979 struct walk_stmt_info wi
;
16982 if (gimple_has_location (stmt
))
16983 input_location
= gimple_location (stmt
);
16985 if (task_shared_vars
)
16986 memset (&wi
, '\0', sizeof (wi
));
16988 /* If we have issued syntax errors, avoid doing any heavy lifting.
16989 Just replace the OMP directives with a NOP to avoid
16990 confusing RTL expansion. */
16991 if (seen_error () && is_gimple_omp (stmt
))
16993 gsi_replace (gsi_p
, gimple_build_nop (), true);
16997 switch (gimple_code (stmt
))
17001 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
17002 if ((ctx
|| task_shared_vars
)
17003 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
17004 lower_omp_regimplify_p
,
17005 ctx
? NULL
: &wi
, NULL
)
17006 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
17007 lower_omp_regimplify_p
,
17008 ctx
? NULL
: &wi
, NULL
)))
17009 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
17013 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
17015 case GIMPLE_EH_FILTER
:
17016 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
17019 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
17020 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
17022 case GIMPLE_TRANSACTION
:
17023 lower_omp (gimple_transaction_body_ptr (
17024 as_a
<gtransaction
*> (stmt
)),
17028 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
17030 case GIMPLE_OMP_PARALLEL
:
17031 case GIMPLE_OMP_TASK
:
17032 ctx
= maybe_lookup_ctx (stmt
);
17034 if (ctx
->cancellable
)
17035 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17036 lower_omp_taskreg (gsi_p
, ctx
);
17038 case GIMPLE_OMP_FOR
:
17039 ctx
= maybe_lookup_ctx (stmt
);
17041 if (ctx
->cancellable
)
17042 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17043 lower_omp_for (gsi_p
, ctx
);
17045 case GIMPLE_OMP_SECTIONS
:
17046 ctx
= maybe_lookup_ctx (stmt
);
17048 if (ctx
->cancellable
)
17049 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17050 lower_omp_sections (gsi_p
, ctx
);
17052 case GIMPLE_OMP_SINGLE
:
17053 ctx
= maybe_lookup_ctx (stmt
);
17055 lower_omp_single (gsi_p
, ctx
);
17057 case GIMPLE_OMP_MASTER
:
17058 ctx
= maybe_lookup_ctx (stmt
);
17060 lower_omp_master (gsi_p
, ctx
);
17062 case GIMPLE_OMP_TASKGROUP
:
17063 ctx
= maybe_lookup_ctx (stmt
);
17065 lower_omp_taskgroup (gsi_p
, ctx
);
17067 case GIMPLE_OMP_ORDERED
:
17068 ctx
= maybe_lookup_ctx (stmt
);
17070 lower_omp_ordered (gsi_p
, ctx
);
17072 case GIMPLE_OMP_CRITICAL
:
17073 ctx
= maybe_lookup_ctx (stmt
);
17075 lower_omp_critical (gsi_p
, ctx
);
17077 case GIMPLE_OMP_ATOMIC_LOAD
:
17078 if ((ctx
|| task_shared_vars
)
17079 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
17080 as_a
<gomp_atomic_load
*> (stmt
)),
17081 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
17082 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17084 case GIMPLE_OMP_TARGET
:
17085 ctx
= maybe_lookup_ctx (stmt
);
17087 lower_omp_target (gsi_p
, ctx
);
17089 case GIMPLE_OMP_TEAMS
:
17090 ctx
= maybe_lookup_ctx (stmt
);
17092 lower_omp_teams (gsi_p
, ctx
);
17094 case GIMPLE_OMP_GRID_BODY
:
17095 ctx
= maybe_lookup_ctx (stmt
);
17097 lower_omp_grid_body (gsi_p
, ctx
);
17101 call_stmt
= as_a
<gcall
*> (stmt
);
17102 fndecl
= gimple_call_fndecl (call_stmt
);
17104 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
17105 switch (DECL_FUNCTION_CODE (fndecl
))
17107 case BUILT_IN_GOMP_BARRIER
:
17111 case BUILT_IN_GOMP_CANCEL
:
17112 case BUILT_IN_GOMP_CANCELLATION_POINT
:
17115 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
17116 cctx
= cctx
->outer
;
17117 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
17118 if (!cctx
->cancellable
)
17120 if (DECL_FUNCTION_CODE (fndecl
)
17121 == BUILT_IN_GOMP_CANCELLATION_POINT
)
17123 stmt
= gimple_build_nop ();
17124 gsi_replace (gsi_p
, stmt
, false);
17128 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
17130 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
17131 gimple_call_set_fndecl (call_stmt
, fndecl
);
17132 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
17135 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
17136 gimple_call_set_lhs (call_stmt
, lhs
);
17137 tree fallthru_label
;
17138 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
17140 g
= gimple_build_label (fallthru_label
);
17141 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17142 g
= gimple_build_cond (NE_EXPR
, lhs
,
17143 fold_convert (TREE_TYPE (lhs
),
17144 boolean_false_node
),
17145 cctx
->cancel_label
, fallthru_label
);
17146 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17153 if ((ctx
|| task_shared_vars
)
17154 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
17157 /* Just remove clobbers, this should happen only if we have
17158 "privatized" local addressable variables in SIMD regions,
17159 the clobber isn't needed in that case and gimplifying address
17160 of the ARRAY_REF into a pointer and creating MEM_REF based
17161 clobber would create worse code than we get with the clobber
17163 if (gimple_clobber_p (stmt
))
17165 gsi_replace (gsi_p
, gimple_build_nop (), true);
17168 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17175 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
17177 location_t saved_location
= input_location
;
17178 gimple_stmt_iterator gsi
;
17179 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17180 lower_omp_1 (&gsi
, ctx
);
17181 /* During gimplification, we haven't folded statments inside offloading
17182 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17183 if (target_nesting_level
|| taskreg_nesting_level
)
17184 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17186 input_location
= saved_location
;
17189 /* Returen true if STMT is an assignment of a register-type into a local
17193 grid_reg_assignment_to_local_var_p (gimple
*stmt
)
17195 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
17198 tree lhs
= gimple_assign_lhs (assign
);
17199 if (TREE_CODE (lhs
) != VAR_DECL
17200 || !is_gimple_reg_type (TREE_TYPE (lhs
))
17201 || is_global_var (lhs
))
17206 /* Return true if all statements in SEQ are assignments to local register-type
17210 grid_seq_only_contains_local_assignments (gimple_seq seq
)
17215 gimple_stmt_iterator gsi
;
17216 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17217 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi
)))
17222 /* Scan statements in SEQ and call itself recursively on any bind. If during
17223 whole search only assignments to register-type local variables and one
17224 single OMP statement is encountered, return true, otherwise return false.
17225 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17226 are used for dumping a note about a failure. */
17229 grid_find_single_omp_among_assignments_1 (gimple_seq seq
, location_t target_loc
,
17230 const char *name
, gimple
**ret
)
17232 gimple_stmt_iterator gsi
;
17233 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17235 gimple
*stmt
= gsi_stmt (gsi
);
17237 if (grid_reg_assignment_to_local_var_p (stmt
))
17239 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17241 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind
),
17242 target_loc
, name
, ret
))
17245 else if (is_gimple_omp (stmt
))
17249 if (dump_enabled_p ())
17250 dump_printf_loc (MSG_NOTE
, target_loc
,
17251 "Will not turn target construct into a simple "
17252 "GPGPU kernel because %s construct contains "
17253 "multiple OpenMP constructs\n", name
);
17260 if (dump_enabled_p ())
17261 dump_printf_loc (MSG_NOTE
, target_loc
,
17262 "Will not turn target construct into a simple "
17263 "GPGPU kernel because %s construct contains "
17264 "a complex statement\n", name
);
17271 /* Scan statements in SEQ and make sure that it and any binds in it contain
17272 only assignments to local register-type variables and one OMP construct. If
17273 so, return that construct, otherwise return NULL. If dumping is enabled and
17274 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17278 grid_find_single_omp_among_assignments (gimple_seq seq
, location_t target_loc
,
17283 if (dump_enabled_p ())
17284 dump_printf_loc (MSG_NOTE
, target_loc
,
17285 "Will not turn target construct into a simple "
17286 "GPGPU kernel because %s construct has empty "
17292 gimple
*ret
= NULL
;
17293 if (grid_find_single_omp_among_assignments_1 (seq
, target_loc
, name
, &ret
))
17295 if (!ret
&& dump_enabled_p ())
17296 dump_printf_loc (MSG_NOTE
, target_loc
,
17297 "Will not turn target construct into a simple "
17298 "GPGPU kernel because %s construct does not contain"
17299 "any other OpenMP construct\n", name
);
17306 /* Walker function looking for statements there is no point gridifying (and for
17307 noreturn function calls which we cannot do). Return non-NULL if such a
17308 function is found. */
17311 grid_find_ungridifiable_statement (gimple_stmt_iterator
*gsi
,
17312 bool *handled_ops_p
,
17313 struct walk_stmt_info
*wi
)
17315 *handled_ops_p
= false;
17316 gimple
*stmt
= gsi_stmt (*gsi
);
17317 switch (gimple_code (stmt
))
17320 if (gimple_call_noreturn_p (as_a
<gcall
*> (stmt
)))
17322 *handled_ops_p
= true;
17324 return error_mark_node
;
17328 /* We may reduce the following list if we find a way to implement the
17329 clauses, but now there is no point trying further. */
17330 case GIMPLE_OMP_CRITICAL
:
17331 case GIMPLE_OMP_TASKGROUP
:
17332 case GIMPLE_OMP_TASK
:
17333 case GIMPLE_OMP_SECTION
:
17334 case GIMPLE_OMP_SECTIONS
:
17335 case GIMPLE_OMP_SECTIONS_SWITCH
:
17336 case GIMPLE_OMP_TARGET
:
17337 case GIMPLE_OMP_ORDERED
:
17338 *handled_ops_p
= true;
17340 return error_mark_node
;
17342 case GIMPLE_OMP_FOR
:
17343 if ((gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
17344 && gimple_omp_for_combined_into_p (stmt
))
17346 *handled_ops_p
= true;
17348 return error_mark_node
;
17359 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17360 kernel, return true, otherwise return false. In the case of success, also
17361 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17365 grid_target_follows_gridifiable_pattern (gomp_target
*target
, tree
*group_size_p
)
17367 if (gimple_omp_target_kind (target
) != GF_OMP_TARGET_KIND_REGION
)
17370 location_t tloc
= gimple_location (target
);
17372 = grid_find_single_omp_among_assignments (gimple_omp_body (target
),
17376 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17377 tree group_size
= NULL
;
17380 dump_printf_loc (MSG_NOTE
, tloc
,
17381 "Will not turn target construct into a simple "
17382 "GPGPU kernel because it does not have a sole teams "
17383 "construct in it.\n");
17387 tree clauses
= gimple_omp_teams_clauses (teams
);
17390 switch (OMP_CLAUSE_CODE (clauses
))
17392 case OMP_CLAUSE_NUM_TEAMS
:
17393 if (dump_enabled_p ())
17394 dump_printf_loc (MSG_NOTE
, tloc
,
17395 "Will not turn target construct into a "
17396 "gridified GPGPU kernel because we cannot "
17397 "handle num_teams clause of teams "
17401 case OMP_CLAUSE_REDUCTION
:
17402 if (dump_enabled_p ())
17403 dump_printf_loc (MSG_NOTE
, tloc
,
17404 "Will not turn target construct into a "
17405 "gridified GPGPU kernel because a reduction "
17406 "clause is present\n ");
17409 case OMP_CLAUSE_LASTPRIVATE
:
17410 if (dump_enabled_p ())
17411 dump_printf_loc (MSG_NOTE
, tloc
,
17412 "Will not turn target construct into a "
17413 "gridified GPGPU kernel because a lastprivate "
17414 "clause is present\n ");
17417 case OMP_CLAUSE_THREAD_LIMIT
:
17418 group_size
= OMP_CLAUSE_OPERAND (clauses
, 0);
17424 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17427 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (teams
), tloc
,
17431 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17434 dump_printf_loc (MSG_NOTE
, tloc
,
17435 "Will not turn target construct into a simple "
17436 "GPGPU kernel because the teams construct does not have "
17437 "a sole distribute construct in it.\n");
17441 gcc_assert (gimple_omp_for_kind (dist
) == GF_OMP_FOR_KIND_DISTRIBUTE
);
17442 if (!gimple_omp_for_combined_p (dist
))
17444 if (dump_enabled_p ())
17445 dump_printf_loc (MSG_NOTE
, tloc
,
17446 "Will not turn target construct into a gridified GPGPU "
17447 "kernel because we cannot handle a standalone "
17448 "distribute construct\n ");
17451 if (dist
->collapse
> 1)
17453 if (dump_enabled_p ())
17454 dump_printf_loc (MSG_NOTE
, tloc
,
17455 "Will not turn target construct into a gridified GPGPU "
17456 "kernel because the distribute construct contains "
17457 "collapse clause\n");
17460 struct omp_for_data fd
;
17461 extract_omp_for_data (dist
, &fd
, NULL
);
17464 if (group_size
&& !operand_equal_p (group_size
, fd
.chunk_size
, 0))
17466 if (dump_enabled_p ())
17467 dump_printf_loc (MSG_NOTE
, tloc
,
17468 "Will not turn target construct into a "
17469 "gridified GPGPU kernel because the teams "
17470 "thread limit is different from distribute "
17471 "schedule chunk\n");
17474 group_size
= fd
.chunk_size
;
17476 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (dist
), tloc
,
17478 gomp_parallel
*par
;
17479 if (!stmt
|| !(par
= dyn_cast
<gomp_parallel
*> (stmt
)))
17482 clauses
= gimple_omp_parallel_clauses (par
);
17485 switch (OMP_CLAUSE_CODE (clauses
))
17487 case OMP_CLAUSE_NUM_THREADS
:
17488 if (dump_enabled_p ())
17489 dump_printf_loc (MSG_NOTE
, tloc
,
17490 "Will not turn target construct into a gridified"
17491 "GPGPU kernel because there is a num_threads "
17492 "clause of the parallel construct\n");
17495 case OMP_CLAUSE_REDUCTION
:
17496 if (dump_enabled_p ())
17497 dump_printf_loc (MSG_NOTE
, tloc
,
17498 "Will not turn target construct into a "
17499 "gridified GPGPU kernel because a reduction "
17500 "clause is present\n ");
17503 case OMP_CLAUSE_LASTPRIVATE
:
17504 if (dump_enabled_p ())
17505 dump_printf_loc (MSG_NOTE
, tloc
,
17506 "Will not turn target construct into a "
17507 "gridified GPGPU kernel because a lastprivate "
17508 "clause is present\n ");
17514 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17517 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (par
), tloc
,
17520 if (!stmt
|| !(gfor
= dyn_cast
<gomp_for
*> (stmt
)))
17523 if (gimple_omp_for_kind (gfor
) != GF_OMP_FOR_KIND_FOR
)
17525 if (dump_enabled_p ())
17526 dump_printf_loc (MSG_NOTE
, tloc
,
17527 "Will not turn target construct into a gridified GPGPU "
17528 "kernel because the inner loop is not a simple for "
17532 if (gfor
->collapse
> 1)
17534 if (dump_enabled_p ())
17535 dump_printf_loc (MSG_NOTE
, tloc
,
17536 "Will not turn target construct into a gridified GPGPU "
17537 "kernel because the inner loop contains collapse "
17542 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor
)))
17544 if (dump_enabled_p ())
17545 dump_printf_loc (MSG_NOTE
, tloc
,
17546 "Will not turn target construct into a gridified GPGPU "
17547 "kernel because the inner loop pre_body contains"
17548 "a complex instruction\n");
17552 clauses
= gimple_omp_for_clauses (gfor
);
17555 switch (OMP_CLAUSE_CODE (clauses
))
17557 case OMP_CLAUSE_SCHEDULE
:
17558 if (OMP_CLAUSE_SCHEDULE_KIND (clauses
) != OMP_CLAUSE_SCHEDULE_AUTO
)
17560 if (dump_enabled_p ())
17561 dump_printf_loc (MSG_NOTE
, tloc
,
17562 "Will not turn target construct into a "
17563 "gridified GPGPU kernel because the inner "
17564 "loop has a non-automatic scheduling clause\n");
17569 case OMP_CLAUSE_REDUCTION
:
17570 if (dump_enabled_p ())
17571 dump_printf_loc (MSG_NOTE
, tloc
,
17572 "Will not turn target construct into a "
17573 "gridified GPGPU kernel because a reduction "
17574 "clause is present\n ");
17577 case OMP_CLAUSE_LASTPRIVATE
:
17578 if (dump_enabled_p ())
17579 dump_printf_loc (MSG_NOTE
, tloc
,
17580 "Will not turn target construct into a "
17581 "gridified GPGPU kernel because a lastprivate "
17582 "clause is present\n ");
17588 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17591 struct walk_stmt_info wi
;
17592 memset (&wi
, 0, sizeof (wi
));
17593 if (walk_gimple_seq (gimple_omp_body (gfor
),
17594 grid_find_ungridifiable_statement
,
17597 gimple
*bad
= (gimple
*) wi
.info
;
17598 if (dump_enabled_p ())
17600 if (is_gimple_call (bad
))
17601 dump_printf_loc (MSG_NOTE
, tloc
,
17602 "Will not turn target construct into a gridified "
17603 " GPGPU kernel because the inner loop contains "
17604 "call to a noreturn function\n");
17605 if (gimple_code (bad
) == GIMPLE_OMP_FOR
)
17606 dump_printf_loc (MSG_NOTE
, tloc
,
17607 "Will not turn target construct into a gridified "
17608 " GPGPU kernel because the inner loop contains "
17609 "a simd construct\n");
17611 dump_printf_loc (MSG_NOTE
, tloc
,
17612 "Will not turn target construct into a gridified "
17613 "GPGPU kernel because the inner loop contains "
17614 "statement %s which cannot be transformed\n",
17615 gimple_code_name
[(int) gimple_code (bad
)]);
17620 *group_size_p
= group_size
;
17624 /* Operand walker, used to remap pre-body declarations according to a hash map
17625 provided in DATA. */
17628 grid_remap_prebody_decls (tree
*tp
, int *walk_subtrees
, void *data
)
17632 if (DECL_P (t
) || TYPE_P (t
))
17633 *walk_subtrees
= 0;
17635 *walk_subtrees
= 1;
17637 if (TREE_CODE (t
) == VAR_DECL
)
17639 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17640 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17641 tree
*repl
= declmap
->get (t
);
17648 /* Copy leading register-type assignments to local variables in SRC to just
17649 before DST, Creating temporaries, adjusting mapping of operands in WI and
17650 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17651 Return the first statement that does not conform to
17652 grid_reg_assignment_to_local_var_p or NULL. */
17655 grid_copy_leading_local_assignments (gimple_seq src
, gimple_stmt_iterator
*dst
,
17656 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17658 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17659 gimple_stmt_iterator gsi
;
17660 for (gsi
= gsi_start (src
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17662 gimple
*stmt
= gsi_stmt (gsi
);
17663 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17665 gimple
*r
= grid_copy_leading_local_assignments
17666 (gimple_bind_body (bind
), dst
, tgt_bind
, wi
);
17672 if (!grid_reg_assignment_to_local_var_p (stmt
))
17674 tree lhs
= gimple_assign_lhs (as_a
<gassign
*> (stmt
));
17675 tree repl
= copy_var_decl (lhs
, create_tmp_var_name (NULL
),
17677 DECL_CONTEXT (repl
) = current_function_decl
;
17678 gimple_bind_append_vars (tgt_bind
, repl
);
17680 declmap
->put (lhs
, repl
);
17681 gassign
*copy
= as_a
<gassign
*> (gimple_copy (stmt
));
17682 walk_gimple_op (copy
, grid_remap_prebody_decls
, wi
);
17683 gsi_insert_before (dst
, copy
, GSI_SAME_STMT
);
17688 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17689 components, mark them as part of kernel and return the inner loop, and copy
17690 assignment leading to them just before DST, remapping them using WI and
17691 adding new temporaries to TGT_BIND. */
17694 grid_process_kernel_body_copy (gimple_seq seq
, gimple_stmt_iterator
*dst
,
17695 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17697 gimple
*stmt
= grid_copy_leading_local_assignments (seq
, dst
, tgt_bind
, wi
);
17698 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17699 gcc_assert (teams
);
17700 gimple_omp_teams_set_grid_phony (teams
, true);
17701 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (teams
), dst
,
17703 gcc_checking_assert (stmt
);
17704 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17706 gimple_seq prebody
= gimple_omp_for_pre_body (dist
);
17708 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17709 gimple_omp_for_set_grid_phony (dist
, true);
17710 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (dist
), dst
,
17712 gcc_checking_assert (stmt
);
17714 gomp_parallel
*parallel
= as_a
<gomp_parallel
*> (stmt
);
17715 gimple_omp_parallel_set_grid_phony (parallel
, true);
17716 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (parallel
), dst
,
17718 gomp_for
*inner_loop
= as_a
<gomp_for
*> (stmt
);
17719 gimple_omp_for_set_kind (inner_loop
, GF_OMP_FOR_KIND_GRID_LOOP
);
17720 prebody
= gimple_omp_for_pre_body (inner_loop
);
17722 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17727 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17728 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17729 is the bind into which temporaries inserted before TARGET should be
17733 grid_attempt_target_gridification (gomp_target
*target
,
17734 gimple_stmt_iterator
*gsi
,
17738 if (!target
|| !grid_target_follows_gridifiable_pattern (target
, &group_size
))
17741 location_t loc
= gimple_location (target
);
17742 if (dump_enabled_p ())
17743 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
17744 "Target construct will be turned into a gridified GPGPU "
17747 /* Copy target body to a GPUKERNEL construct: */
17748 gimple_seq kernel_seq
= copy_gimple_seq_and_replace_locals
17749 (gimple_omp_body (target
));
17751 hash_map
<tree
, tree
> *declmap
= new hash_map
<tree
, tree
>;
17752 struct walk_stmt_info wi
;
17753 memset (&wi
, 0, sizeof (struct walk_stmt_info
));
17756 /* Copy assignments in between OMP statements before target, mark OMP
17757 statements within copy appropriatly. */
17758 gomp_for
*inner_loop
= grid_process_kernel_body_copy (kernel_seq
, gsi
,
17761 gbind
*old_bind
= as_a
<gbind
*> (gimple_seq_first (gimple_omp_body (target
)));
17762 gbind
*new_bind
= as_a
<gbind
*> (gimple_seq_first (kernel_seq
));
17763 tree new_block
= gimple_bind_block (new_bind
);
17764 tree enc_block
= BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind
));
17765 BLOCK_CHAIN (new_block
) = BLOCK_SUBBLOCKS (enc_block
);
17766 BLOCK_SUBBLOCKS (enc_block
) = new_block
;
17767 BLOCK_SUPERCONTEXT (new_block
) = enc_block
;
17768 gimple
*gpukernel
= gimple_build_omp_grid_body (kernel_seq
);
17769 gimple_seq_add_stmt
17770 (gimple_bind_body_ptr (as_a
<gbind
*> (gimple_omp_body (target
))),
17773 walk_tree (&group_size
, grid_remap_prebody_decls
, &wi
, NULL
);
17774 push_gimplify_context ();
17775 size_t collapse
= gimple_omp_for_collapse (inner_loop
);
17776 for (size_t i
= 0; i
< collapse
; i
++)
17778 tree itype
, type
= TREE_TYPE (gimple_omp_for_index (inner_loop
, i
));
17779 if (POINTER_TYPE_P (type
))
17780 itype
= signed_type_for (type
);
17784 enum tree_code cond_code
= gimple_omp_for_cond (inner_loop
, i
);
17785 tree n1
= unshare_expr (gimple_omp_for_initial (inner_loop
, i
));
17786 walk_tree (&n1
, grid_remap_prebody_decls
, &wi
, NULL
);
17787 tree n2
= unshare_expr (gimple_omp_for_final (inner_loop
, i
));
17788 walk_tree (&n2
, grid_remap_prebody_decls
, &wi
, NULL
);
17789 adjust_for_condition (loc
, &cond_code
, &n2
);
17791 step
= get_omp_for_step_from_incr (loc
,
17792 gimple_omp_for_incr (inner_loop
, i
));
17793 gimple_seq tmpseq
= NULL
;
17794 n1
= fold_convert (itype
, n1
);
17795 n2
= fold_convert (itype
, n2
);
17796 tree t
= build_int_cst (itype
, (cond_code
== LT_EXPR
? -1 : 1));
17797 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
17798 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
17799 t
= fold_build2 (MINUS_EXPR
, itype
, t
, n1
);
17800 if (TYPE_UNSIGNED (itype
) && cond_code
== GT_EXPR
)
17801 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
17802 fold_build1 (NEGATE_EXPR
, itype
, t
),
17803 fold_build1 (NEGATE_EXPR
, itype
, step
));
17805 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
17806 tree gs
= fold_convert (uint32_type_node
, t
);
17807 gimplify_expr (&gs
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17808 if (!gimple_seq_empty_p (tmpseq
))
17809 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17812 if (i
== 0 && group_size
)
17814 ws
= fold_convert (uint32_type_node
, group_size
);
17816 gimplify_expr (&ws
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17817 if (!gimple_seq_empty_p (tmpseq
))
17818 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17821 ws
= build_zero_cst (uint32_type_node
);
17823 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__GRIDDIM_
);
17824 OMP_CLAUSE__GRIDDIM__DIMENSION (c
) = i
;
17825 OMP_CLAUSE__GRIDDIM__SIZE (c
) = gs
;
17826 OMP_CLAUSE__GRIDDIM__GROUP (c
) = ws
;
17827 OMP_CLAUSE_CHAIN (c
) = gimple_omp_target_clauses (target
);
17828 gimple_omp_target_set_clauses (target
, c
);
17830 pop_gimplify_context (tgt_bind
);
17835 /* Walker function doing all the work for create_target_kernels. */
17838 grid_gridify_all_targets_stmt (gimple_stmt_iterator
*gsi
,
17839 bool *handled_ops_p
,
17840 struct walk_stmt_info
*incoming
)
17842 *handled_ops_p
= false;
17844 gimple
*stmt
= gsi_stmt (*gsi
);
17845 gomp_target
*target
= dyn_cast
<gomp_target
*> (stmt
);
17848 gbind
*tgt_bind
= (gbind
*) incoming
->info
;
17849 gcc_checking_assert (tgt_bind
);
17850 grid_attempt_target_gridification (target
, gsi
, tgt_bind
);
17853 gbind
*bind
= dyn_cast
<gbind
*> (stmt
);
17856 *handled_ops_p
= true;
17857 struct walk_stmt_info wi
;
17858 memset (&wi
, 0, sizeof (wi
));
17860 walk_gimple_seq_mod (gimple_bind_body_ptr (bind
),
17861 grid_gridify_all_targets_stmt
, NULL
, &wi
);
17866 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17867 have their bodies duplicated, with the new copy being put into a
17868 gimple_omp_grid_body statement. All kernel-related construct within the
17869 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17870 re-structuring is often needed, such as copying pre-bodies before the target
17871 construct so that kernel grid sizes can be computed. */
17874 grid_gridify_all_targets (gimple_seq
*body_p
)
17876 struct walk_stmt_info wi
;
17877 memset (&wi
, 0, sizeof (wi
));
17878 walk_gimple_seq_mod (body_p
, grid_gridify_all_targets_stmt
, NULL
, &wi
);
17882 /* Main entry point. */
17884 static unsigned int
17885 execute_lower_omp (void)
17891 /* This pass always runs, to provide PROP_gimple_lomp.
17892 But often, there is nothing to do. */
17893 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
17894 && flag_openmp_simd
== 0)
17897 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
17898 delete_omp_context
);
17900 body
= gimple_body (current_function_decl
);
17902 if (hsa_gen_requested_p ())
17903 grid_gridify_all_targets (&body
);
17905 scan_omp (&body
, NULL
);
17906 gcc_assert (taskreg_nesting_level
== 0);
17907 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
17908 finish_taskreg_scan (ctx
);
17909 taskreg_contexts
.release ();
17911 if (all_contexts
->root
)
17913 if (task_shared_vars
)
17914 push_gimplify_context ();
17915 lower_omp (&body
, NULL
);
17916 if (task_shared_vars
)
17917 pop_gimplify_context (NULL
);
17922 splay_tree_delete (all_contexts
);
17923 all_contexts
= NULL
;
17925 BITMAP_FREE (task_shared_vars
);
17931 const pass_data pass_data_lower_omp
=
17933 GIMPLE_PASS
, /* type */
17934 "omplower", /* name */
17935 OPTGROUP_NONE
, /* optinfo_flags */
17936 TV_NONE
, /* tv_id */
17937 PROP_gimple_any
, /* properties_required */
17938 PROP_gimple_lomp
, /* properties_provided */
17939 0, /* properties_destroyed */
17940 0, /* todo_flags_start */
17941 0, /* todo_flags_finish */
17944 class pass_lower_omp
: public gimple_opt_pass
17947 pass_lower_omp (gcc::context
*ctxt
)
17948 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
17951 /* opt_pass methods: */
17952 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
17954 }; // class pass_lower_omp
17956 } // anon namespace
17959 make_pass_lower_omp (gcc::context
*ctxt
)
17961 return new pass_lower_omp (ctxt
);
17964 /* The following is a utility to diagnose structured block violations.
17965 It is not part of the "omplower" pass, as that's invoked too late. It
17966 should be invoked by the respective front ends after gimplification. */
17968 static splay_tree all_labels
;
17970 /* Check for mismatched contexts and generate an error if needed. Return
17971 true if an error is detected. */
17974 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
17975 gimple
*branch_ctx
, gimple
*label_ctx
)
17977 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
17978 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
17980 if (label_ctx
== branch_ctx
)
17983 const char* kind
= NULL
;
17988 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
17989 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
17991 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
17992 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
17993 kind
= "Cilk Plus";
17997 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
17998 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
18000 gcc_checking_assert (kind
== NULL
);
18006 gcc_checking_assert (flag_openmp
);
18011 Previously we kept track of the label's entire context in diagnose_sb_[12]
18012 so we could traverse it and issue a correct "exit" or "enter" error
18013 message upon a structured block violation.
18015 We built the context by building a list with tree_cons'ing, but there is
18016 no easy counterpart in gimple tuples. It seems like far too much work
18017 for issuing exit/enter error messages. If someone really misses the
18018 distinct error message... patches welcome.
18022 /* Try to avoid confusing the user by producing and error message
18023 with correct "exit" or "enter" verbiage. We prefer "exit"
18024 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
18025 if (branch_ctx
== NULL
)
18031 if (TREE_VALUE (label_ctx
) == branch_ctx
)
18036 label_ctx
= TREE_CHAIN (label_ctx
);
18041 error ("invalid exit from %s structured block", kind
);
18043 error ("invalid entry to %s structured block", kind
);
18046 /* If it's obvious we have an invalid entry, be specific about the error. */
18047 if (branch_ctx
== NULL
)
18048 error ("invalid entry to %s structured block", kind
);
18051 /* Otherwise, be vague and lazy, but efficient. */
18052 error ("invalid branch to/from %s structured block", kind
);
18055 gsi_replace (gsi_p
, gimple_build_nop (), false);
18059 /* Pass 1: Create a minimal tree of structured blocks, and record
18060 where each label is found. */
18063 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18064 struct walk_stmt_info
*wi
)
18066 gimple
*context
= (gimple
*) wi
->info
;
18067 gimple
*inner_context
;
18068 gimple
*stmt
= gsi_stmt (*gsi_p
);
18070 *handled_ops_p
= true;
18072 switch (gimple_code (stmt
))
18076 case GIMPLE_OMP_PARALLEL
:
18077 case GIMPLE_OMP_TASK
:
18078 case GIMPLE_OMP_SECTIONS
:
18079 case GIMPLE_OMP_SINGLE
:
18080 case GIMPLE_OMP_SECTION
:
18081 case GIMPLE_OMP_MASTER
:
18082 case GIMPLE_OMP_ORDERED
:
18083 case GIMPLE_OMP_CRITICAL
:
18084 case GIMPLE_OMP_TARGET
:
18085 case GIMPLE_OMP_TEAMS
:
18086 case GIMPLE_OMP_TASKGROUP
:
18087 /* The minimal context here is just the current OMP construct. */
18088 inner_context
= stmt
;
18089 wi
->info
= inner_context
;
18090 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18091 wi
->info
= context
;
18094 case GIMPLE_OMP_FOR
:
18095 inner_context
= stmt
;
18096 wi
->info
= inner_context
;
18097 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18099 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
18100 diagnose_sb_1
, NULL
, wi
);
18101 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18102 wi
->info
= context
;
18106 splay_tree_insert (all_labels
,
18107 (splay_tree_key
) gimple_label_label (
18108 as_a
<glabel
*> (stmt
)),
18109 (splay_tree_value
) context
);
18119 /* Pass 2: Check each branch and see if its context differs from that of
18120 the destination label's context. */
18123 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18124 struct walk_stmt_info
*wi
)
18126 gimple
*context
= (gimple
*) wi
->info
;
18128 gimple
*stmt
= gsi_stmt (*gsi_p
);
18130 *handled_ops_p
= true;
18132 switch (gimple_code (stmt
))
18136 case GIMPLE_OMP_PARALLEL
:
18137 case GIMPLE_OMP_TASK
:
18138 case GIMPLE_OMP_SECTIONS
:
18139 case GIMPLE_OMP_SINGLE
:
18140 case GIMPLE_OMP_SECTION
:
18141 case GIMPLE_OMP_MASTER
:
18142 case GIMPLE_OMP_ORDERED
:
18143 case GIMPLE_OMP_CRITICAL
:
18144 case GIMPLE_OMP_TARGET
:
18145 case GIMPLE_OMP_TEAMS
:
18146 case GIMPLE_OMP_TASKGROUP
:
18148 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18149 wi
->info
= context
;
18152 case GIMPLE_OMP_FOR
:
18154 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18156 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
18157 diagnose_sb_2
, NULL
, wi
);
18158 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18159 wi
->info
= context
;
18164 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
18165 tree lab
= gimple_cond_true_label (cond_stmt
);
18168 n
= splay_tree_lookup (all_labels
,
18169 (splay_tree_key
) lab
);
18170 diagnose_sb_0 (gsi_p
, context
,
18171 n
? (gimple
*) n
->value
: NULL
);
18173 lab
= gimple_cond_false_label (cond_stmt
);
18176 n
= splay_tree_lookup (all_labels
,
18177 (splay_tree_key
) lab
);
18178 diagnose_sb_0 (gsi_p
, context
,
18179 n
? (gimple
*) n
->value
: NULL
);
18186 tree lab
= gimple_goto_dest (stmt
);
18187 if (TREE_CODE (lab
) != LABEL_DECL
)
18190 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18191 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
18195 case GIMPLE_SWITCH
:
18197 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
18199 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
18201 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
18202 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18203 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
18209 case GIMPLE_RETURN
:
18210 diagnose_sb_0 (gsi_p
, context
, NULL
);
18220 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18223 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
18226 gimple
*last
= last_stmt (bb
);
18227 enum gimple_code code
= gimple_code (last
);
18228 struct omp_region
*cur_region
= *region
;
18229 bool fallthru
= false;
18233 case GIMPLE_OMP_PARALLEL
:
18234 case GIMPLE_OMP_TASK
:
18235 case GIMPLE_OMP_FOR
:
18236 case GIMPLE_OMP_SINGLE
:
18237 case GIMPLE_OMP_TEAMS
:
18238 case GIMPLE_OMP_MASTER
:
18239 case GIMPLE_OMP_TASKGROUP
:
18240 case GIMPLE_OMP_CRITICAL
:
18241 case GIMPLE_OMP_SECTION
:
18242 case GIMPLE_OMP_GRID_BODY
:
18243 cur_region
= new_omp_region (bb
, code
, cur_region
);
18247 case GIMPLE_OMP_ORDERED
:
18248 cur_region
= new_omp_region (bb
, code
, cur_region
);
18250 if (find_omp_clause (gimple_omp_ordered_clauses
18251 (as_a
<gomp_ordered
*> (last
)),
18252 OMP_CLAUSE_DEPEND
))
18253 cur_region
= cur_region
->outer
;
18256 case GIMPLE_OMP_TARGET
:
18257 cur_region
= new_omp_region (bb
, code
, cur_region
);
18259 switch (gimple_omp_target_kind (last
))
18261 case GF_OMP_TARGET_KIND_REGION
:
18262 case GF_OMP_TARGET_KIND_DATA
:
18263 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
18264 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
18265 case GF_OMP_TARGET_KIND_OACC_DATA
:
18266 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
18268 case GF_OMP_TARGET_KIND_UPDATE
:
18269 case GF_OMP_TARGET_KIND_ENTER_DATA
:
18270 case GF_OMP_TARGET_KIND_EXIT_DATA
:
18271 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
18272 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
18273 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
18274 cur_region
= cur_region
->outer
;
18277 gcc_unreachable ();
18281 case GIMPLE_OMP_SECTIONS
:
18282 cur_region
= new_omp_region (bb
, code
, cur_region
);
18286 case GIMPLE_OMP_SECTIONS_SWITCH
:
18290 case GIMPLE_OMP_ATOMIC_LOAD
:
18291 case GIMPLE_OMP_ATOMIC_STORE
:
18295 case GIMPLE_OMP_RETURN
:
18296 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18297 somewhere other than the next block. This will be
18299 cur_region
->exit
= bb
;
18300 if (cur_region
->type
== GIMPLE_OMP_TASK
)
18301 /* Add an edge corresponding to not scheduling the task
18303 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
18304 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
18305 cur_region
= cur_region
->outer
;
18308 case GIMPLE_OMP_CONTINUE
:
18309 cur_region
->cont
= bb
;
18310 switch (cur_region
->type
)
18312 case GIMPLE_OMP_FOR
:
18313 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18314 succs edges as abnormal to prevent splitting
18316 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
18317 /* Make the loopback edge. */
18318 make_edge (bb
, single_succ (cur_region
->entry
),
18321 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18322 corresponds to the case that the body of the loop
18323 is not executed at all. */
18324 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
18325 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
18329 case GIMPLE_OMP_SECTIONS
:
18330 /* Wire up the edges into and out of the nested sections. */
18332 basic_block switch_bb
= single_succ (cur_region
->entry
);
18334 struct omp_region
*i
;
18335 for (i
= cur_region
->inner
; i
; i
= i
->next
)
18337 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
18338 make_edge (switch_bb
, i
->entry
, 0);
18339 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
18342 /* Make the loopback edge to the block with
18343 GIMPLE_OMP_SECTIONS_SWITCH. */
18344 make_edge (bb
, switch_bb
, 0);
18346 /* Make the edge from the switch to exit. */
18347 make_edge (switch_bb
, bb
->next_bb
, 0);
18352 case GIMPLE_OMP_TASK
:
18357 gcc_unreachable ();
18362 gcc_unreachable ();
18365 if (*region
!= cur_region
)
18367 *region
= cur_region
;
18369 *region_idx
= cur_region
->entry
->index
;
18377 static unsigned int
18378 diagnose_omp_structured_block_errors (void)
18380 struct walk_stmt_info wi
;
18381 gimple_seq body
= gimple_body (current_function_decl
);
18383 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
18385 memset (&wi
, 0, sizeof (wi
));
18386 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
18388 memset (&wi
, 0, sizeof (wi
));
18389 wi
.want_locations
= true;
18390 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
18392 gimple_set_body (current_function_decl
, body
);
18394 splay_tree_delete (all_labels
);
18402 const pass_data pass_data_diagnose_omp_blocks
=
18404 GIMPLE_PASS
, /* type */
18405 "*diagnose_omp_blocks", /* name */
18406 OPTGROUP_NONE
, /* optinfo_flags */
18407 TV_NONE
, /* tv_id */
18408 PROP_gimple_any
, /* properties_required */
18409 0, /* properties_provided */
18410 0, /* properties_destroyed */
18411 0, /* todo_flags_start */
18412 0, /* todo_flags_finish */
18415 class pass_diagnose_omp_blocks
: public gimple_opt_pass
18418 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18419 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
18422 /* opt_pass methods: */
18423 virtual bool gate (function
*)
18425 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
18427 virtual unsigned int execute (function
*)
18429 return diagnose_omp_structured_block_errors ();
18432 }; // class pass_diagnose_omp_blocks
18434 } // anon namespace
18437 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18439 return new pass_diagnose_omp_blocks (ctxt
);
18442 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18443 adds their addresses and sizes to constructor-vector V_CTOR. */
18445 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18446 vec
<constructor_elt
, va_gc
> *v_ctor
)
18448 unsigned len
= vec_safe_length (v_decls
);
18449 for (unsigned i
= 0; i
< len
; i
++)
18451 tree it
= (*v_decls
)[i
];
18452 bool is_var
= TREE_CODE (it
) == VAR_DECL
;
18455 #ifdef ACCEL_COMPILER
18456 && DECL_HAS_VALUE_EXPR_P (it
)
18458 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
18460 tree size
= NULL_TREE
;
18462 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
18466 addr
= build_fold_addr_expr (it
);
18469 #ifdef ACCEL_COMPILER
18470 /* For "omp declare target link" vars add address of the pointer to
18471 the target table, instead of address of the var. */
18472 tree value_expr
= DECL_VALUE_EXPR (it
);
18473 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
18474 varpool_node::finalize_decl (link_ptr_decl
);
18475 addr
= build_fold_addr_expr (link_ptr_decl
);
18477 addr
= build_fold_addr_expr (it
);
18480 /* Most significant bit of the size marks "omp declare target link"
18481 vars in host and target tables. */
18482 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
18483 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
18484 * BITS_PER_UNIT
- 1);
18485 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
18488 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
18490 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
18494 /* Create new symbols containing (address, size) pairs for global variables,
18495 marked with "omp declare target" attribute, as well as addresses for the
18496 functions, which are outlined offloading regions. */
18498 omp_finish_file (void)
18500 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18501 unsigned num_vars
= vec_safe_length (offload_vars
);
18503 if (num_funcs
== 0 && num_vars
== 0)
18506 if (targetm_common
.have_named_sections
)
18508 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18509 vec_alloc (v_f
, num_funcs
);
18510 vec_alloc (v_v
, num_vars
* 2);
18512 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18513 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18515 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18517 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18519 SET_TYPE_ALIGN (vars_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
18520 SET_TYPE_ALIGN (funcs_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
18521 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18522 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18523 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18524 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18525 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18526 get_identifier (".offload_func_table"),
18528 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18529 get_identifier (".offload_var_table"),
18531 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18532 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18533 otherwise a joint table in a binary will contain padding between
18534 tables from multiple object files. */
18535 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18536 SET_DECL_ALIGN (funcs_decl
, TYPE_ALIGN (funcs_decl_type
));
18537 SET_DECL_ALIGN (vars_decl
, TYPE_ALIGN (vars_decl_type
));
18538 DECL_INITIAL (funcs_decl
) = ctor_f
;
18539 DECL_INITIAL (vars_decl
) = ctor_v
;
18540 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18541 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18543 varpool_node::finalize_decl (vars_decl
);
18544 varpool_node::finalize_decl (funcs_decl
);
18548 for (unsigned i
= 0; i
< num_funcs
; i
++)
18550 tree it
= (*offload_funcs
)[i
];
18551 targetm
.record_offload_symbol (it
);
18553 for (unsigned i
= 0; i
< num_vars
; i
++)
18555 tree it
= (*offload_vars
)[i
];
18556 targetm
.record_offload_symbol (it
);
18561 /* Find the number of threads (POS = false), or thread number (POS =
18562 true) for an OpenACC region partitioned as MASK. Setup code
18563 required for the calculation is added to SEQ. */
18566 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18568 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18571 /* Start at gang level, and examine relevant dimension indices. */
18572 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18573 if (GOMP_DIM_MASK (ix
) & mask
)
18575 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18579 /* We had an outer index, so scale that by the size of
18581 tree n
= create_tmp_var (integer_type_node
);
18583 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18585 gimple_call_set_lhs (call
, n
);
18586 gimple_seq_add_stmt (seq
, call
);
18587 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18591 /* Determine index in this dimension. */
18592 tree id
= create_tmp_var (integer_type_node
);
18593 gimple
*call
= gimple_build_call_internal
18594 (IFN_GOACC_DIM_POS
, 1, arg
);
18596 gimple_call_set_lhs (call
, id
);
18597 gimple_seq_add_stmt (seq
, call
);
18599 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18605 if (res
== NULL_TREE
)
18606 res
= integer_zero_node
;
18611 /* Transform IFN_GOACC_LOOP calls to actual code. See
18612 expand_oacc_for for where these are generated. At the vector
18613 level, we stride loops, such that each member of a warp will
18614 operate on adjacent iterations. At the worker and gang level,
18615 each gang/warp executes a set of contiguous iterations. Chunking
18616 can override this such that each iteration engine executes a
18617 contiguous chunk, and then moves on to stride to the next chunk. */
18620 oacc_xform_loop (gcall
*call
)
18622 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18623 enum ifn_goacc_loop_kind code
18624 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18625 tree dir
= gimple_call_arg (call
, 1);
18626 tree range
= gimple_call_arg (call
, 2);
18627 tree step
= gimple_call_arg (call
, 3);
18628 tree chunk_size
= NULL_TREE
;
18629 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18630 tree lhs
= gimple_call_lhs (call
);
18631 tree type
= TREE_TYPE (lhs
);
18632 tree diff_type
= TREE_TYPE (range
);
18633 tree r
= NULL_TREE
;
18634 gimple_seq seq
= NULL
;
18635 bool chunking
= false, striding
= true;
18636 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18637 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18639 #ifdef ACCEL_COMPILER
18640 chunk_size
= gimple_call_arg (call
, 4);
18641 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18642 || integer_zerop (chunk_size
)) /* Default (also static). */
18644 /* If we're at the gang level, we want each to execute a
18645 contiguous run of iterations. Otherwise we want each element
18647 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18652 /* Chunk of size 1 is striding. */
18653 striding
= integer_onep (chunk_size
);
18654 chunking
= !striding
;
18658 /* striding=true, chunking=true
18660 striding=true, chunking=false
18662 striding=false,chunking=true
18663 -> chunks=ceil (range/(chunksize*threads*step))
18664 striding=false,chunking=false
18665 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18666 push_gimplify_context (true);
18670 default: gcc_unreachable ();
18672 case IFN_GOACC_LOOP_CHUNKS
:
18674 r
= build_int_cst (type
, 1);
18678 = (range - dir) / (chunks * step * num_threads) + dir */
18679 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18680 per
= fold_convert (type
, per
);
18681 chunk_size
= fold_convert (type
, chunk_size
);
18682 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18683 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18684 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18685 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18686 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18690 case IFN_GOACC_LOOP_STEP
:
18692 /* If striding, step by the entire compute volume, otherwise
18693 step by the inner volume. */
18694 unsigned volume
= striding
? mask
: inner_mask
;
18696 r
= oacc_thread_numbers (false, volume
, &seq
);
18697 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18701 case IFN_GOACC_LOOP_OFFSET
:
18704 r
= oacc_thread_numbers (true, mask
, &seq
);
18705 r
= fold_convert (diff_type
, r
);
18709 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18710 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18711 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18712 inner_size
, outer_size
);
18714 volume
= fold_convert (diff_type
, volume
);
18716 chunk_size
= fold_convert (diff_type
, chunk_size
);
18719 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18721 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18722 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18723 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18726 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18727 fold_convert (diff_type
, inner_size
));
18728 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18729 r
= fold_convert (diff_type
, r
);
18730 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18732 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18733 inner
= fold_convert (diff_type
, inner
);
18734 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18738 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18740 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18741 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18743 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18746 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18747 if (type
!= diff_type
)
18748 r
= fold_convert (type
, r
);
18751 case IFN_GOACC_LOOP_BOUND
:
18756 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18757 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18758 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18759 inner_size
, outer_size
);
18761 volume
= fold_convert (diff_type
, volume
);
18763 chunk_size
= fold_convert (diff_type
, chunk_size
);
18766 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18768 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18769 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18770 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18773 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18774 fold_convert (diff_type
, inner_size
));
18776 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18778 tree offset
= gimple_call_arg (call
, 6);
18779 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18780 fold_convert (diff_type
, offset
));
18781 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18782 diff_type
, r
, range
);
18784 if (diff_type
!= type
)
18785 r
= fold_convert (type
, r
);
18789 gimplify_assign (lhs
, r
, &seq
);
18791 pop_gimplify_context (NULL
);
18793 gsi_replace_with_seq (&gsi
, seq
, true);
18796 /* Default partitioned and minimum partitioned dimensions. */
18798 static int oacc_default_dims
[GOMP_DIM_MAX
];
18799 static int oacc_min_dims
[GOMP_DIM_MAX
];
18801 /* Parse the default dimension parameter. This is a set of
18802 :-separated optional compute dimensions. Each specified dimension
18803 is a positive integer. When device type support is added, it is
18804 planned to be a comma separated list of such compute dimensions,
18805 with all but the first prefixed by the colon-terminated device
18809 oacc_parse_default_dims (const char *dims
)
18813 for (ix
= GOMP_DIM_MAX
; ix
--;)
18815 oacc_default_dims
[ix
] = -1;
18816 oacc_min_dims
[ix
] = 1;
18819 #ifndef ACCEL_COMPILER
18820 /* Cannot be overridden on the host. */
18825 const char *pos
= dims
;
18827 for (ix
= 0; *pos
&& ix
!= GOMP_DIM_MAX
; ix
++)
18842 val
= strtol (pos
, CONST_CAST (char **, &eptr
), 10);
18843 if (errno
|| val
<= 0 || (int) val
!= val
)
18846 oacc_default_dims
[ix
] = (int) val
;
18852 error_at (UNKNOWN_LOCATION
,
18853 "-fopenacc-dim operand is malformed at '%s'", pos
);
18857 /* Allow the backend to validate the dimensions. */
18858 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_default_dims
, -1);
18859 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_min_dims
, -2);
18862 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18863 raw attribute. DIMS is an array of dimensions, which is filled in.
18864 LEVEL is the partitioning level of a routine, or -1 for an offload
18865 region itself. USED is the mask of partitioned execution in the
18869 oacc_validate_dims (tree fn
, tree attrs
, int *dims
, int level
, unsigned used
)
18871 tree purpose
[GOMP_DIM_MAX
];
18873 tree pos
= TREE_VALUE (attrs
);
18874 bool is_kernel
= oacc_fn_attrib_kernels_p (attrs
);
18876 /* Make sure the attribute creator attached the dimension
18880 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18882 purpose
[ix
] = TREE_PURPOSE (pos
);
18883 tree val
= TREE_VALUE (pos
);
18884 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18885 pos
= TREE_CHAIN (pos
);
18888 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, level
);
18890 /* Default anything left to 1 or a partitioned default. */
18891 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18894 /* The OpenACC spec says 'If the [num_gangs] clause is not
18895 specified, an implementation-defined default will be used;
18896 the default may depend on the code within the construct.'
18897 (2.5.6). Thus an implementation is free to choose
18898 non-unity default for a parallel region that doesn't have
18899 any gang-partitioned loops. However, it appears that there
18900 is a sufficient body of user code that expects non-gang
18901 partitioned regions to not execute in gang-redundant mode.
18902 So we (a) don't warn about the non-portability and (b) pick
18903 the minimum permissible dimension size when there is no
18904 partitioned execution. Otherwise we pick the global
18905 default for the dimension, which the user can control. The
18906 same wording and logic applies to num_workers and
18907 vector_length, however the worker- or vector- single
18908 execution doesn't have the same impact as gang-redundant
18909 execution. (If the minimum gang-level partioning is not 1,
18910 the target is probably too confusing.) */
18911 dims
[ix
] = (used
& GOMP_DIM_MASK (ix
)
18912 ? oacc_default_dims
[ix
] : oacc_min_dims
[ix
]);
18918 /* Replace the attribute with new values. */
18920 for (ix
= GOMP_DIM_MAX
; ix
--;)
18922 pos
= tree_cons (purpose
[ix
],
18923 build_int_cst (integer_type_node
, dims
[ix
]),
18926 TREE_PUBLIC (pos
) = 1;
18928 replace_oacc_fn_attrib (fn
, pos
);
18932 /* Create an empty OpenACC loop structure at LOC. */
18935 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18937 oacc_loop
*loop
= XCNEW (oacc_loop
);
18939 loop
->parent
= parent
;
18940 loop
->child
= loop
->sibling
= NULL
;
18944 loop
->sibling
= parent
->child
;
18945 parent
->child
= loop
;
18949 loop
->marker
= NULL
;
18950 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18951 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18952 loop
->routine
= NULL_TREE
;
18954 loop
->mask
= loop
->flags
= loop
->inner
= 0;
18956 loop
->chunk_size
= 0;
18957 loop
->head_end
= NULL
;
18962 /* Create an outermost, dummy OpenACC loop for offloaded function
18966 new_oacc_loop_outer (tree decl
)
18968 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18971 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18972 Link into PARENT loop. Return the new loop. */
18975 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18977 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18979 loop
->marker
= marker
;
18981 /* TODO: This is where device_type flattening would occur for the loop
18984 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18986 tree chunk_size
= integer_zero_node
;
18987 if (loop
->flags
& OLF_GANG_STATIC
)
18988 chunk_size
= gimple_call_arg (marker
, 4);
18989 loop
->chunk_size
= chunk_size
;
18994 /* Create a dummy loop encompassing a call to a openACC routine.
18995 Extract the routine's partitioning requirements. */
18998 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
19000 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
19001 int level
= oacc_fn_attrib_level (attrs
);
19003 gcc_assert (level
>= 0);
19005 loop
->marker
= call
;
19006 loop
->routine
= decl
;
19007 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
19008 ^ (GOMP_DIM_MASK (level
) - 1));
19011 /* Finish off the current OpenACC loop ending at tail marker TAIL.
19012 Return the parent loop. */
19015 finish_oacc_loop (oacc_loop
*loop
)
19017 /* If the loop has been collapsed, don't partition it. */
19019 loop
->mask
= loop
->flags
= 0;
19020 return loop
->parent
;
19023 /* Free all OpenACC loop structures within LOOP (inclusive). */
19026 free_oacc_loop (oacc_loop
*loop
)
19029 free_oacc_loop (loop
->sibling
);
19031 free_oacc_loop (loop
->child
);
19036 /* Dump out the OpenACC loop head or tail beginning at FROM. */
19039 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
19040 const char *title
, int level
)
19042 enum ifn_unique_kind kind
19043 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19045 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
19046 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19048 gimple
*stmt
= gsi_stmt (gsi
);
19050 if (gimple_call_internal_p (stmt
, IFN_UNIQUE
))
19052 enum ifn_unique_kind k
19053 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
19054 (gimple_call_arg (stmt
, 0)));
19056 if (k
== kind
&& stmt
!= from
)
19059 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
19062 while (gsi_end_p (gsi
))
19063 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19067 /* Dump OpenACC loops LOOP, its siblings and its children. */
19070 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
19074 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
19075 loop
->flags
, loop
->mask
,
19076 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
19079 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
19082 fprintf (file
, "%*sRoutine %s:%u:%s\n",
19083 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
19084 DECL_SOURCE_LINE (loop
->routine
),
19085 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
19087 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
19088 if (loop
->heads
[ix
])
19089 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
19090 for (ix
= GOMP_DIM_MAX
; ix
--;)
19091 if (loop
->tails
[ix
])
19092 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
19095 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
19097 dump_oacc_loop (file
, loop
->sibling
, depth
);
19100 void debug_oacc_loop (oacc_loop
*);
19102 /* Dump loops to stderr. */
19104 DEBUG_FUNCTION
void
19105 debug_oacc_loop (oacc_loop
*loop
)
19107 dump_oacc_loop (stderr
, loop
, 0);
19110 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
19111 structures as we go. By construction these loops are properly
19115 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
19120 if (bb
->flags
& BB_VISITED
)
19124 bb
->flags
|= BB_VISITED
;
19126 /* Scan for loop markers. */
19127 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
19130 gimple
*stmt
= gsi_stmt (gsi
);
19132 if (!is_gimple_call (stmt
))
19135 gcall
*call
= as_a
<gcall
*> (stmt
);
19137 /* If this is a routine, make a dummy loop for it. */
19138 if (tree decl
= gimple_call_fndecl (call
))
19139 if (tree attrs
= get_oacc_fn_attrib (decl
))
19141 gcc_assert (!marker
);
19142 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
19145 if (!gimple_call_internal_p (call
))
19148 switch (gimple_call_internal_fn (call
))
19153 case IFN_GOACC_LOOP
:
19154 /* Count the goacc loop abstraction fns, to determine if the
19155 loop was collapsed already. */
19160 enum ifn_unique_kind kind
19161 = (enum ifn_unique_kind
) (TREE_INT_CST_LOW
19162 (gimple_call_arg (call
, 0)));
19163 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
19164 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19166 if (gimple_call_num_args (call
) == 2)
19168 gcc_assert (marker
&& !remaining
);
19170 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19171 loop
= finish_oacc_loop (loop
);
19173 loop
->head_end
= call
;
19177 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
19181 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19182 loop
= new_oacc_loop (loop
, call
);
19185 gcc_assert (count
== remaining
);
19189 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19190 loop
->heads
[marker
] = call
;
19192 loop
->tails
[remaining
] = call
;
19199 if (remaining
|| marker
)
19201 bb
= single_succ (bb
);
19202 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
19206 /* Walk successor blocks. */
19210 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
19211 oacc_loop_discover_walk (loop
, e
->dest
);
19214 /* LOOP is the first sibling. Reverse the order in place and return
19215 the new first sibling. Recurse to child loops. */
19218 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
19220 oacc_loop
*last
= NULL
;
19224 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
19226 oacc_loop
*next
= loop
->sibling
;
19227 loop
->sibling
= last
;
19236 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19237 the current function. */
19240 oacc_loop_discovery ()
19244 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19245 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19247 /* The siblings were constructed in reverse order, reverse them so
19248 that diagnostics come out in an unsurprising order. */
19249 top
= oacc_loop_sibling_nreverse (top
);
19251 /* Reset the visited flags. */
19252 FOR_ALL_BB_FN (bb
, cfun
)
19253 bb
->flags
&= ~BB_VISITED
;
19258 /* Transform the abstract internal function markers starting at FROM
19259 to be for partitioning level LEVEL. Stop when we meet another HEAD
19263 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19265 enum ifn_unique_kind kind
19266 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19267 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19269 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19271 gimple
*stmt
= gsi_stmt (gsi
);
19273 if (gimple_call_internal_p (stmt
, IFN_UNIQUE
))
19275 enum ifn_unique_kind k
19276 = ((enum ifn_unique_kind
)
19277 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19279 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19280 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19281 else if (k
== kind
&& stmt
!= from
)
19284 else if (gimple_call_internal_p (stmt
, IFN_GOACC_REDUCTION
))
19285 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
19288 while (gsi_end_p (gsi
))
19289 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19293 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19294 determined partitioning mask and chunking argument. END_MARKER
19295 points at the end IFN_HEAD_TAIL call intgroducing the loop. IFNS
19296 is the number of IFN_GOACC_LOOP calls for the loop. MASK_ARG is
19297 the replacement partitioning mask and CHUNK_ARG is the replacement
19301 oacc_loop_xform_loop (gcall
*end_marker
, unsigned ifns
,
19302 tree mask_arg
, tree chunk_arg
)
19304 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19306 gcc_checking_assert (ifns
);
19309 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19311 gimple
*stmt
= gsi_stmt (gsi
);
19313 if (!is_gimple_call (stmt
))
19316 gcall
*call
= as_a
<gcall
*> (stmt
);
19318 if (!gimple_call_internal_p (call
))
19321 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19324 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19325 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19331 /* The LOOP_BOUND ifn could be in the single successor
19333 basic_block bb
= single_succ (gsi_bb (gsi
));
19334 gsi
= gsi_start_bb (bb
);
19338 /* Process the discovered OpenACC loops, setting the correct
19339 partitioning level etc. */
19342 oacc_loop_process (oacc_loop
*loop
)
19345 oacc_loop_process (loop
->child
);
19347 if (loop
->mask
&& !loop
->routine
)
19350 unsigned mask
= loop
->mask
;
19351 unsigned dim
= GOMP_DIM_GANG
;
19352 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19353 tree chunk_arg
= loop
->chunk_size
;
19355 oacc_loop_xform_loop (loop
->head_end
, loop
->ifns
, mask_arg
, chunk_arg
);
19357 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& mask
; ix
++)
19359 while (!(GOMP_DIM_MASK (dim
) & mask
))
19362 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19363 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19365 mask
^= GOMP_DIM_MASK (dim
);
19370 oacc_loop_process (loop
->sibling
);
19373 /* Walk the OpenACC loop heirarchy checking and assigning the
19374 programmer-specified partitionings. OUTER_MASK is the partitioning
19375 this loop is contained within. Return mask of partitioning
19376 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
19380 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19382 unsigned this_mask
= loop
->mask
;
19383 unsigned mask_all
= 0;
19386 #ifdef ACCEL_COMPILER
19387 /* When device_type is supported, we want the device compiler to be
19388 noisy, if the loop parameters are device_type-specific. */
19392 if (!loop
->routine
)
19394 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19395 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19397 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19398 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19400 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19403 error_at (loop
->loc
,
19405 ? "%<seq%> overrides other OpenACC loop specifiers"
19406 : "%<auto%> conflicts with other OpenACC loop specifiers");
19408 loop
->flags
&= ~OLF_AUTO
;
19412 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19416 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19417 mask_all
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19420 if (this_mask
& outer_mask
)
19422 const oacc_loop
*outer
;
19423 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19424 if (outer
->mask
& this_mask
)
19431 error_at (loop
->loc
,
19432 "%s uses same OpenACC parallelism as containing loop",
19433 loop
->routine
? "routine call" : "inner loop");
19434 inform (outer
->loc
, "containing loop here");
19437 error_at (loop
->loc
,
19438 "%s uses OpenACC parallelism disallowed by containing routine",
19439 loop
->routine
? "routine call" : "loop");
19442 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19443 "routine %qD declared here", loop
->routine
);
19445 this_mask
&= ~outer_mask
;
19449 unsigned outermost
= least_bit_hwi (this_mask
);
19451 if (outermost
&& outermost
<= outer_mask
)
19455 error_at (loop
->loc
,
19456 "incorrectly nested OpenACC loop parallelism");
19458 const oacc_loop
*outer
;
19459 for (outer
= loop
->parent
;
19460 outer
->flags
&& outer
->flags
< outermost
;
19461 outer
= outer
->parent
)
19463 inform (outer
->loc
, "containing loop here");
19466 this_mask
&= ~outermost
;
19470 loop
->mask
= this_mask
;
19471 mask_all
|= this_mask
;
19475 loop
->inner
= oacc_loop_fixed_partitions (loop
->child
,
19476 outer_mask
| this_mask
);
19477 mask_all
|= loop
->inner
;
19481 mask_all
|= oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
);
19486 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
19487 OUTER_MASK is the partitioning this loop is contained within.
19488 Return the cumulative partitioning used by this loop, siblings and
19492 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19494 bool assign
= (loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
);
19497 #ifdef ACCEL_COMPILER
19498 /* When device_type is supported, we want the device compiler to be
19499 noisy, if the loop parameters are device_type-specific. */
19503 if (assign
&& outer_mask
< GOMP_DIM_MASK (GOMP_DIM_MAX
- 1))
19505 /* Allocate the outermost loop at the outermost available
19507 unsigned this_mask
= outer_mask
+ 1;
19509 if (!(this_mask
& loop
->inner
))
19510 loop
->mask
= this_mask
;
19515 unsigned child_mask
= outer_mask
| loop
->mask
;
19517 if (loop
->mask
|| assign
)
19518 child_mask
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19520 loop
->inner
= oacc_loop_auto_partitions (loop
->child
, child_mask
);
19523 if (assign
&& !loop
->mask
)
19525 /* Allocate the loop at the innermost available level. */
19526 unsigned this_mask
= 0;
19528 /* Determine the outermost partitioning used within this loop. */
19529 this_mask
= loop
->inner
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
19530 this_mask
= least_bit_hwi (this_mask
);
19532 /* Pick the partitioning just inside that one. */
19535 /* And avoid picking one use by an outer loop. */
19536 this_mask
&= ~outer_mask
;
19538 if (!this_mask
&& noisy
)
19539 warning_at (loop
->loc
, 0,
19540 "insufficient partitioning available to parallelize loop");
19542 loop
->mask
= this_mask
;
19545 if (assign
&& dump_file
)
19546 fprintf (dump_file
, "Auto loop %s:%d assigned %d\n",
19547 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
19550 unsigned inner_mask
= 0;
19553 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
19555 inner_mask
|= loop
->inner
| loop
->mask
;
19560 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19561 axes. Return mask of partitioning. */
19564 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
19566 unsigned mask_all
= oacc_loop_fixed_partitions (loop
, outer_mask
);
19568 if (mask_all
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
19570 mask_all
^= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19571 mask_all
|= oacc_loop_auto_partitions (loop
, outer_mask
);
19576 /* Default fork/join early expander. Delete the function calls if
19577 there is no RTL expander. */
19580 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19581 const int *ARG_UNUSED (dims
), bool is_fork
)
19584 return targetm
.have_oacc_fork ();
19586 return targetm
.have_oacc_join ();
19589 /* Default goacc.reduction early expander.
19591 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19592 If RES_PTR is not integer-zerop:
19593 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19594 TEARDOWN - emit '*RES_PTR = VAR'
19596 emit 'LHS = VAR' */
19599 default_goacc_reduction (gcall
*call
)
19601 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19602 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19603 tree lhs
= gimple_call_lhs (call
);
19604 tree var
= gimple_call_arg (call
, 2);
19605 gimple_seq seq
= NULL
;
19607 if (code
== IFN_GOACC_REDUCTION_SETUP
19608 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19610 /* Setup and Teardown need to copy from/to the receiver object,
19611 if there is one. */
19612 tree ref_to_res
= gimple_call_arg (call
, 1);
19614 if (!integer_zerop (ref_to_res
))
19616 tree dst
= build_simple_mem_ref (ref_to_res
);
19619 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19625 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19629 /* Copy VAR to LHS, if there is an LHS. */
19631 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19633 gsi_replace_with_seq (&gsi
, seq
, true);
19636 /* Main entry point for oacc transformations which run on the device
19637 compiler after LTO, so we know what the target device is at this
19638 point (including the host fallback). */
19640 static unsigned int
19641 execute_oacc_device_lower ()
19643 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19646 /* Not an offloaded function. */
19649 /* Parse the default dim argument exactly once. */
19650 if ((const void *)flag_openacc_dims
!= &flag_openacc_dims
)
19652 oacc_parse_default_dims (flag_openacc_dims
);
19653 flag_openacc_dims
= (char *)&flag_openacc_dims
;
19656 /* Discover, partition and process the loops. */
19657 oacc_loop
*loops
= oacc_loop_discovery ();
19658 int fn_level
= oacc_fn_attrib_level (attrs
);
19661 fprintf (dump_file
, oacc_fn_attrib_kernels_p (attrs
)
19662 ? "Function is kernels offload\n"
19663 : fn_level
< 0 ? "Function is parallel offload\n"
19664 : "Function is routine level %d\n", fn_level
);
19666 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
19667 unsigned used_mask
= oacc_loop_partition (loops
, outer_mask
);
19668 int dims
[GOMP_DIM_MAX
];
19670 oacc_validate_dims (current_function_decl
, attrs
, dims
, fn_level
, used_mask
);
19674 const char *comma
= "Compute dimensions [";
19675 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, comma
= ", ")
19676 fprintf (dump_file
, "%s%d", comma
, dims
[ix
]);
19677 fprintf (dump_file
, "]\n");
19680 oacc_loop_process (loops
);
19683 fprintf (dump_file
, "OpenACC loops\n");
19684 dump_oacc_loop (dump_file
, loops
, 0);
19685 fprintf (dump_file
, "\n");
19688 /* Offloaded targets may introduce new basic blocks, which require
19689 dominance information to update SSA. */
19690 calculate_dominance_info (CDI_DOMINATORS
);
19692 /* Now lower internal loop functions to target-specific code
19695 FOR_ALL_BB_FN (bb
, cfun
)
19696 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19698 gimple
*stmt
= gsi_stmt (gsi
);
19699 if (!is_gimple_call (stmt
))
19705 gcall
*call
= as_a
<gcall
*> (stmt
);
19706 if (!gimple_call_internal_p (call
))
19712 /* Rewind to allow rescan. */
19714 bool rescan
= false, remove
= false;
19715 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19721 case IFN_GOACC_LOOP
:
19722 oacc_xform_loop (call
);
19726 case IFN_GOACC_REDUCTION
:
19727 /* Mark the function for SSA renaming. */
19728 mark_virtual_operands_for_renaming (cfun
);
19730 /* If the level is -1, this ended up being an unused
19731 axis. Handle as a default. */
19732 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19733 default_goacc_reduction (call
);
19735 targetm
.goacc
.reduction (call
);
19741 enum ifn_unique_kind kind
19742 = ((enum ifn_unique_kind
)
19743 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19748 gcc_unreachable ();
19750 case IFN_UNIQUE_OACC_FORK
:
19751 case IFN_UNIQUE_OACC_JOIN
:
19752 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19754 else if (!targetm
.goacc
.fork_join
19755 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19759 case IFN_UNIQUE_OACC_HEAD_MARK
:
19760 case IFN_UNIQUE_OACC_TAIL_MARK
:
19768 if (gsi_end_p (gsi
))
19769 /* We rewound past the beginning of the BB. */
19770 gsi
= gsi_start_bb (bb
);
19772 /* Undo the rewind. */
19777 if (gimple_vdef (call
))
19778 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19779 if (gimple_call_lhs (call
))
19781 /* Propagate the data dependency var. */
19782 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19783 gimple_call_arg (call
, 1));
19784 gsi_replace (&gsi
, ass
, false);
19787 gsi_remove (&gsi
, true);
19790 /* If not rescanning, advance over the call. */
19794 free_oacc_loop (loops
);
19799 /* Default launch dimension validator. Force everything to 1. A
19800 backend that wants to provide larger dimensions must override this
19804 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19805 int ARG_UNUSED (fn_level
))
19807 bool changed
= false;
19809 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19821 /* Default dimension bound is unknown on accelerator and 1 on host. */
19824 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19826 #ifdef ACCEL_COMPILER
19835 const pass_data pass_data_oacc_device_lower
=
19837 GIMPLE_PASS
, /* type */
19838 "oaccdevlow", /* name */
19839 OPTGROUP_NONE
, /* optinfo_flags */
19840 TV_NONE
, /* tv_id */
19841 PROP_cfg
, /* properties_required */
19842 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19843 0, /* properties_destroyed */
19844 0, /* todo_flags_start */
19845 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19848 class pass_oacc_device_lower
: public gimple_opt_pass
19851 pass_oacc_device_lower (gcc::context
*ctxt
)
19852 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19855 /* opt_pass methods: */
19856 virtual unsigned int execute (function
*)
19858 bool gate
= flag_openacc
!= 0;
19863 return execute_oacc_device_lower ();
19866 }; // class pass_oacc_device_lower
19868 } // anon namespace
19871 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19873 return new pass_oacc_device_lower (ctxt
);
19876 /* "omp declare target link" handling pass. */
19880 const pass_data pass_data_omp_target_link
=
19882 GIMPLE_PASS
, /* type */
19883 "omptargetlink", /* name */
19884 OPTGROUP_NONE
, /* optinfo_flags */
19885 TV_NONE
, /* tv_id */
19886 PROP_ssa
, /* properties_required */
19887 0, /* properties_provided */
19888 0, /* properties_destroyed */
19889 0, /* todo_flags_start */
19890 TODO_update_ssa
, /* todo_flags_finish */
19893 class pass_omp_target_link
: public gimple_opt_pass
19896 pass_omp_target_link (gcc::context
*ctxt
)
19897 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
19900 /* opt_pass methods: */
19901 virtual bool gate (function
*fun
)
19903 #ifdef ACCEL_COMPILER
19904 tree attrs
= DECL_ATTRIBUTES (fun
->decl
);
19905 return lookup_attribute ("omp declare target", attrs
)
19906 || lookup_attribute ("omp target entrypoint", attrs
);
19913 virtual unsigned execute (function
*);
19916 /* Callback for walk_gimple_stmt used to scan for link var operands. */
19919 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
19923 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
)
19924 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
19926 *walk_subtrees
= 0;
19934 pass_omp_target_link::execute (function
*fun
)
19937 FOR_EACH_BB_FN (bb
, fun
)
19939 gimple_stmt_iterator gsi
;
19940 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
19941 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
19942 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
19948 } // anon namespace
19951 make_pass_omp_target_link (gcc::context
*ctxt
)
19953 return new pass_omp_target_link (ctxt
);
19956 #include "gt-omp-low.h"