1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region
*outer
;
107 /* First child region. */
108 struct omp_region
*inner
;
110 /* Next peer region. */
111 struct omp_region
*next
;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec
<tree
, va_gc
> *ws_args
;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type
;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind
;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered
*ord_stmt
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind
;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v
, n1
, n2
, step
;
204 enum tree_code cond_code
;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop
;
217 bool have_nowait
, have_ordered
, simd_schedule
;
218 unsigned char sched_modifiers
;
219 enum omp_clause_schedule_kind sched_kind
;
220 struct omp_for_data_loop
*loops
;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop
*parent
; /* Containing loop. */
230 oacc_loop
*child
; /* First inner loop. */
232 oacc_loop
*sibling
; /* Next loop within same parent. */
234 location_t loc
; /* Location of the loop start. */
236 gcall
*marker
; /* Initial head marker. */
238 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
239 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
241 tree routine
; /* Pseudo-loop enclosing a routine. */
243 unsigned mask
; /* Partitioning mask. */
244 unsigned inner
; /* Partitioning of inner loops. */
245 unsigned flags
; /* Partitioning flags. */
246 unsigned ifns
; /* Contained loop abstraction functions. */
247 tree chunk_size
; /* Chunk size. */
248 gcall
*head_end
; /* Final marker of head sequence. */
251 /* Flags for an OpenACC loop. */
253 enum oacc_loop_flags
{
254 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
255 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
256 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
257 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
259 /* Explicitly specified loop axes. */
261 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
262 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
263 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
265 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
269 static splay_tree all_contexts
;
270 static int taskreg_nesting_level
;
271 static int target_nesting_level
;
272 static struct omp_region
*root_omp_region
;
273 static bitmap task_shared_vars
;
274 static vec
<omp_context
*> taskreg_contexts
;
275 static bool omp_any_child_fn_dumped
;
277 static void scan_omp (gimple_seq
*, omp_context
*);
278 static tree
scan_omp_1_op (tree
*, int *, void *);
279 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
281 #define WALK_SUBSTMTS \
285 case GIMPLE_EH_FILTER: \
286 case GIMPLE_TRANSACTION: \
287 /* The sub-statements for these should be walked. */ \
288 *handled_ops_p = false; \
291 /* Return true if CTX corresponds to an oacc parallel region. */
294 is_oacc_parallel (omp_context
*ctx
)
296 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
297 return ((outer_type
== GIMPLE_OMP_TARGET
)
298 && (gimple_omp_target_kind (ctx
->stmt
)
299 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
302 /* Return true if CTX corresponds to an oacc kernels region. */
305 is_oacc_kernels (omp_context
*ctx
)
307 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
308 return ((outer_type
== GIMPLE_OMP_TARGET
)
309 && (gimple_omp_target_kind (ctx
->stmt
)
310 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
313 /* If DECL is the artificial dummy VAR_DECL created for non-static
314 data member privatization, return the underlying "this" parameter,
315 otherwise return NULL. */
318 omp_member_access_dummy_var (tree decl
)
321 || !DECL_ARTIFICIAL (decl
)
322 || !DECL_IGNORED_P (decl
)
323 || !DECL_HAS_VALUE_EXPR_P (decl
)
324 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
327 tree v
= DECL_VALUE_EXPR (decl
);
328 if (TREE_CODE (v
) != COMPONENT_REF
)
332 switch (TREE_CODE (v
))
338 case POINTER_PLUS_EXPR
:
339 v
= TREE_OPERAND (v
, 0);
342 if (DECL_CONTEXT (v
) == current_function_decl
343 && DECL_ARTIFICIAL (v
)
344 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
352 /* Helper for unshare_and_remap, called through walk_tree. */
355 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
357 tree
*pair
= (tree
*) data
;
360 *tp
= unshare_expr (pair
[1]);
363 else if (IS_TYPE_OR_DECL_P (*tp
))
368 /* Return unshare_expr (X) with all occurrences of FROM
372 unshare_and_remap (tree x
, tree from
, tree to
)
374 tree pair
[2] = { from
, to
};
375 x
= unshare_expr (x
);
376 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
380 /* Holds offload tables with decls. */
381 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
383 /* Convenience function for calling scan_omp_1_op on tree operands. */
386 scan_omp_op (tree
*tp
, omp_context
*ctx
)
388 struct walk_stmt_info wi
;
390 memset (&wi
, 0, sizeof (wi
));
392 wi
.want_locations
= true;
394 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
397 static void lower_omp (gimple_seq
*, omp_context
*);
398 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
399 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
401 /* Find an OMP clause of type KIND within CLAUSES. */
404 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
406 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
407 if (OMP_CLAUSE_CODE (clauses
) == kind
)
413 /* Return true if CTX is for an omp parallel. */
416 is_parallel_ctx (omp_context
*ctx
)
418 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
422 /* Return true if CTX is for an omp task. */
425 is_task_ctx (omp_context
*ctx
)
427 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
431 /* Return true if CTX is for an omp taskloop. */
434 is_taskloop_ctx (omp_context
*ctx
)
436 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
437 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
441 /* Return true if CTX is for an omp parallel or omp task. */
444 is_taskreg_ctx (omp_context
*ctx
)
446 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
450 /* Return true if REGION is a combined parallel+workshare region. */
453 is_combined_parallel (struct omp_region
*region
)
455 return region
->is_combined_parallel
;
458 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
462 adjust_for_condition (location_t loc
, enum tree_code
*cond_code
, tree
*n2
)
471 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
472 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, 1);
474 *n2
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (*n2
), *n2
,
475 build_int_cst (TREE_TYPE (*n2
), 1));
476 *cond_code
= LT_EXPR
;
479 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
480 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, -1);
482 *n2
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (*n2
), *n2
,
483 build_int_cst (TREE_TYPE (*n2
), 1));
484 *cond_code
= GT_EXPR
;
491 /* Return the looping step from INCR, extracted from the step of a gimple omp
495 get_omp_for_step_from_incr (location_t loc
, tree incr
)
498 switch (TREE_CODE (incr
))
501 step
= TREE_OPERAND (incr
, 1);
503 case POINTER_PLUS_EXPR
:
504 step
= fold_convert (ssizetype
, TREE_OPERAND (incr
, 1));
507 step
= TREE_OPERAND (incr
, 1);
508 step
= fold_build1_loc (loc
, NEGATE_EXPR
, TREE_TYPE (step
), step
);
516 /* Extract the header elements of parallel loop FOR_STMT and store
520 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
521 struct omp_for_data_loop
*loops
)
523 tree t
, var
, *collapse_iter
, *collapse_count
;
524 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
525 struct omp_for_data_loop
*loop
;
527 struct omp_for_data_loop dummy_loop
;
528 location_t loc
= gimple_location (for_stmt
);
529 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
530 bool distribute
= gimple_omp_for_kind (for_stmt
)
531 == GF_OMP_FOR_KIND_DISTRIBUTE
;
532 bool taskloop
= gimple_omp_for_kind (for_stmt
)
533 == GF_OMP_FOR_KIND_TASKLOOP
;
536 fd
->for_stmt
= for_stmt
;
538 if (gimple_omp_for_collapse (for_stmt
) > 1)
541 fd
->loops
= &fd
->loop
;
543 fd
->have_nowait
= distribute
|| simd
;
544 fd
->have_ordered
= false;
547 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
548 fd
->sched_modifiers
= 0;
549 fd
->chunk_size
= NULL_TREE
;
550 fd
->simd_schedule
= false;
551 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
552 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
553 collapse_iter
= NULL
;
554 collapse_count
= NULL
;
556 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
557 switch (OMP_CLAUSE_CODE (t
))
559 case OMP_CLAUSE_NOWAIT
:
560 fd
->have_nowait
= true;
562 case OMP_CLAUSE_ORDERED
:
563 fd
->have_ordered
= true;
564 if (OMP_CLAUSE_ORDERED_EXPR (t
))
565 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
567 case OMP_CLAUSE_SCHEDULE
:
568 gcc_assert (!distribute
&& !taskloop
);
570 = (enum omp_clause_schedule_kind
)
571 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
572 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
573 & ~OMP_CLAUSE_SCHEDULE_MASK
);
574 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
575 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
577 case OMP_CLAUSE_DIST_SCHEDULE
:
578 gcc_assert (distribute
);
579 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
581 case OMP_CLAUSE_COLLAPSE
:
582 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
583 if (fd
->collapse
> 1)
585 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
586 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
592 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
597 collapse_iter
= &iterv
;
598 collapse_count
= &countv
;
601 /* FIXME: for now map schedule(auto) to schedule(static).
602 There should be analysis to determine whether all iterations
603 are approximately the same amount of work (then schedule(static)
604 is best) or if it varies (then schedule(dynamic,N) is better). */
605 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
607 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
608 gcc_assert (fd
->chunk_size
== NULL
);
610 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
612 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
613 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
614 gcc_assert (fd
->chunk_size
== NULL
);
615 else if (fd
->chunk_size
== NULL
)
617 /* We only need to compute a default chunk size for ordered
618 static loops and dynamic loops. */
619 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
621 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
622 ? integer_zero_node
: integer_one_node
;
625 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
626 for (i
= 0; i
< cnt
; i
++)
628 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
630 else if (loops
!= NULL
)
635 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
636 gcc_assert (SSA_VAR_P (loop
->v
));
637 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
638 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
639 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
640 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
642 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
643 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
644 gcc_assert (loop
->cond_code
!= NE_EXPR
645 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKSIMD
646 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
);
647 adjust_for_condition (loc
, &loop
->cond_code
, &loop
->n2
);
649 t
= gimple_omp_for_incr (for_stmt
, i
);
650 gcc_assert (TREE_OPERAND (t
, 0) == var
);
651 loop
->step
= get_omp_for_step_from_incr (loc
, t
);
654 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
655 && !fd
->have_ordered
))
657 if (fd
->collapse
== 1)
658 iter_type
= TREE_TYPE (loop
->v
);
660 || TYPE_PRECISION (iter_type
)
661 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
663 = build_nonstandard_integer_type
664 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
666 else if (iter_type
!= long_long_unsigned_type_node
)
668 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
669 iter_type
= long_long_unsigned_type_node
;
670 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
671 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
672 >= TYPE_PRECISION (iter_type
))
676 if (loop
->cond_code
== LT_EXPR
)
677 n
= fold_build2_loc (loc
,
678 PLUS_EXPR
, TREE_TYPE (loop
->v
),
679 loop
->n2
, loop
->step
);
682 if (TREE_CODE (n
) != INTEGER_CST
683 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
684 iter_type
= long_long_unsigned_type_node
;
686 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
687 > TYPE_PRECISION (iter_type
))
691 if (loop
->cond_code
== LT_EXPR
)
694 n2
= fold_build2_loc (loc
,
695 PLUS_EXPR
, TREE_TYPE (loop
->v
),
696 loop
->n2
, loop
->step
);
700 n1
= fold_build2_loc (loc
,
701 MINUS_EXPR
, TREE_TYPE (loop
->v
),
702 loop
->n2
, loop
->step
);
705 if (TREE_CODE (n1
) != INTEGER_CST
706 || TREE_CODE (n2
) != INTEGER_CST
707 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
708 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
709 iter_type
= long_long_unsigned_type_node
;
713 if (i
>= fd
->collapse
)
716 if (collapse_count
&& *collapse_count
== NULL
)
718 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
719 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
720 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
721 if (t
&& integer_zerop (t
))
722 count
= build_zero_cst (long_long_unsigned_type_node
);
723 else if ((i
== 0 || count
!= NULL_TREE
)
724 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
725 && TREE_CONSTANT (loop
->n1
)
726 && TREE_CONSTANT (loop
->n2
)
727 && TREE_CODE (loop
->step
) == INTEGER_CST
)
729 tree itype
= TREE_TYPE (loop
->v
);
731 if (POINTER_TYPE_P (itype
))
732 itype
= signed_type_for (itype
);
733 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
734 t
= fold_build2_loc (loc
,
736 fold_convert_loc (loc
, itype
, loop
->step
), t
);
737 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
738 fold_convert_loc (loc
, itype
, loop
->n2
));
739 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
740 fold_convert_loc (loc
, itype
, loop
->n1
));
741 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
742 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
743 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
744 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
745 fold_convert_loc (loc
, itype
,
748 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
749 fold_convert_loc (loc
, itype
, loop
->step
));
750 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
751 if (count
!= NULL_TREE
)
752 count
= fold_build2_loc (loc
,
753 MULT_EXPR
, long_long_unsigned_type_node
,
757 if (TREE_CODE (count
) != INTEGER_CST
)
760 else if (count
&& !integer_zerop (count
))
767 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
768 || fd
->have_ordered
))
770 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
771 iter_type
= long_long_unsigned_type_node
;
773 iter_type
= long_integer_type_node
;
775 else if (collapse_iter
&& *collapse_iter
!= NULL
)
776 iter_type
= TREE_TYPE (*collapse_iter
);
777 fd
->iter_type
= iter_type
;
778 if (collapse_iter
&& *collapse_iter
== NULL
)
779 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
780 if (collapse_count
&& *collapse_count
== NULL
)
783 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
785 *collapse_count
= create_tmp_var (iter_type
, ".count");
788 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
790 fd
->loop
.v
= *collapse_iter
;
791 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
792 fd
->loop
.n2
= *collapse_count
;
793 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
794 fd
->loop
.cond_code
= LT_EXPR
;
801 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
802 is the immediate dominator of PAR_ENTRY_BB, return true if there
803 are no data dependencies that would prevent expanding the parallel
804 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
806 When expanding a combined parallel+workshare region, the call to
807 the child function may need additional arguments in the case of
808 GIMPLE_OMP_FOR regions. In some cases, these arguments are
809 computed out of variables passed in from the parent to the child
810 via 'struct .omp_data_s'. For instance:
812 #pragma omp parallel for schedule (guided, i * 4)
817 # BLOCK 2 (PAR_ENTRY_BB)
819 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
821 # BLOCK 3 (WS_ENTRY_BB)
822 .omp_data_i = &.omp_data_o;
823 D.1667 = .omp_data_i->i;
825 #pragma omp for schedule (guided, D.1598)
827 When we outline the parallel region, the call to the child function
828 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
829 that value is computed *after* the call site. So, in principle we
830 cannot do the transformation.
832 To see whether the code in WS_ENTRY_BB blocks the combined
833 parallel+workshare call, we collect all the variables used in the
834 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
835 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
838 FIXME. If we had the SSA form built at this point, we could merely
839 hoist the code in block 3 into block 2 and be done with it. But at
840 this point we don't have dataflow information and though we could
841 hack something up here, it is really not worth the aggravation. */
844 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
846 struct omp_for_data fd
;
847 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
849 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
852 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
854 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
856 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
858 if (fd
.iter_type
!= long_integer_type_node
)
861 /* FIXME. We give up too easily here. If any of these arguments
862 are not constants, they will likely involve variables that have
863 been mapped into fields of .omp_data_s for sharing with the child
864 function. With appropriate data flow, it would be possible to
866 if (!is_gimple_min_invariant (fd
.loop
.n1
)
867 || !is_gimple_min_invariant (fd
.loop
.n2
)
868 || !is_gimple_min_invariant (fd
.loop
.step
)
869 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
876 static int omp_max_vf (void);
878 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
879 presence (SIMD_SCHEDULE). */
882 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
887 int vf
= omp_max_vf ();
891 tree type
= TREE_TYPE (chunk_size
);
892 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
893 build_int_cst (type
, vf
- 1));
894 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
895 build_int_cst (type
, -vf
));
899 /* Collect additional arguments needed to emit a combined
900 parallel+workshare call. WS_STMT is the workshare directive being
903 static vec
<tree
, va_gc
> *
904 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
907 location_t loc
= gimple_location (ws_stmt
);
908 vec
<tree
, va_gc
> *ws_args
;
910 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
912 struct omp_for_data fd
;
915 extract_omp_for_data (for_stmt
, &fd
, NULL
);
919 if (gimple_omp_for_combined_into_p (for_stmt
))
922 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
923 OMP_CLAUSE__LOOPTEMP_
);
925 n1
= OMP_CLAUSE_DECL (innerc
);
926 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
927 OMP_CLAUSE__LOOPTEMP_
);
929 n2
= OMP_CLAUSE_DECL (innerc
);
932 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
934 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
935 ws_args
->quick_push (t
);
937 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
938 ws_args
->quick_push (t
);
940 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
941 ws_args
->quick_push (t
);
945 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
946 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
947 ws_args
->quick_push (t
);
952 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
954 /* Number of sections is equal to the number of edges from the
955 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
956 the exit of the sections region. */
957 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
958 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
959 vec_alloc (ws_args
, 1);
960 ws_args
->quick_push (t
);
968 /* Discover whether REGION is a combined parallel+workshare region. */
971 determine_parallel_type (struct omp_region
*region
)
973 basic_block par_entry_bb
, par_exit_bb
;
974 basic_block ws_entry_bb
, ws_exit_bb
;
976 if (region
== NULL
|| region
->inner
== NULL
977 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
978 || region
->inner
->cont
== NULL
)
981 /* We only support parallel+for and parallel+sections. */
982 if (region
->type
!= GIMPLE_OMP_PARALLEL
983 || (region
->inner
->type
!= GIMPLE_OMP_FOR
984 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
987 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
988 WS_EXIT_BB -> PAR_EXIT_BB. */
989 par_entry_bb
= region
->entry
;
990 par_exit_bb
= region
->exit
;
991 ws_entry_bb
= region
->inner
->entry
;
992 ws_exit_bb
= region
->inner
->exit
;
994 if (single_succ (par_entry_bb
) == ws_entry_bb
995 && single_succ (ws_exit_bb
) == par_exit_bb
996 && workshare_safe_to_combine_p (ws_entry_bb
)
997 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
998 || (last_and_only_stmt (ws_entry_bb
)
999 && last_and_only_stmt (par_exit_bb
))))
1001 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1002 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1004 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1006 /* If this is a combined parallel loop, we need to determine
1007 whether or not to use the combined library calls. There
1008 are two cases where we do not apply the transformation:
1009 static loops and any kind of ordered loop. In the first
1010 case, we already open code the loop so there is no need
1011 to do anything else. In the latter case, the combined
1012 parallel loop call would still need extra synchronization
1013 to implement ordered semantics, so there would not be any
1014 gain in using the combined call. */
1015 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1016 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1018 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1019 == OMP_CLAUSE_SCHEDULE_STATIC
)
1020 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1022 region
->is_combined_parallel
= false;
1023 region
->inner
->is_combined_parallel
= false;
1028 region
->is_combined_parallel
= true;
1029 region
->inner
->is_combined_parallel
= true;
1030 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1035 /* Return true if EXPR is variable sized. */
1038 is_variable_sized (const_tree expr
)
1040 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1043 /* Return true if DECL is a reference type. */
1046 is_reference (tree decl
)
1048 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1051 /* Return the type of a decl. If the decl is reference type,
1052 return its base type. */
1054 get_base_type (tree decl
)
1056 tree type
= TREE_TYPE (decl
);
1057 if (is_reference (decl
))
1058 type
= TREE_TYPE (type
);
1062 /* Lookup variables. The "maybe" form
1063 allows for the variable form to not have been entered, otherwise we
1064 assert that the variable must have been entered. */
1067 lookup_decl (tree var
, omp_context
*ctx
)
1069 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1074 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1076 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1077 return n
? *n
: NULL_TREE
;
1081 lookup_field (tree var
, omp_context
*ctx
)
1084 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1085 return (tree
) n
->value
;
1089 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1092 n
= splay_tree_lookup (ctx
->sfield_map
1093 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1094 return (tree
) n
->value
;
1098 lookup_sfield (tree var
, omp_context
*ctx
)
1100 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1104 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1107 n
= splay_tree_lookup (ctx
->field_map
, key
);
1108 return n
? (tree
) n
->value
: NULL_TREE
;
1112 maybe_lookup_field (tree var
, omp_context
*ctx
)
1114 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1117 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1118 the parallel context if DECL is to be shared. */
1121 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1123 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1126 /* We can only use copy-in/copy-out semantics for shared variables
1127 when we know the value is not accessible from an outer scope. */
1130 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1132 /* ??? Trivially accessible from anywhere. But why would we even
1133 be passing an address in this case? Should we simply assert
1134 this to be false, or should we have a cleanup pass that removes
1135 these from the list of mappings? */
1136 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1139 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1140 without analyzing the expression whether or not its location
1141 is accessible to anyone else. In the case of nested parallel
1142 regions it certainly may be. */
1143 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1146 /* Do not use copy-in/copy-out for variables that have their
1148 if (TREE_ADDRESSABLE (decl
))
1151 /* lower_send_shared_vars only uses copy-in, but not copy-out
1153 if (TREE_READONLY (decl
)
1154 || ((TREE_CODE (decl
) == RESULT_DECL
1155 || TREE_CODE (decl
) == PARM_DECL
)
1156 && DECL_BY_REFERENCE (decl
)))
1159 /* Disallow copy-in/out in nested parallel if
1160 decl is shared in outer parallel, otherwise
1161 each thread could store the shared variable
1162 in its own copy-in location, making the
1163 variable no longer really shared. */
1164 if (shared_ctx
->is_nested
)
1168 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1169 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1176 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1177 c
; c
= OMP_CLAUSE_CHAIN (c
))
1178 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1179 && OMP_CLAUSE_DECL (c
) == decl
)
1183 goto maybe_mark_addressable_and_ret
;
1187 /* For tasks avoid using copy-in/out. As tasks can be
1188 deferred or executed in different thread, when GOMP_task
1189 returns, the task hasn't necessarily terminated. */
1190 if (is_task_ctx (shared_ctx
))
1193 maybe_mark_addressable_and_ret
:
1194 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1195 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1197 /* Taking address of OUTER in lower_send_shared_vars
1198 might need regimplification of everything that uses the
1200 if (!task_shared_vars
)
1201 task_shared_vars
= BITMAP_ALLOC (NULL
);
1202 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1203 TREE_ADDRESSABLE (outer
) = 1;
1212 /* Construct a new automatic decl similar to VAR. */
1215 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1217 tree copy
= copy_var_decl (var
, name
, type
);
1219 DECL_CONTEXT (copy
) = current_function_decl
;
1220 DECL_CHAIN (copy
) = ctx
->block_vars
;
1221 /* If VAR is listed in task_shared_vars, it means it wasn't
1222 originally addressable and is just because task needs to take
1223 it's address. But we don't need to take address of privatizations
1225 if (TREE_ADDRESSABLE (var
)
1227 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1228 TREE_ADDRESSABLE (copy
) = 0;
1229 ctx
->block_vars
= copy
;
1235 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1237 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1240 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1243 omp_build_component_ref (tree obj
, tree field
)
1245 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1246 if (TREE_THIS_VOLATILE (field
))
1247 TREE_THIS_VOLATILE (ret
) |= 1;
1248 if (TREE_READONLY (field
))
1249 TREE_READONLY (ret
) |= 1;
1253 /* Build tree nodes to access the field for VAR on the receiver side. */
1256 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1258 tree x
, field
= lookup_field (var
, ctx
);
1260 /* If the receiver record type was remapped in the child function,
1261 remap the field into the new record type. */
1262 x
= maybe_lookup_field (field
, ctx
);
1266 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1267 TREE_THIS_NOTRAP (x
) = 1;
1268 x
= omp_build_component_ref (x
, field
);
1271 x
= build_simple_mem_ref (x
);
1272 TREE_THIS_NOTRAP (x
) = 1;
1278 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1279 of a parallel, this is a component reference; for workshare constructs
1280 this is some variable. */
1283 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1287 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1289 else if (is_variable_sized (var
))
1291 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1292 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1293 x
= build_simple_mem_ref (x
);
1295 else if (is_taskreg_ctx (ctx
))
1297 bool by_ref
= use_pointer_for_field (var
, NULL
);
1298 x
= build_receiver_ref (var
, by_ref
, ctx
);
1300 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1301 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1303 /* #pragma omp simd isn't a worksharing construct, and can reference even
1304 private vars in its linear etc. clauses. */
1306 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1307 x
= lookup_decl (var
, ctx
->outer
);
1308 else if (ctx
->outer
)
1309 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1313 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1315 gcc_assert (ctx
->outer
);
1317 = splay_tree_lookup (ctx
->outer
->field_map
,
1318 (splay_tree_key
) &DECL_UID (var
));
1321 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1324 x
= lookup_decl (var
, ctx
->outer
);
1328 tree field
= (tree
) n
->value
;
1329 /* If the receiver record type was remapped in the child function,
1330 remap the field into the new record type. */
1331 x
= maybe_lookup_field (field
, ctx
->outer
);
1335 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1336 x
= omp_build_component_ref (x
, field
);
1337 if (use_pointer_for_field (var
, ctx
->outer
))
1338 x
= build_simple_mem_ref (x
);
1341 else if (ctx
->outer
)
1343 omp_context
*outer
= ctx
->outer
;
1344 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
1346 outer
= outer
->outer
;
1348 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
1350 x
= lookup_decl (var
, outer
);
1352 else if (is_reference (var
))
1353 /* This can happen with orphaned constructs. If var is reference, it is
1354 possible it is shared and as such valid. */
1356 else if (omp_member_access_dummy_var (var
))
1363 tree t
= omp_member_access_dummy_var (var
);
1366 x
= DECL_VALUE_EXPR (var
);
1367 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1369 x
= unshare_and_remap (x
, t
, o
);
1371 x
= unshare_expr (x
);
1375 if (is_reference (var
))
1376 x
= build_simple_mem_ref (x
);
1381 /* Build tree nodes to access the field for VAR on the sender side. */
1384 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1386 tree field
= lookup_sfield (key
, ctx
);
1387 return omp_build_component_ref (ctx
->sender_decl
, field
);
1391 build_sender_ref (tree var
, omp_context
*ctx
)
1393 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1396 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1397 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1400 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
,
1401 bool base_pointers_restrict
= false)
1403 tree field
, type
, sfield
= NULL_TREE
;
1404 splay_tree_key key
= (splay_tree_key
) var
;
1406 if ((mask
& 8) != 0)
1408 key
= (splay_tree_key
) &DECL_UID (var
);
1409 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1411 gcc_assert ((mask
& 1) == 0
1412 || !splay_tree_lookup (ctx
->field_map
, key
));
1413 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1414 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1415 gcc_assert ((mask
& 3) == 3
1416 || !is_gimple_omp_oacc (ctx
->stmt
));
1418 type
= TREE_TYPE (var
);
1419 /* Prevent redeclaring the var in the split-off function with a restrict
1420 pointer type. Note that we only clear type itself, restrict qualifiers in
1421 the pointed-to type will be ignored by points-to analysis. */
1422 if (POINTER_TYPE_P (type
)
1423 && TYPE_RESTRICT (type
))
1424 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
1428 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1429 type
= build_pointer_type (build_pointer_type (type
));
1433 type
= build_pointer_type (type
);
1434 if (base_pointers_restrict
)
1435 type
= build_qualified_type (type
, TYPE_QUAL_RESTRICT
);
1437 else if ((mask
& 3) == 1 && is_reference (var
))
1438 type
= TREE_TYPE (type
);
1440 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1441 FIELD_DECL
, DECL_NAME (var
), type
);
1443 /* Remember what variable this field was created for. This does have a
1444 side effect of making dwarf2out ignore this member, so for helpful
1445 debugging we clear it later in delete_omp_context. */
1446 DECL_ABSTRACT_ORIGIN (field
) = var
;
1447 if (type
== TREE_TYPE (var
))
1449 SET_DECL_ALIGN (field
, DECL_ALIGN (var
));
1450 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1451 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1454 SET_DECL_ALIGN (field
, TYPE_ALIGN (type
));
1456 if ((mask
& 3) == 3)
1458 insert_field_into_struct (ctx
->record_type
, field
);
1459 if (ctx
->srecord_type
)
1461 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1462 FIELD_DECL
, DECL_NAME (var
), type
);
1463 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1464 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
1465 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1466 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1467 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1472 if (ctx
->srecord_type
== NULL_TREE
)
1476 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1477 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1478 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1480 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1481 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1482 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1483 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1484 splay_tree_insert (ctx
->sfield_map
,
1485 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1486 (splay_tree_value
) sfield
);
1490 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1491 : ctx
->srecord_type
, field
);
1495 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1496 if ((mask
& 2) && ctx
->sfield_map
)
1497 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1501 install_var_local (tree var
, omp_context
*ctx
)
1503 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1504 insert_decl_map (&ctx
->cb
, var
, new_var
);
1508 /* Adjust the replacement for DECL in CTX for the new context. This means
1509 copying the DECL_VALUE_EXPR, and fixing up the type. */
1512 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1514 tree new_decl
, size
;
1516 new_decl
= lookup_decl (decl
, ctx
);
1518 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1520 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1521 && DECL_HAS_VALUE_EXPR_P (decl
))
1523 tree ve
= DECL_VALUE_EXPR (decl
);
1524 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1525 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1526 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1529 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1531 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1532 if (size
== error_mark_node
)
1533 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1534 DECL_SIZE (new_decl
) = size
;
1536 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1537 if (size
== error_mark_node
)
1538 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1539 DECL_SIZE_UNIT (new_decl
) = size
;
1543 /* The callback for remap_decl. Search all containing contexts for a
1544 mapping of the variable; this avoids having to duplicate the splay
1545 tree ahead of time. We know a mapping doesn't already exist in the
1546 given context. Create new mappings to implement default semantics. */
1549 omp_copy_decl (tree var
, copy_body_data
*cb
)
1551 omp_context
*ctx
= (omp_context
*) cb
;
1554 if (TREE_CODE (var
) == LABEL_DECL
)
1556 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1557 DECL_CONTEXT (new_var
) = current_function_decl
;
1558 insert_decl_map (&ctx
->cb
, var
, new_var
);
1562 while (!is_taskreg_ctx (ctx
))
1567 new_var
= maybe_lookup_decl (var
, ctx
);
1572 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1575 return error_mark_node
;
1579 /* Debugging dumps for parallel regions. */
1580 void dump_omp_region (FILE *, struct omp_region
*, int);
1581 void debug_omp_region (struct omp_region
*);
1582 void debug_all_omp_regions (void);
1584 /* Dump the parallel region tree rooted at REGION. */
1587 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1589 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1590 gimple_code_name
[region
->type
]);
1593 dump_omp_region (file
, region
->inner
, indent
+ 4);
1597 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1598 region
->cont
->index
);
1602 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1603 region
->exit
->index
);
1605 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1608 dump_omp_region (file
, region
->next
, indent
);
1612 debug_omp_region (struct omp_region
*region
)
1614 dump_omp_region (stderr
, region
, 0);
1618 debug_all_omp_regions (void)
1620 dump_omp_region (stderr
, root_omp_region
, 0);
1624 /* Create a new parallel region starting at STMT inside region PARENT. */
1626 static struct omp_region
*
1627 new_omp_region (basic_block bb
, enum gimple_code type
,
1628 struct omp_region
*parent
)
1630 struct omp_region
*region
= XCNEW (struct omp_region
);
1632 region
->outer
= parent
;
1634 region
->type
= type
;
1638 /* This is a nested region. Add it to the list of inner
1639 regions in PARENT. */
1640 region
->next
= parent
->inner
;
1641 parent
->inner
= region
;
1645 /* This is a toplevel region. Add it to the list of toplevel
1646 regions in ROOT_OMP_REGION. */
1647 region
->next
= root_omp_region
;
1648 root_omp_region
= region
;
1654 /* Release the memory associated with the region tree rooted at REGION. */
1657 free_omp_region_1 (struct omp_region
*region
)
1659 struct omp_region
*i
, *n
;
1661 for (i
= region
->inner
; i
; i
= n
)
1664 free_omp_region_1 (i
);
1670 /* Release the memory for the entire omp region tree. */
1673 free_omp_regions (void)
1675 struct omp_region
*r
, *n
;
1676 for (r
= root_omp_region
; r
; r
= n
)
1679 free_omp_region_1 (r
);
1681 root_omp_region
= NULL
;
1685 /* Create a new context, with OUTER_CTX being the surrounding context. */
1687 static omp_context
*
1688 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1690 omp_context
*ctx
= XCNEW (omp_context
);
1692 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1693 (splay_tree_value
) ctx
);
1698 ctx
->outer
= outer_ctx
;
1699 ctx
->cb
= outer_ctx
->cb
;
1700 ctx
->cb
.block
= NULL
;
1701 ctx
->depth
= outer_ctx
->depth
+ 1;
1705 ctx
->cb
.src_fn
= current_function_decl
;
1706 ctx
->cb
.dst_fn
= current_function_decl
;
1707 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1708 gcc_checking_assert (ctx
->cb
.src_node
);
1709 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1710 ctx
->cb
.src_cfun
= cfun
;
1711 ctx
->cb
.copy_decl
= omp_copy_decl
;
1712 ctx
->cb
.eh_lp_nr
= 0;
1713 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1717 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1722 static gimple_seq
maybe_catch_exception (gimple_seq
);
1724 /* Finalize task copyfn. */
1727 finalize_task_copyfn (gomp_task
*task_stmt
)
1729 struct function
*child_cfun
;
1731 gimple_seq seq
= NULL
, new_seq
;
1734 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1735 if (child_fn
== NULL_TREE
)
1738 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1739 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1741 push_cfun (child_cfun
);
1742 bind
= gimplify_body (child_fn
, false);
1743 gimple_seq_add_stmt (&seq
, bind
);
1744 new_seq
= maybe_catch_exception (seq
);
1747 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1749 gimple_seq_add_stmt (&seq
, bind
);
1751 gimple_set_body (child_fn
, seq
);
1754 /* Inform the callgraph about the new function. */
1755 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1756 node
->parallelized_function
= 1;
1757 cgraph_node::add_new_function (child_fn
, false);
1760 /* Destroy a omp_context data structures. Called through the splay tree
1761 value delete callback. */
1764 delete_omp_context (splay_tree_value value
)
1766 omp_context
*ctx
= (omp_context
*) value
;
1768 delete ctx
->cb
.decl_map
;
1771 splay_tree_delete (ctx
->field_map
);
1772 if (ctx
->sfield_map
)
1773 splay_tree_delete (ctx
->sfield_map
);
1775 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1776 it produces corrupt debug information. */
1777 if (ctx
->record_type
)
1780 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1781 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1783 if (ctx
->srecord_type
)
1786 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1787 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1790 if (is_task_ctx (ctx
))
1791 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1796 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1800 fixup_child_record_type (omp_context
*ctx
)
1802 tree f
, type
= ctx
->record_type
;
1804 if (!ctx
->receiver_decl
)
1806 /* ??? It isn't sufficient to just call remap_type here, because
1807 variably_modified_type_p doesn't work the way we expect for
1808 record types. Testing each field for whether it needs remapping
1809 and creating a new record by hand works, however. */
1810 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1811 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1815 tree name
, new_fields
= NULL
;
1817 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1818 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1819 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1820 TYPE_DECL
, name
, type
);
1821 TYPE_NAME (type
) = name
;
1823 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1825 tree new_f
= copy_node (f
);
1826 DECL_CONTEXT (new_f
) = type
;
1827 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1828 DECL_CHAIN (new_f
) = new_fields
;
1829 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1830 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1832 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1836 /* Arrange to be able to look up the receiver field
1837 given the sender field. */
1838 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1839 (splay_tree_value
) new_f
);
1841 TYPE_FIELDS (type
) = nreverse (new_fields
);
1845 /* In a target region we never modify any of the pointers in *.omp_data_i,
1846 so attempt to help the optimizers. */
1847 if (is_gimple_omp_offloaded (ctx
->stmt
))
1848 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1850 TREE_TYPE (ctx
->receiver_decl
)
1851 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1854 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1855 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1859 scan_sharing_clauses (tree clauses
, omp_context
*ctx
,
1860 bool base_pointers_restrict
= false)
1863 bool scan_array_reductions
= false;
1865 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1869 switch (OMP_CLAUSE_CODE (c
))
1871 case OMP_CLAUSE_PRIVATE
:
1872 decl
= OMP_CLAUSE_DECL (c
);
1873 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1875 else if (!is_variable_sized (decl
))
1876 install_var_local (decl
, ctx
);
1879 case OMP_CLAUSE_SHARED
:
1880 decl
= OMP_CLAUSE_DECL (c
);
1881 /* Ignore shared directives in teams construct. */
1882 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1884 /* Global variables don't need to be copied,
1885 the receiver side will use them directly. */
1886 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1887 if (is_global_var (odecl
))
1889 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1892 gcc_assert (is_taskreg_ctx (ctx
));
1893 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1894 || !is_variable_sized (decl
));
1895 /* Global variables don't need to be copied,
1896 the receiver side will use them directly. */
1897 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1899 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1901 use_pointer_for_field (decl
, ctx
);
1904 by_ref
= use_pointer_for_field (decl
, NULL
);
1905 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1906 || TREE_ADDRESSABLE (decl
)
1908 || is_reference (decl
))
1910 by_ref
= use_pointer_for_field (decl
, ctx
);
1911 install_var_field (decl
, by_ref
, 3, ctx
);
1912 install_var_local (decl
, ctx
);
1915 /* We don't need to copy const scalar vars back. */
1916 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1919 case OMP_CLAUSE_REDUCTION
:
1920 decl
= OMP_CLAUSE_DECL (c
);
1921 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1922 && TREE_CODE (decl
) == MEM_REF
)
1924 tree t
= TREE_OPERAND (decl
, 0);
1925 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1926 t
= TREE_OPERAND (t
, 0);
1927 if (TREE_CODE (t
) == INDIRECT_REF
1928 || TREE_CODE (t
) == ADDR_EXPR
)
1929 t
= TREE_OPERAND (t
, 0);
1930 install_var_local (t
, ctx
);
1931 if (is_taskreg_ctx (ctx
)
1932 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1933 && !is_variable_sized (t
))
1935 by_ref
= use_pointer_for_field (t
, ctx
);
1936 install_var_field (t
, by_ref
, 3, ctx
);
1942 case OMP_CLAUSE_LASTPRIVATE
:
1943 /* Let the corresponding firstprivate clause create
1945 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1949 case OMP_CLAUSE_FIRSTPRIVATE
:
1950 case OMP_CLAUSE_LINEAR
:
1951 decl
= OMP_CLAUSE_DECL (c
);
1953 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1954 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1955 && is_gimple_omp_offloaded (ctx
->stmt
))
1957 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1958 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1959 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1960 install_var_field (decl
, true, 3, ctx
);
1962 install_var_field (decl
, false, 3, ctx
);
1964 if (is_variable_sized (decl
))
1966 if (is_task_ctx (ctx
))
1967 install_var_field (decl
, false, 1, ctx
);
1970 else if (is_taskreg_ctx (ctx
))
1973 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1974 by_ref
= use_pointer_for_field (decl
, NULL
);
1976 if (is_task_ctx (ctx
)
1977 && (global
|| by_ref
|| is_reference (decl
)))
1979 install_var_field (decl
, false, 1, ctx
);
1981 install_var_field (decl
, by_ref
, 2, ctx
);
1984 install_var_field (decl
, by_ref
, 3, ctx
);
1986 install_var_local (decl
, ctx
);
1989 case OMP_CLAUSE_USE_DEVICE_PTR
:
1990 decl
= OMP_CLAUSE_DECL (c
);
1991 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1992 install_var_field (decl
, true, 3, ctx
);
1994 install_var_field (decl
, false, 3, ctx
);
1995 if (DECL_SIZE (decl
)
1996 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1998 tree decl2
= DECL_VALUE_EXPR (decl
);
1999 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2000 decl2
= TREE_OPERAND (decl2
, 0);
2001 gcc_assert (DECL_P (decl2
));
2002 install_var_local (decl2
, ctx
);
2004 install_var_local (decl
, ctx
);
2007 case OMP_CLAUSE_IS_DEVICE_PTR
:
2008 decl
= OMP_CLAUSE_DECL (c
);
2011 case OMP_CLAUSE__LOOPTEMP_
:
2012 gcc_assert (is_taskreg_ctx (ctx
));
2013 decl
= OMP_CLAUSE_DECL (c
);
2014 install_var_field (decl
, false, 3, ctx
);
2015 install_var_local (decl
, ctx
);
2018 case OMP_CLAUSE_COPYPRIVATE
:
2019 case OMP_CLAUSE_COPYIN
:
2020 decl
= OMP_CLAUSE_DECL (c
);
2021 by_ref
= use_pointer_for_field (decl
, NULL
);
2022 install_var_field (decl
, by_ref
, 3, ctx
);
2025 case OMP_CLAUSE_DEFAULT
:
2026 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2029 case OMP_CLAUSE_FINAL
:
2031 case OMP_CLAUSE_NUM_THREADS
:
2032 case OMP_CLAUSE_NUM_TEAMS
:
2033 case OMP_CLAUSE_THREAD_LIMIT
:
2034 case OMP_CLAUSE_DEVICE
:
2035 case OMP_CLAUSE_SCHEDULE
:
2036 case OMP_CLAUSE_DIST_SCHEDULE
:
2037 case OMP_CLAUSE_DEPEND
:
2038 case OMP_CLAUSE_PRIORITY
:
2039 case OMP_CLAUSE_GRAINSIZE
:
2040 case OMP_CLAUSE_NUM_TASKS
:
2041 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2042 case OMP_CLAUSE_NUM_GANGS
:
2043 case OMP_CLAUSE_NUM_WORKERS
:
2044 case OMP_CLAUSE_VECTOR_LENGTH
:
2046 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2050 case OMP_CLAUSE_FROM
:
2051 case OMP_CLAUSE_MAP
:
2053 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2054 decl
= OMP_CLAUSE_DECL (c
);
2055 /* Global variables with "omp declare target" attribute
2056 don't need to be copied, the receiver side will use them
2057 directly. However, global variables with "omp declare target link"
2058 attribute need to be copied. */
2059 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2061 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2062 && (OMP_CLAUSE_MAP_KIND (c
)
2063 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2064 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2065 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2066 && varpool_node::get_create (decl
)->offloadable
2067 && !lookup_attribute ("omp declare target link",
2068 DECL_ATTRIBUTES (decl
)))
2070 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2071 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2073 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2074 not offloaded; there is nothing to map for those. */
2075 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2076 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2077 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2080 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2081 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2082 || (OMP_CLAUSE_MAP_KIND (c
)
2083 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2085 if (TREE_CODE (decl
) == COMPONENT_REF
2086 || (TREE_CODE (decl
) == INDIRECT_REF
2087 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2088 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2089 == REFERENCE_TYPE
)))
2091 if (DECL_SIZE (decl
)
2092 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2094 tree decl2
= DECL_VALUE_EXPR (decl
);
2095 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2096 decl2
= TREE_OPERAND (decl2
, 0);
2097 gcc_assert (DECL_P (decl2
));
2098 install_var_local (decl2
, ctx
);
2100 install_var_local (decl
, ctx
);
2105 if (DECL_SIZE (decl
)
2106 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2108 tree decl2
= DECL_VALUE_EXPR (decl
);
2109 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2110 decl2
= TREE_OPERAND (decl2
, 0);
2111 gcc_assert (DECL_P (decl2
));
2112 install_var_field (decl2
, true, 3, ctx
);
2113 install_var_local (decl2
, ctx
);
2114 install_var_local (decl
, ctx
);
2118 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2119 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2120 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2121 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2122 install_var_field (decl
, true, 7, ctx
);
2124 install_var_field (decl
, true, 3, ctx
,
2125 base_pointers_restrict
);
2126 if (is_gimple_omp_offloaded (ctx
->stmt
)
2127 && !OMP_CLAUSE_MAP_IN_REDUCTION (c
))
2128 install_var_local (decl
, ctx
);
2133 tree base
= get_base_address (decl
);
2134 tree nc
= OMP_CLAUSE_CHAIN (c
);
2137 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2138 && OMP_CLAUSE_DECL (nc
) == base
2139 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2140 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2142 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2143 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2149 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2150 decl
= OMP_CLAUSE_DECL (c
);
2152 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2153 (splay_tree_key
) decl
));
2155 = build_decl (OMP_CLAUSE_LOCATION (c
),
2156 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2157 SET_DECL_ALIGN (field
, TYPE_ALIGN (ptr_type_node
));
2158 insert_field_into_struct (ctx
->record_type
, field
);
2159 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2160 (splay_tree_value
) field
);
2165 case OMP_CLAUSE__GRIDDIM_
:
2168 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
2169 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
2173 case OMP_CLAUSE_NOWAIT
:
2174 case OMP_CLAUSE_ORDERED
:
2175 case OMP_CLAUSE_COLLAPSE
:
2176 case OMP_CLAUSE_UNTIED
:
2177 case OMP_CLAUSE_MERGEABLE
:
2178 case OMP_CLAUSE_PROC_BIND
:
2179 case OMP_CLAUSE_SAFELEN
:
2180 case OMP_CLAUSE_SIMDLEN
:
2181 case OMP_CLAUSE_THREADS
:
2182 case OMP_CLAUSE_SIMD
:
2183 case OMP_CLAUSE_NOGROUP
:
2184 case OMP_CLAUSE_DEFAULTMAP
:
2185 case OMP_CLAUSE_ASYNC
:
2186 case OMP_CLAUSE_WAIT
:
2187 case OMP_CLAUSE_GANG
:
2188 case OMP_CLAUSE_WORKER
:
2189 case OMP_CLAUSE_VECTOR
:
2190 case OMP_CLAUSE_INDEPENDENT
:
2191 case OMP_CLAUSE_AUTO
:
2192 case OMP_CLAUSE_SEQ
:
2195 case OMP_CLAUSE_ALIGNED
:
2196 decl
= OMP_CLAUSE_DECL (c
);
2197 if (is_global_var (decl
)
2198 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2199 install_var_local (decl
, ctx
);
2202 case OMP_CLAUSE_TILE
:
2203 case OMP_CLAUSE__CACHE_
:
2209 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2211 switch (OMP_CLAUSE_CODE (c
))
2213 case OMP_CLAUSE_LASTPRIVATE
:
2214 /* Let the corresponding firstprivate clause create
2216 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2217 scan_array_reductions
= true;
2218 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2222 case OMP_CLAUSE_FIRSTPRIVATE
:
2223 case OMP_CLAUSE_PRIVATE
:
2224 case OMP_CLAUSE_LINEAR
:
2225 case OMP_CLAUSE_IS_DEVICE_PTR
:
2226 decl
= OMP_CLAUSE_DECL (c
);
2227 if (is_variable_sized (decl
))
2229 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2230 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2231 && is_gimple_omp_offloaded (ctx
->stmt
))
2233 tree decl2
= DECL_VALUE_EXPR (decl
);
2234 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2235 decl2
= TREE_OPERAND (decl2
, 0);
2236 gcc_assert (DECL_P (decl2
));
2237 install_var_local (decl2
, ctx
);
2238 fixup_remapped_decl (decl2
, ctx
, false);
2240 install_var_local (decl
, ctx
);
2242 fixup_remapped_decl (decl
, ctx
,
2243 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2244 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2245 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2246 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2247 scan_array_reductions
= true;
2250 case OMP_CLAUSE_REDUCTION
:
2251 decl
= OMP_CLAUSE_DECL (c
);
2252 if (TREE_CODE (decl
) != MEM_REF
)
2254 if (is_variable_sized (decl
))
2255 install_var_local (decl
, ctx
);
2256 fixup_remapped_decl (decl
, ctx
, false);
2258 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2259 scan_array_reductions
= true;
2262 case OMP_CLAUSE_SHARED
:
2263 /* Ignore shared directives in teams construct. */
2264 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2266 decl
= OMP_CLAUSE_DECL (c
);
2267 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2269 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2271 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2274 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2275 install_var_field (decl
, by_ref
, 11, ctx
);
2278 fixup_remapped_decl (decl
, ctx
, false);
2281 case OMP_CLAUSE_MAP
:
2282 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2284 decl
= OMP_CLAUSE_DECL (c
);
2286 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2287 && (OMP_CLAUSE_MAP_KIND (c
)
2288 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2289 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2290 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2291 && varpool_node::get_create (decl
)->offloadable
)
2295 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2296 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2297 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2298 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2300 tree new_decl
= lookup_decl (decl
, ctx
);
2301 TREE_TYPE (new_decl
)
2302 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2304 else if (DECL_SIZE (decl
)
2305 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2307 tree decl2
= DECL_VALUE_EXPR (decl
);
2308 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2309 decl2
= TREE_OPERAND (decl2
, 0);
2310 gcc_assert (DECL_P (decl2
));
2311 fixup_remapped_decl (decl2
, ctx
, false);
2312 fixup_remapped_decl (decl
, ctx
, true);
2315 fixup_remapped_decl (decl
, ctx
, false);
2319 case OMP_CLAUSE_COPYPRIVATE
:
2320 case OMP_CLAUSE_COPYIN
:
2321 case OMP_CLAUSE_DEFAULT
:
2323 case OMP_CLAUSE_NUM_THREADS
:
2324 case OMP_CLAUSE_NUM_TEAMS
:
2325 case OMP_CLAUSE_THREAD_LIMIT
:
2326 case OMP_CLAUSE_DEVICE
:
2327 case OMP_CLAUSE_SCHEDULE
:
2328 case OMP_CLAUSE_DIST_SCHEDULE
:
2329 case OMP_CLAUSE_NOWAIT
:
2330 case OMP_CLAUSE_ORDERED
:
2331 case OMP_CLAUSE_COLLAPSE
:
2332 case OMP_CLAUSE_UNTIED
:
2333 case OMP_CLAUSE_FINAL
:
2334 case OMP_CLAUSE_MERGEABLE
:
2335 case OMP_CLAUSE_PROC_BIND
:
2336 case OMP_CLAUSE_SAFELEN
:
2337 case OMP_CLAUSE_SIMDLEN
:
2338 case OMP_CLAUSE_ALIGNED
:
2339 case OMP_CLAUSE_DEPEND
:
2340 case OMP_CLAUSE__LOOPTEMP_
:
2342 case OMP_CLAUSE_FROM
:
2343 case OMP_CLAUSE_PRIORITY
:
2344 case OMP_CLAUSE_GRAINSIZE
:
2345 case OMP_CLAUSE_NUM_TASKS
:
2346 case OMP_CLAUSE_THREADS
:
2347 case OMP_CLAUSE_SIMD
:
2348 case OMP_CLAUSE_NOGROUP
:
2349 case OMP_CLAUSE_DEFAULTMAP
:
2350 case OMP_CLAUSE_USE_DEVICE_PTR
:
2351 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2352 case OMP_CLAUSE_ASYNC
:
2353 case OMP_CLAUSE_WAIT
:
2354 case OMP_CLAUSE_NUM_GANGS
:
2355 case OMP_CLAUSE_NUM_WORKERS
:
2356 case OMP_CLAUSE_VECTOR_LENGTH
:
2357 case OMP_CLAUSE_GANG
:
2358 case OMP_CLAUSE_WORKER
:
2359 case OMP_CLAUSE_VECTOR
:
2360 case OMP_CLAUSE_INDEPENDENT
:
2361 case OMP_CLAUSE_AUTO
:
2362 case OMP_CLAUSE_SEQ
:
2363 case OMP_CLAUSE__GRIDDIM_
:
2366 case OMP_CLAUSE_TILE
:
2367 case OMP_CLAUSE__CACHE_
:
2373 gcc_checking_assert (!scan_array_reductions
2374 || !is_gimple_omp_oacc (ctx
->stmt
));
2375 if (scan_array_reductions
)
2377 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2378 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2379 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2381 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2382 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2384 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2385 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2386 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2387 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2388 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2389 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2393 /* Create a new name for omp child function. Returns an identifier. If
2394 IS_CILK_FOR is true then the suffix for the child function is
2398 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2401 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2402 return clone_function_name (current_function_decl
,
2403 task_copy
? "_omp_cpyfn" : "_omp_fn");
2406 /* Returns the type of the induction variable for the child function for
2407 _Cilk_for and the types for _high and _low variables based on TYPE. */
2410 cilk_for_check_loop_diff_type (tree type
)
2412 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2414 if (TYPE_UNSIGNED (type
))
2415 return uint32_type_node
;
2417 return integer_type_node
;
2421 if (TYPE_UNSIGNED (type
))
2422 return uint64_type_node
;
2424 return long_long_integer_type_node
;
2428 /* Build a decl for the omp child function. It'll not contain a body
2429 yet, just the bare decl. */
2432 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2434 tree decl
, type
, name
, t
;
2437 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2438 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2439 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2440 tree cilk_var_type
= NULL_TREE
;
2442 name
= create_omp_child_function_name (task_copy
,
2443 cilk_for_count
!= NULL_TREE
);
2445 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2446 ptr_type_node
, NULL_TREE
);
2447 else if (cilk_for_count
)
2449 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2450 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2451 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2452 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2455 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2457 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2459 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2462 ctx
->cb
.dst_fn
= decl
;
2464 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2466 TREE_STATIC (decl
) = 1;
2467 TREE_USED (decl
) = 1;
2468 DECL_ARTIFICIAL (decl
) = 1;
2469 DECL_IGNORED_P (decl
) = 0;
2470 TREE_PUBLIC (decl
) = 0;
2471 DECL_UNINLINABLE (decl
) = 1;
2472 DECL_EXTERNAL (decl
) = 0;
2473 DECL_CONTEXT (decl
) = NULL_TREE
;
2474 DECL_INITIAL (decl
) = make_node (BLOCK
);
2475 if (cgraph_node::get (current_function_decl
)->offloadable
)
2476 cgraph_node::get_create (decl
)->offloadable
= 1;
2480 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2481 if (is_gimple_omp_offloaded (octx
->stmt
))
2483 cgraph_node::get_create (decl
)->offloadable
= 1;
2484 if (ENABLE_OFFLOADING
)
2485 g
->have_offload
= true;
2491 if (cgraph_node::get_create (decl
)->offloadable
2492 && !lookup_attribute ("omp declare target",
2493 DECL_ATTRIBUTES (current_function_decl
)))
2494 DECL_ATTRIBUTES (decl
)
2495 = tree_cons (get_identifier ("omp target entrypoint"),
2496 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2498 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2499 RESULT_DECL
, NULL_TREE
, void_type_node
);
2500 DECL_ARTIFICIAL (t
) = 1;
2501 DECL_IGNORED_P (t
) = 1;
2502 DECL_CONTEXT (t
) = decl
;
2503 DECL_RESULT (decl
) = t
;
2505 /* _Cilk_for's child function requires two extra parameters called
2506 __low and __high that are set the by Cilk runtime when it calls this
2510 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2511 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2512 DECL_ARTIFICIAL (t
) = 1;
2513 DECL_NAMELESS (t
) = 1;
2514 DECL_ARG_TYPE (t
) = ptr_type_node
;
2515 DECL_CONTEXT (t
) = current_function_decl
;
2517 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2518 DECL_ARGUMENTS (decl
) = t
;
2520 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2521 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2522 DECL_ARTIFICIAL (t
) = 1;
2523 DECL_NAMELESS (t
) = 1;
2524 DECL_ARG_TYPE (t
) = ptr_type_node
;
2525 DECL_CONTEXT (t
) = current_function_decl
;
2527 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2528 DECL_ARGUMENTS (decl
) = t
;
2531 tree data_name
= get_identifier (".omp_data_i");
2532 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2534 DECL_ARTIFICIAL (t
) = 1;
2535 DECL_NAMELESS (t
) = 1;
2536 DECL_ARG_TYPE (t
) = ptr_type_node
;
2537 DECL_CONTEXT (t
) = current_function_decl
;
2539 TREE_READONLY (t
) = 1;
2541 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2542 DECL_ARGUMENTS (decl
) = t
;
2544 ctx
->receiver_decl
= t
;
2547 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2548 PARM_DECL
, get_identifier (".omp_data_o"),
2550 DECL_ARTIFICIAL (t
) = 1;
2551 DECL_NAMELESS (t
) = 1;
2552 DECL_ARG_TYPE (t
) = ptr_type_node
;
2553 DECL_CONTEXT (t
) = current_function_decl
;
2555 TREE_ADDRESSABLE (t
) = 1;
2556 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2557 DECL_ARGUMENTS (decl
) = t
;
2560 /* Allocate memory for the function structure. The call to
2561 allocate_struct_function clobbers CFUN, so we need to restore
2563 push_struct_function (decl
);
2564 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2565 init_tree_ssa (cfun
);
2569 /* Callback for walk_gimple_seq. Check if combined parallel
2570 contains gimple_omp_for_combined_into_p OMP_FOR. */
2573 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2574 bool *handled_ops_p
,
2575 struct walk_stmt_info
*wi
)
2577 gimple
*stmt
= gsi_stmt (*gsi_p
);
2579 *handled_ops_p
= true;
2580 switch (gimple_code (stmt
))
2584 case GIMPLE_OMP_FOR
:
2585 if (gimple_omp_for_combined_into_p (stmt
)
2586 && gimple_omp_for_kind (stmt
)
2587 == *(const enum gf_mask
*) (wi
->info
))
2590 return integer_zero_node
;
2599 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2602 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2603 omp_context
*outer_ctx
)
2605 struct walk_stmt_info wi
;
2607 memset (&wi
, 0, sizeof (wi
));
2609 wi
.info
= (void *) &msk
;
2610 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2611 if (wi
.info
!= (void *) &msk
)
2613 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2614 struct omp_for_data fd
;
2615 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2616 /* We need two temporaries with fd.loop.v type (istart/iend)
2617 and then (fd.collapse - 1) temporaries with the same
2618 type for count2 ... countN-1 vars if not constant. */
2619 size_t count
= 2, i
;
2620 tree type
= fd
.iter_type
;
2622 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2624 count
+= fd
.collapse
- 1;
2625 /* If there are lastprivate clauses on the inner
2626 GIMPLE_OMP_FOR, add one more temporaries for the total number
2627 of iterations (product of count1 ... countN-1). */
2628 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2629 OMP_CLAUSE_LASTPRIVATE
))
2631 else if (msk
== GF_OMP_FOR_KIND_FOR
2632 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2633 OMP_CLAUSE_LASTPRIVATE
))
2636 for (i
= 0; i
< count
; i
++)
2638 tree temp
= create_tmp_var (type
);
2639 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2640 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2641 OMP_CLAUSE_DECL (c
) = temp
;
2642 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2643 gimple_omp_taskreg_set_clauses (stmt
, c
);
2648 /* Scan an OpenMP parallel directive. */
2651 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2655 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2657 /* Ignore parallel directives with empty bodies, unless there
2658 are copyin clauses. */
2660 && empty_body_p (gimple_omp_body (stmt
))
2661 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2662 OMP_CLAUSE_COPYIN
) == NULL
)
2664 gsi_replace (gsi
, gimple_build_nop (), false);
2668 if (gimple_omp_parallel_combined_p (stmt
))
2669 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2671 ctx
= new_omp_context (stmt
, outer_ctx
);
2672 taskreg_contexts
.safe_push (ctx
);
2673 if (taskreg_nesting_level
> 1)
2674 ctx
->is_nested
= true;
2675 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2676 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2677 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2678 name
= create_tmp_var_name (".omp_data_s");
2679 name
= build_decl (gimple_location (stmt
),
2680 TYPE_DECL
, name
, ctx
->record_type
);
2681 DECL_ARTIFICIAL (name
) = 1;
2682 DECL_NAMELESS (name
) = 1;
2683 TYPE_NAME (ctx
->record_type
) = name
;
2684 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2685 if (!gimple_omp_parallel_grid_phony (stmt
))
2687 create_omp_child_function (ctx
, false);
2688 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2691 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2692 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2694 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2695 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2698 /* Scan an OpenMP task directive. */
2701 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2705 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2707 /* Ignore task directives with empty bodies. */
2709 && empty_body_p (gimple_omp_body (stmt
)))
2711 gsi_replace (gsi
, gimple_build_nop (), false);
2715 if (gimple_omp_task_taskloop_p (stmt
))
2716 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2718 ctx
= new_omp_context (stmt
, outer_ctx
);
2719 taskreg_contexts
.safe_push (ctx
);
2720 if (taskreg_nesting_level
> 1)
2721 ctx
->is_nested
= true;
2722 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2723 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2724 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2725 name
= create_tmp_var_name (".omp_data_s");
2726 name
= build_decl (gimple_location (stmt
),
2727 TYPE_DECL
, name
, ctx
->record_type
);
2728 DECL_ARTIFICIAL (name
) = 1;
2729 DECL_NAMELESS (name
) = 1;
2730 TYPE_NAME (ctx
->record_type
) = name
;
2731 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2732 create_omp_child_function (ctx
, false);
2733 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2735 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2737 if (ctx
->srecord_type
)
2739 name
= create_tmp_var_name (".omp_data_a");
2740 name
= build_decl (gimple_location (stmt
),
2741 TYPE_DECL
, name
, ctx
->srecord_type
);
2742 DECL_ARTIFICIAL (name
) = 1;
2743 DECL_NAMELESS (name
) = 1;
2744 TYPE_NAME (ctx
->srecord_type
) = name
;
2745 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2746 create_omp_child_function (ctx
, true);
2749 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2751 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2753 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2754 t
= build_int_cst (long_integer_type_node
, 0);
2755 gimple_omp_task_set_arg_size (stmt
, t
);
2756 t
= build_int_cst (long_integer_type_node
, 1);
2757 gimple_omp_task_set_arg_align (stmt
, t
);
2762 /* If any decls have been made addressable during scan_omp,
2763 adjust their fields if needed, and layout record types
2764 of parallel/task constructs. */
2767 finish_taskreg_scan (omp_context
*ctx
)
2769 if (ctx
->record_type
== NULL_TREE
)
2772 /* If any task_shared_vars were needed, verify all
2773 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2774 statements if use_pointer_for_field hasn't changed
2775 because of that. If it did, update field types now. */
2776 if (task_shared_vars
)
2780 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2781 c
; c
= OMP_CLAUSE_CHAIN (c
))
2782 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2783 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2785 tree decl
= OMP_CLAUSE_DECL (c
);
2787 /* Global variables don't need to be copied,
2788 the receiver side will use them directly. */
2789 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2791 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2792 || !use_pointer_for_field (decl
, ctx
))
2794 tree field
= lookup_field (decl
, ctx
);
2795 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2796 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2798 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2799 TREE_THIS_VOLATILE (field
) = 0;
2800 DECL_USER_ALIGN (field
) = 0;
2801 SET_DECL_ALIGN (field
, TYPE_ALIGN (TREE_TYPE (field
)));
2802 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2803 SET_TYPE_ALIGN (ctx
->record_type
, DECL_ALIGN (field
));
2804 if (ctx
->srecord_type
)
2806 tree sfield
= lookup_sfield (decl
, ctx
);
2807 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2808 TREE_THIS_VOLATILE (sfield
) = 0;
2809 DECL_USER_ALIGN (sfield
) = 0;
2810 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
2811 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2812 SET_TYPE_ALIGN (ctx
->srecord_type
, DECL_ALIGN (sfield
));
2817 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2819 layout_type (ctx
->record_type
);
2820 fixup_child_record_type (ctx
);
2824 location_t loc
= gimple_location (ctx
->stmt
);
2825 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2826 /* Move VLA fields to the end. */
2827 p
= &TYPE_FIELDS (ctx
->record_type
);
2829 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2830 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2833 *p
= TREE_CHAIN (*p
);
2834 TREE_CHAIN (*q
) = NULL_TREE
;
2835 q
= &TREE_CHAIN (*q
);
2838 p
= &DECL_CHAIN (*p
);
2840 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2842 /* Move fields corresponding to first and second _looptemp_
2843 clause first. There are filled by GOMP_taskloop
2844 and thus need to be in specific positions. */
2845 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2846 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2847 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2848 OMP_CLAUSE__LOOPTEMP_
);
2849 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2850 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2851 p
= &TYPE_FIELDS (ctx
->record_type
);
2853 if (*p
== f1
|| *p
== f2
)
2854 *p
= DECL_CHAIN (*p
);
2856 p
= &DECL_CHAIN (*p
);
2857 DECL_CHAIN (f1
) = f2
;
2858 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2859 TYPE_FIELDS (ctx
->record_type
) = f1
;
2860 if (ctx
->srecord_type
)
2862 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2863 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2864 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2866 if (*p
== f1
|| *p
== f2
)
2867 *p
= DECL_CHAIN (*p
);
2869 p
= &DECL_CHAIN (*p
);
2870 DECL_CHAIN (f1
) = f2
;
2871 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2872 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2875 layout_type (ctx
->record_type
);
2876 fixup_child_record_type (ctx
);
2877 if (ctx
->srecord_type
)
2878 layout_type (ctx
->srecord_type
);
2879 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2880 TYPE_SIZE_UNIT (ctx
->record_type
));
2881 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2882 t
= build_int_cst (long_integer_type_node
,
2883 TYPE_ALIGN_UNIT (ctx
->record_type
));
2884 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2888 /* Find the enclosing offload context. */
2890 static omp_context
*
2891 enclosing_target_ctx (omp_context
*ctx
)
2893 for (; ctx
; ctx
= ctx
->outer
)
2894 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2900 /* Return true if ctx is part of an oacc kernels region. */
2903 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2905 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2907 gimple
*stmt
= ctx
->stmt
;
2908 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2909 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2916 /* Check the parallelism clauses inside a kernels regions.
2917 Until kernels handling moves to use the same loop indirection
2918 scheme as parallel, we need to do this checking early. */
2921 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2923 bool checking
= true;
2924 unsigned outer_mask
= 0;
2925 unsigned this_mask
= 0;
2926 bool has_seq
= false, has_auto
= false;
2929 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2933 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2935 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2938 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2940 switch (OMP_CLAUSE_CODE (c
))
2942 case OMP_CLAUSE_GANG
:
2943 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2945 case OMP_CLAUSE_WORKER
:
2946 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2948 case OMP_CLAUSE_VECTOR
:
2949 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2951 case OMP_CLAUSE_SEQ
:
2954 case OMP_CLAUSE_AUTO
:
2964 if (has_seq
&& (this_mask
|| has_auto
))
2965 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2966 " OpenACC loop specifiers");
2967 else if (has_auto
&& this_mask
)
2968 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2969 " OpenACC loop specifiers");
2971 if (this_mask
& outer_mask
)
2972 error_at (gimple_location (stmt
), "inner loop uses same"
2973 " OpenACC parallelism as containing loop");
2976 return outer_mask
| this_mask
;
2979 /* Scan a GIMPLE_OMP_FOR. */
2982 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2986 tree clauses
= gimple_omp_for_clauses (stmt
);
2988 ctx
= new_omp_context (stmt
, outer_ctx
);
2990 if (is_gimple_omp_oacc (stmt
))
2992 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2994 if (!tgt
|| is_oacc_parallel (tgt
))
2995 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2997 char const *check
= NULL
;
2999 switch (OMP_CLAUSE_CODE (c
))
3001 case OMP_CLAUSE_GANG
:
3005 case OMP_CLAUSE_WORKER
:
3009 case OMP_CLAUSE_VECTOR
:
3017 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3018 error_at (gimple_location (stmt
),
3019 "argument not permitted on %qs clause in"
3020 " OpenACC %<parallel%>", check
);
3023 if (tgt
&& is_oacc_kernels (tgt
))
3025 /* Strip out reductions, as they are not handled yet. */
3026 tree
*prev_ptr
= &clauses
;
3028 while (tree probe
= *prev_ptr
)
3030 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3032 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3033 *prev_ptr
= *next_ptr
;
3035 prev_ptr
= next_ptr
;
3038 gimple_omp_for_set_clauses (stmt
, clauses
);
3039 check_oacc_kernel_gwv (stmt
, ctx
);
3043 scan_sharing_clauses (clauses
, ctx
);
3045 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3046 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3048 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3049 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3050 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3051 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3053 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3056 /* Scan an OpenMP sections directive. */
3059 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3063 ctx
= new_omp_context (stmt
, outer_ctx
);
3064 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3065 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3068 /* Scan an OpenMP single directive. */
3071 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3076 ctx
= new_omp_context (stmt
, outer_ctx
);
3077 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3078 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3079 name
= create_tmp_var_name (".omp_copy_s");
3080 name
= build_decl (gimple_location (stmt
),
3081 TYPE_DECL
, name
, ctx
->record_type
);
3082 TYPE_NAME (ctx
->record_type
) = name
;
3084 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3085 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3087 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3088 ctx
->record_type
= NULL
;
3090 layout_type (ctx
->record_type
);
3093 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3094 used in the corresponding offloaded function are restrict. */
3097 omp_target_base_pointers_restrict_p (tree clauses
)
3099 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3101 if (flag_openacc
== 0)
3104 /* I. Basic example:
3108 unsigned int a[2], b[2];
3110 #pragma acc kernels \
3119 After gimplification, we have:
3121 #pragma omp target oacc_kernels \
3122 map(force_from:a [len: 8]) \
3123 map(force_from:b [len: 8])
3129 Because both mappings have the force prefix, we know that they will be
3130 allocated when calling the corresponding offloaded function, which means we
3131 can mark the base pointers for a and b in the offloaded function as
3135 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3137 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
3140 switch (OMP_CLAUSE_MAP_KIND (c
))
3142 case GOMP_MAP_FORCE_ALLOC
:
3143 case GOMP_MAP_FORCE_TO
:
3144 case GOMP_MAP_FORCE_FROM
:
3145 case GOMP_MAP_FORCE_TOFROM
:
3155 /* Scan a GIMPLE_OMP_TARGET. */
3158 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3162 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3163 tree clauses
= gimple_omp_target_clauses (stmt
);
3165 ctx
= new_omp_context (stmt
, outer_ctx
);
3166 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3167 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3168 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3169 name
= create_tmp_var_name (".omp_data_t");
3170 name
= build_decl (gimple_location (stmt
),
3171 TYPE_DECL
, name
, ctx
->record_type
);
3172 DECL_ARTIFICIAL (name
) = 1;
3173 DECL_NAMELESS (name
) = 1;
3174 TYPE_NAME (ctx
->record_type
) = name
;
3175 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3177 bool base_pointers_restrict
= false;
3180 create_omp_child_function (ctx
, false);
3181 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3183 base_pointers_restrict
= omp_target_base_pointers_restrict_p (clauses
);
3184 if (base_pointers_restrict
3185 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3187 "Base pointers in offloaded function are restrict\n");
3190 scan_sharing_clauses (clauses
, ctx
, base_pointers_restrict
);
3191 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3193 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3194 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3197 TYPE_FIELDS (ctx
->record_type
)
3198 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3201 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3202 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3204 field
= DECL_CHAIN (field
))
3205 gcc_assert (DECL_ALIGN (field
) == align
);
3207 layout_type (ctx
->record_type
);
3209 fixup_child_record_type (ctx
);
3213 /* Scan an OpenMP teams directive. */
3216 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3218 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3219 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3220 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3223 /* Check nesting restrictions. */
3225 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3229 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
3230 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3231 the original copy of its contents. */
3234 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3235 inside an OpenACC CTX. */
3236 if (!(is_gimple_omp (stmt
)
3237 && is_gimple_omp_oacc (stmt
))
3238 /* Except for atomic codes that we share with OpenMP. */
3239 && !(gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3240 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3242 if (get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3244 error_at (gimple_location (stmt
),
3245 "non-OpenACC construct inside of OpenACC routine");
3249 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3250 if (is_gimple_omp (octx
->stmt
)
3251 && is_gimple_omp_oacc (octx
->stmt
))
3253 error_at (gimple_location (stmt
),
3254 "non-OpenACC construct inside of OpenACC region");
3261 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3262 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3265 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3267 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3268 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3270 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3271 && (ctx
->outer
== NULL
3272 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3273 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3274 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3275 != GF_OMP_FOR_KIND_FOR
)
3276 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3278 error_at (gimple_location (stmt
),
3279 "%<ordered simd threads%> must be closely "
3280 "nested inside of %<for simd%> region");
3286 error_at (gimple_location (stmt
),
3287 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3288 " may not be nested inside %<simd%> region");
3291 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3293 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3294 || (gimple_omp_for_kind (stmt
)
3295 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3296 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3298 error_at (gimple_location (stmt
),
3299 "only %<distribute%> or %<parallel%> regions are "
3300 "allowed to be strictly nested inside %<teams%> "
3306 switch (gimple_code (stmt
))
3308 case GIMPLE_OMP_FOR
:
3309 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3311 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3313 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3315 error_at (gimple_location (stmt
),
3316 "%<distribute%> region must be strictly nested "
3317 "inside %<teams%> construct");
3322 /* We split taskloop into task and nested taskloop in it. */
3323 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3325 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3330 switch (gimple_code (ctx
->stmt
))
3332 case GIMPLE_OMP_FOR
:
3333 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3334 == GF_OMP_FOR_KIND_OACC_LOOP
);
3337 case GIMPLE_OMP_TARGET
:
3338 switch (gimple_omp_target_kind (ctx
->stmt
))
3340 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3341 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3352 else if (get_oacc_fn_attrib (current_function_decl
))
3356 error_at (gimple_location (stmt
),
3357 "OpenACC loop directive must be associated with"
3358 " an OpenACC compute region");
3364 if (is_gimple_call (stmt
)
3365 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3366 == BUILT_IN_GOMP_CANCEL
3367 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3368 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3370 const char *bad
= NULL
;
3371 const char *kind
= NULL
;
3372 const char *construct
3373 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3374 == BUILT_IN_GOMP_CANCEL
)
3375 ? "#pragma omp cancel"
3376 : "#pragma omp cancellation point";
3379 error_at (gimple_location (stmt
), "orphaned %qs construct",
3383 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3384 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3388 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3389 bad
= "#pragma omp parallel";
3390 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3391 == BUILT_IN_GOMP_CANCEL
3392 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3393 ctx
->cancellable
= true;
3397 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3398 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3399 bad
= "#pragma omp for";
3400 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3401 == BUILT_IN_GOMP_CANCEL
3402 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3404 ctx
->cancellable
= true;
3405 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3407 warning_at (gimple_location (stmt
), 0,
3408 "%<#pragma omp cancel for%> inside "
3409 "%<nowait%> for construct");
3410 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3411 OMP_CLAUSE_ORDERED
))
3412 warning_at (gimple_location (stmt
), 0,
3413 "%<#pragma omp cancel for%> inside "
3414 "%<ordered%> for construct");
3419 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3420 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3421 bad
= "#pragma omp sections";
3422 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3423 == BUILT_IN_GOMP_CANCEL
3424 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3426 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3428 ctx
->cancellable
= true;
3429 if (find_omp_clause (gimple_omp_sections_clauses
3432 warning_at (gimple_location (stmt
), 0,
3433 "%<#pragma omp cancel sections%> inside "
3434 "%<nowait%> sections construct");
3438 gcc_assert (ctx
->outer
3439 && gimple_code (ctx
->outer
->stmt
)
3440 == GIMPLE_OMP_SECTIONS
);
3441 ctx
->outer
->cancellable
= true;
3442 if (find_omp_clause (gimple_omp_sections_clauses
3445 warning_at (gimple_location (stmt
), 0,
3446 "%<#pragma omp cancel sections%> inside "
3447 "%<nowait%> sections construct");
3453 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3454 bad
= "#pragma omp task";
3457 for (omp_context
*octx
= ctx
->outer
;
3458 octx
; octx
= octx
->outer
)
3460 switch (gimple_code (octx
->stmt
))
3462 case GIMPLE_OMP_TASKGROUP
:
3464 case GIMPLE_OMP_TARGET
:
3465 if (gimple_omp_target_kind (octx
->stmt
)
3466 != GF_OMP_TARGET_KIND_REGION
)
3469 case GIMPLE_OMP_PARALLEL
:
3470 case GIMPLE_OMP_TEAMS
:
3471 error_at (gimple_location (stmt
),
3472 "%<%s taskgroup%> construct not closely "
3473 "nested inside of %<taskgroup%> region",
3481 ctx
->cancellable
= true;
3486 error_at (gimple_location (stmt
), "invalid arguments");
3491 error_at (gimple_location (stmt
),
3492 "%<%s %s%> construct not closely nested inside of %qs",
3493 construct
, kind
, bad
);
3498 case GIMPLE_OMP_SECTIONS
:
3499 case GIMPLE_OMP_SINGLE
:
3500 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3501 switch (gimple_code (ctx
->stmt
))
3503 case GIMPLE_OMP_FOR
:
3504 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3505 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3508 case GIMPLE_OMP_SECTIONS
:
3509 case GIMPLE_OMP_SINGLE
:
3510 case GIMPLE_OMP_ORDERED
:
3511 case GIMPLE_OMP_MASTER
:
3512 case GIMPLE_OMP_TASK
:
3513 case GIMPLE_OMP_CRITICAL
:
3514 if (is_gimple_call (stmt
))
3516 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3517 != BUILT_IN_GOMP_BARRIER
)
3519 error_at (gimple_location (stmt
),
3520 "barrier region may not be closely nested inside "
3521 "of work-sharing, %<critical%>, %<ordered%>, "
3522 "%<master%>, explicit %<task%> or %<taskloop%> "
3526 error_at (gimple_location (stmt
),
3527 "work-sharing region may not be closely nested inside "
3528 "of work-sharing, %<critical%>, %<ordered%>, "
3529 "%<master%>, explicit %<task%> or %<taskloop%> region");
3531 case GIMPLE_OMP_PARALLEL
:
3532 case GIMPLE_OMP_TEAMS
:
3534 case GIMPLE_OMP_TARGET
:
3535 if (gimple_omp_target_kind (ctx
->stmt
)
3536 == GF_OMP_TARGET_KIND_REGION
)
3543 case GIMPLE_OMP_MASTER
:
3544 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3545 switch (gimple_code (ctx
->stmt
))
3547 case GIMPLE_OMP_FOR
:
3548 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3549 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3552 case GIMPLE_OMP_SECTIONS
:
3553 case GIMPLE_OMP_SINGLE
:
3554 case GIMPLE_OMP_TASK
:
3555 error_at (gimple_location (stmt
),
3556 "%<master%> region may not be closely nested inside "
3557 "of work-sharing, explicit %<task%> or %<taskloop%> "
3560 case GIMPLE_OMP_PARALLEL
:
3561 case GIMPLE_OMP_TEAMS
:
3563 case GIMPLE_OMP_TARGET
:
3564 if (gimple_omp_target_kind (ctx
->stmt
)
3565 == GF_OMP_TARGET_KIND_REGION
)
3572 case GIMPLE_OMP_TASK
:
3573 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3574 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3575 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3576 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3578 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3579 error_at (OMP_CLAUSE_LOCATION (c
),
3580 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3581 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3585 case GIMPLE_OMP_ORDERED
:
3586 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3587 c
; c
= OMP_CLAUSE_CHAIN (c
))
3589 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3591 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3592 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3595 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3596 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3597 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3600 /* Look for containing ordered(N) loop. */
3602 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3604 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3605 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3607 error_at (OMP_CLAUSE_LOCATION (c
),
3608 "%<ordered%> construct with %<depend%> clause "
3609 "must be closely nested inside an %<ordered%> "
3613 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3615 error_at (OMP_CLAUSE_LOCATION (c
),
3616 "%<ordered%> construct with %<depend%> clause "
3617 "must be closely nested inside a loop with "
3618 "%<ordered%> clause with a parameter");
3624 error_at (OMP_CLAUSE_LOCATION (c
),
3625 "invalid depend kind in omp %<ordered%> %<depend%>");
3629 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3630 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3632 /* ordered simd must be closely nested inside of simd region,
3633 and simd region must not encounter constructs other than
3634 ordered simd, therefore ordered simd may be either orphaned,
3635 or ctx->stmt must be simd. The latter case is handled already
3639 error_at (gimple_location (stmt
),
3640 "%<ordered%> %<simd%> must be closely nested inside "
3645 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3646 switch (gimple_code (ctx
->stmt
))
3648 case GIMPLE_OMP_CRITICAL
:
3649 case GIMPLE_OMP_TASK
:
3650 case GIMPLE_OMP_ORDERED
:
3651 ordered_in_taskloop
:
3652 error_at (gimple_location (stmt
),
3653 "%<ordered%> region may not be closely nested inside "
3654 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3655 "%<taskloop%> region");
3657 case GIMPLE_OMP_FOR
:
3658 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3659 goto ordered_in_taskloop
;
3660 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3661 OMP_CLAUSE_ORDERED
) == NULL
)
3663 error_at (gimple_location (stmt
),
3664 "%<ordered%> region must be closely nested inside "
3665 "a loop region with an %<ordered%> clause");
3669 case GIMPLE_OMP_TARGET
:
3670 if (gimple_omp_target_kind (ctx
->stmt
)
3671 != GF_OMP_TARGET_KIND_REGION
)
3674 case GIMPLE_OMP_PARALLEL
:
3675 case GIMPLE_OMP_TEAMS
:
3676 error_at (gimple_location (stmt
),
3677 "%<ordered%> region must be closely nested inside "
3678 "a loop region with an %<ordered%> clause");
3684 case GIMPLE_OMP_CRITICAL
:
3687 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3688 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3689 if (gomp_critical
*other_crit
3690 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3691 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3693 error_at (gimple_location (stmt
),
3694 "%<critical%> region may not be nested inside "
3695 "a %<critical%> region with the same name");
3700 case GIMPLE_OMP_TEAMS
:
3702 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3703 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3705 error_at (gimple_location (stmt
),
3706 "%<teams%> construct not closely nested inside of "
3707 "%<target%> construct");
3711 case GIMPLE_OMP_TARGET
:
3712 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3713 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3714 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3715 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3717 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3718 error_at (OMP_CLAUSE_LOCATION (c
),
3719 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3720 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3723 if (is_gimple_omp_offloaded (stmt
)
3724 && get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3726 error_at (gimple_location (stmt
),
3727 "OpenACC region inside of OpenACC routine, nested "
3728 "parallelism not supported yet");
3731 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3733 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3735 if (is_gimple_omp (stmt
)
3736 && is_gimple_omp_oacc (stmt
)
3737 && is_gimple_omp (ctx
->stmt
))
3739 error_at (gimple_location (stmt
),
3740 "OpenACC construct inside of non-OpenACC region");
3746 const char *stmt_name
, *ctx_stmt_name
;
3747 switch (gimple_omp_target_kind (stmt
))
3749 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3750 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3751 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3752 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3753 stmt_name
= "target enter data"; break;
3754 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3755 stmt_name
= "target exit data"; break;
3756 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3757 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3758 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3759 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3760 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3761 stmt_name
= "enter/exit data"; break;
3762 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3764 default: gcc_unreachable ();
3766 switch (gimple_omp_target_kind (ctx
->stmt
))
3768 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3769 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3770 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3771 ctx_stmt_name
= "parallel"; break;
3772 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3773 ctx_stmt_name
= "kernels"; break;
3774 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3775 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3776 ctx_stmt_name
= "host_data"; break;
3777 default: gcc_unreachable ();
3780 /* OpenACC/OpenMP mismatch? */
3781 if (is_gimple_omp_oacc (stmt
)
3782 != is_gimple_omp_oacc (ctx
->stmt
))
3784 error_at (gimple_location (stmt
),
3785 "%s %qs construct inside of %s %qs region",
3786 (is_gimple_omp_oacc (stmt
)
3787 ? "OpenACC" : "OpenMP"), stmt_name
,
3788 (is_gimple_omp_oacc (ctx
->stmt
)
3789 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3792 if (is_gimple_omp_offloaded (ctx
->stmt
))
3794 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3795 if (is_gimple_omp_oacc (ctx
->stmt
))
3797 error_at (gimple_location (stmt
),
3798 "%qs construct inside of %qs region",
3799 stmt_name
, ctx_stmt_name
);
3804 warning_at (gimple_location (stmt
), 0,
3805 "%qs construct inside of %qs region",
3806 stmt_name
, ctx_stmt_name
);
3818 /* Helper function scan_omp.
3820 Callback for walk_tree or operators in walk_gimple_stmt used to
3821 scan for OMP directives in TP. */
3824 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3826 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3827 omp_context
*ctx
= (omp_context
*) wi
->info
;
3830 switch (TREE_CODE (t
))
3838 tree repl
= remap_decl (t
, &ctx
->cb
);
3839 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3845 if (ctx
&& TYPE_P (t
))
3846 *tp
= remap_type (t
, &ctx
->cb
);
3847 else if (!DECL_P (t
))
3852 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3853 if (tem
!= TREE_TYPE (t
))
3855 if (TREE_CODE (t
) == INTEGER_CST
)
3856 *tp
= wide_int_to_tree (tem
, t
);
3858 TREE_TYPE (t
) = tem
;
3868 /* Return true if FNDECL is a setjmp or a longjmp. */
3871 setjmp_or_longjmp_p (const_tree fndecl
)
3873 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3874 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3875 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3878 tree declname
= DECL_NAME (fndecl
);
3881 const char *name
= IDENTIFIER_POINTER (declname
);
3882 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3886 /* Helper function for scan_omp.
3888 Callback for walk_gimple_stmt used to scan for OMP directives in
3889 the current statement in GSI. */
3892 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3893 struct walk_stmt_info
*wi
)
3895 gimple
*stmt
= gsi_stmt (*gsi
);
3896 omp_context
*ctx
= (omp_context
*) wi
->info
;
3898 if (gimple_has_location (stmt
))
3899 input_location
= gimple_location (stmt
);
3901 /* Check the nesting restrictions. */
3902 bool remove
= false;
3903 if (is_gimple_omp (stmt
))
3904 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3905 else if (is_gimple_call (stmt
))
3907 tree fndecl
= gimple_call_fndecl (stmt
);
3910 if (setjmp_or_longjmp_p (fndecl
)
3912 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3913 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3916 error_at (gimple_location (stmt
),
3917 "setjmp/longjmp inside simd construct");
3919 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3920 switch (DECL_FUNCTION_CODE (fndecl
))
3922 case BUILT_IN_GOMP_BARRIER
:
3923 case BUILT_IN_GOMP_CANCEL
:
3924 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3925 case BUILT_IN_GOMP_TASKYIELD
:
3926 case BUILT_IN_GOMP_TASKWAIT
:
3927 case BUILT_IN_GOMP_TASKGROUP_START
:
3928 case BUILT_IN_GOMP_TASKGROUP_END
:
3929 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3938 stmt
= gimple_build_nop ();
3939 gsi_replace (gsi
, stmt
, false);
3942 *handled_ops_p
= true;
3944 switch (gimple_code (stmt
))
3946 case GIMPLE_OMP_PARALLEL
:
3947 taskreg_nesting_level
++;
3948 scan_omp_parallel (gsi
, ctx
);
3949 taskreg_nesting_level
--;
3952 case GIMPLE_OMP_TASK
:
3953 taskreg_nesting_level
++;
3954 scan_omp_task (gsi
, ctx
);
3955 taskreg_nesting_level
--;
3958 case GIMPLE_OMP_FOR
:
3959 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3962 case GIMPLE_OMP_SECTIONS
:
3963 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3966 case GIMPLE_OMP_SINGLE
:
3967 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3970 case GIMPLE_OMP_SECTION
:
3971 case GIMPLE_OMP_MASTER
:
3972 case GIMPLE_OMP_TASKGROUP
:
3973 case GIMPLE_OMP_ORDERED
:
3974 case GIMPLE_OMP_CRITICAL
:
3975 case GIMPLE_OMP_GRID_BODY
:
3976 ctx
= new_omp_context (stmt
, ctx
);
3977 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3980 case GIMPLE_OMP_TARGET
:
3981 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3984 case GIMPLE_OMP_TEAMS
:
3985 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3992 *handled_ops_p
= false;
3994 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3996 var
= DECL_CHAIN (var
))
3997 insert_decl_map (&ctx
->cb
, var
, var
);
4001 *handled_ops_p
= false;
4009 /* Scan all the statements starting at the current statement. CTX
4010 contains context information about the OMP directives and
4011 clauses found during the scan. */
4014 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
4016 location_t saved_location
;
4017 struct walk_stmt_info wi
;
4019 memset (&wi
, 0, sizeof (wi
));
4021 wi
.want_locations
= true;
4023 saved_location
= input_location
;
4024 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
4025 input_location
= saved_location
;
4028 /* Re-gimplification and code generation routines. */
4030 /* Build a call to GOMP_barrier. */
4033 build_omp_barrier (tree lhs
)
4035 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
4036 : BUILT_IN_GOMP_BARRIER
);
4037 gcall
*g
= gimple_build_call (fndecl
, 0);
4039 gimple_call_set_lhs (g
, lhs
);
4043 /* If a context was created for STMT when it was scanned, return it. */
4045 static omp_context
*
4046 maybe_lookup_ctx (gimple
*stmt
)
4049 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
4050 return n
? (omp_context
*) n
->value
: NULL
;
4054 /* Find the mapping for DECL in CTX or the immediately enclosing
4055 context that has a mapping for DECL.
4057 If CTX is a nested parallel directive, we may have to use the decl
4058 mappings created in CTX's parent context. Suppose that we have the
4059 following parallel nesting (variable UIDs showed for clarity):
4062 #omp parallel shared(iD.1562) -> outer parallel
4063 iD.1562 = iD.1562 + 1;
4065 #omp parallel shared (iD.1562) -> inner parallel
4066 iD.1562 = iD.1562 - 1;
4068 Each parallel structure will create a distinct .omp_data_s structure
4069 for copying iD.1562 in/out of the directive:
4071 outer parallel .omp_data_s.1.i -> iD.1562
4072 inner parallel .omp_data_s.2.i -> iD.1562
4074 A shared variable mapping will produce a copy-out operation before
4075 the parallel directive and a copy-in operation after it. So, in
4076 this case we would have:
4079 .omp_data_o.1.i = iD.1562;
4080 #omp parallel shared(iD.1562) -> outer parallel
4081 .omp_data_i.1 = &.omp_data_o.1
4082 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4084 .omp_data_o.2.i = iD.1562; -> **
4085 #omp parallel shared(iD.1562) -> inner parallel
4086 .omp_data_i.2 = &.omp_data_o.2
4087 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4090 ** This is a problem. The symbol iD.1562 cannot be referenced
4091 inside the body of the outer parallel region. But since we are
4092 emitting this copy operation while expanding the inner parallel
4093 directive, we need to access the CTX structure of the outer
4094 parallel directive to get the correct mapping:
4096 .omp_data_o.2.i = .omp_data_i.1->i
4098 Since there may be other workshare or parallel directives enclosing
4099 the parallel directive, it may be necessary to walk up the context
4100 parent chain. This is not a problem in general because nested
4101 parallelism happens only rarely. */
4104 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4109 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4110 t
= maybe_lookup_decl (decl
, up
);
4112 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
4114 return t
? t
: decl
;
4118 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4119 in outer contexts. */
4122 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4127 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4128 t
= maybe_lookup_decl (decl
, up
);
4130 return t
? t
: decl
;
4134 /* Construct the initialization value for reduction operation OP. */
4137 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4146 case TRUTH_ORIF_EXPR
:
4147 case TRUTH_XOR_EXPR
:
4149 return build_zero_cst (type
);
4152 case TRUTH_AND_EXPR
:
4153 case TRUTH_ANDIF_EXPR
:
4155 return fold_convert_loc (loc
, type
, integer_one_node
);
4158 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4161 if (SCALAR_FLOAT_TYPE_P (type
))
4163 REAL_VALUE_TYPE max
, min
;
4164 if (HONOR_INFINITIES (type
))
4167 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4170 real_maxval (&min
, 1, TYPE_MODE (type
));
4171 return build_real (type
, min
);
4173 else if (POINTER_TYPE_P (type
))
4176 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4177 return wide_int_to_tree (type
, min
);
4181 gcc_assert (INTEGRAL_TYPE_P (type
));
4182 return TYPE_MIN_VALUE (type
);
4186 if (SCALAR_FLOAT_TYPE_P (type
))
4188 REAL_VALUE_TYPE max
;
4189 if (HONOR_INFINITIES (type
))
4192 real_maxval (&max
, 0, TYPE_MODE (type
));
4193 return build_real (type
, max
);
4195 else if (POINTER_TYPE_P (type
))
4198 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4199 return wide_int_to_tree (type
, max
);
4203 gcc_assert (INTEGRAL_TYPE_P (type
));
4204 return TYPE_MAX_VALUE (type
);
4212 /* Construct the initialization value for reduction CLAUSE. */
4215 omp_reduction_init (tree clause
, tree type
)
4217 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4218 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4221 /* Return alignment to be assumed for var in CLAUSE, which should be
4222 OMP_CLAUSE_ALIGNED. */
4225 omp_clause_aligned_alignment (tree clause
)
4227 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4228 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4230 /* Otherwise return implementation defined alignment. */
4231 unsigned int al
= 1;
4232 machine_mode mode
, vmode
;
4233 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4235 vs
= 1 << floor_log2 (vs
);
4236 static enum mode_class classes
[]
4237 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4238 for (int i
= 0; i
< 4; i
+= 2)
4239 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4241 mode
= GET_MODE_WIDER_MODE (mode
))
4243 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4244 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4247 && GET_MODE_SIZE (vmode
) < vs
4248 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4249 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4251 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4252 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4254 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4255 / GET_MODE_SIZE (mode
));
4256 if (TYPE_MODE (type
) != vmode
)
4258 if (TYPE_ALIGN_UNIT (type
) > al
)
4259 al
= TYPE_ALIGN_UNIT (type
);
4261 return build_int_cst (integer_type_node
, al
);
4264 /* Return maximum possible vectorization factor for the target. */
4271 || !flag_tree_loop_optimize
4272 || (!flag_tree_loop_vectorize
4273 && (global_options_set
.x_flag_tree_loop_vectorize
4274 || global_options_set
.x_flag_tree_vectorize
)))
4277 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4280 vs
= 1 << floor_log2 (vs
);
4283 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4284 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4285 return GET_MODE_NUNITS (vqimode
);
4289 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4293 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4294 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4298 max_vf
= omp_max_vf ();
4301 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4302 OMP_CLAUSE_SAFELEN
);
4303 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4305 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4307 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4311 idx
= create_tmp_var (unsigned_type_node
);
4312 lane
= create_tmp_var (unsigned_type_node
);
4318 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4319 tree avar
= create_tmp_var_raw (atype
);
4320 if (TREE_ADDRESSABLE (new_var
))
4321 TREE_ADDRESSABLE (avar
) = 1;
4322 DECL_ATTRIBUTES (avar
)
4323 = tree_cons (get_identifier ("omp simd array"), NULL
,
4324 DECL_ATTRIBUTES (avar
));
4325 gimple_add_tmp_var (avar
);
4326 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4327 NULL_TREE
, NULL_TREE
);
4328 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4329 NULL_TREE
, NULL_TREE
);
4330 if (DECL_P (new_var
))
4332 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4333 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4338 /* Helper function of lower_rec_input_clauses. For a reference
4339 in simd reduction, add an underlying variable it will reference. */
4342 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4344 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4345 if (TREE_CONSTANT (z
))
4347 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4348 get_name (new_vard
));
4349 gimple_add_tmp_var (z
);
4350 TREE_ADDRESSABLE (z
) = 1;
4351 z
= build_fold_addr_expr_loc (loc
, z
);
4352 gimplify_assign (new_vard
, z
, ilist
);
4356 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4357 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4358 private variables. Initialization statements go in ILIST, while calls
4359 to destructors go in DLIST. */
4362 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4363 omp_context
*ctx
, struct omp_for_data
*fd
)
4365 tree c
, dtor
, copyin_seq
, x
, ptr
;
4366 bool copyin_by_ref
= false;
4367 bool lastprivate_firstprivate
= false;
4368 bool reduction_omp_orig_ref
= false;
4370 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4371 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4373 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4374 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4375 gimple_seq llist
[2] = { NULL
, NULL
};
4379 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4380 with data sharing clauses referencing variable sized vars. That
4381 is unnecessarily hard to support and very unlikely to result in
4382 vectorized code anyway. */
4384 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4385 switch (OMP_CLAUSE_CODE (c
))
4387 case OMP_CLAUSE_LINEAR
:
4388 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4391 case OMP_CLAUSE_PRIVATE
:
4392 case OMP_CLAUSE_FIRSTPRIVATE
:
4393 case OMP_CLAUSE_LASTPRIVATE
:
4394 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4397 case OMP_CLAUSE_REDUCTION
:
4398 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4399 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4406 /* Do all the fixed sized types in the first pass, and the variable sized
4407 types in the second pass. This makes sure that the scalar arguments to
4408 the variable sized types are processed before we use them in the
4409 variable sized operations. */
4410 for (pass
= 0; pass
< 2; ++pass
)
4412 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4414 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4417 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4421 case OMP_CLAUSE_PRIVATE
:
4422 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4425 case OMP_CLAUSE_SHARED
:
4426 /* Ignore shared directives in teams construct. */
4427 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4429 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4431 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4432 || is_global_var (OMP_CLAUSE_DECL (c
)));
4435 case OMP_CLAUSE_FIRSTPRIVATE
:
4436 case OMP_CLAUSE_COPYIN
:
4438 case OMP_CLAUSE_LINEAR
:
4439 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4440 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4441 lastprivate_firstprivate
= true;
4443 case OMP_CLAUSE_REDUCTION
:
4444 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4445 reduction_omp_orig_ref
= true;
4447 case OMP_CLAUSE__LOOPTEMP_
:
4448 /* Handle _looptemp_ clauses only on parallel/task. */
4452 case OMP_CLAUSE_LASTPRIVATE
:
4453 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4455 lastprivate_firstprivate
= true;
4456 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4459 /* Even without corresponding firstprivate, if
4460 decl is Fortran allocatable, it needs outer var
4463 && lang_hooks
.decls
.omp_private_outer_ref
4464 (OMP_CLAUSE_DECL (c
)))
4465 lastprivate_firstprivate
= true;
4467 case OMP_CLAUSE_ALIGNED
:
4470 var
= OMP_CLAUSE_DECL (c
);
4471 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4472 && !is_global_var (var
))
4474 new_var
= maybe_lookup_decl (var
, ctx
);
4475 if (new_var
== NULL_TREE
)
4476 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4477 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4478 tree alarg
= omp_clause_aligned_alignment (c
);
4479 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4480 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
, alarg
);
4481 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4482 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4483 gimplify_and_add (x
, ilist
);
4485 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4486 && is_global_var (var
))
4488 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4489 new_var
= lookup_decl (var
, ctx
);
4490 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4491 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4492 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4493 tree alarg
= omp_clause_aligned_alignment (c
);
4494 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4495 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
, alarg
);
4496 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4497 x
= create_tmp_var (ptype
);
4498 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4499 gimplify_and_add (t
, ilist
);
4500 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4501 SET_DECL_VALUE_EXPR (new_var
, t
);
4502 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4509 new_var
= var
= OMP_CLAUSE_DECL (c
);
4510 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4512 var
= TREE_OPERAND (var
, 0);
4513 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4514 var
= TREE_OPERAND (var
, 0);
4515 if (TREE_CODE (var
) == INDIRECT_REF
4516 || TREE_CODE (var
) == ADDR_EXPR
)
4517 var
= TREE_OPERAND (var
, 0);
4518 if (is_variable_sized (var
))
4520 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4521 var
= DECL_VALUE_EXPR (var
);
4522 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4523 var
= TREE_OPERAND (var
, 0);
4524 gcc_assert (DECL_P (var
));
4528 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4529 new_var
= lookup_decl (var
, ctx
);
4531 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4536 /* C/C++ array section reductions. */
4537 else if (c_kind
== OMP_CLAUSE_REDUCTION
4538 && var
!= OMP_CLAUSE_DECL (c
))
4543 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4544 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4545 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4547 tree b
= TREE_OPERAND (orig_var
, 1);
4548 b
= maybe_lookup_decl (b
, ctx
);
4551 b
= TREE_OPERAND (orig_var
, 1);
4552 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4554 if (integer_zerop (bias
))
4558 bias
= fold_convert_loc (clause_loc
,
4559 TREE_TYPE (b
), bias
);
4560 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4561 TREE_TYPE (b
), b
, bias
);
4563 orig_var
= TREE_OPERAND (orig_var
, 0);
4565 if (TREE_CODE (orig_var
) == INDIRECT_REF
4566 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4567 orig_var
= TREE_OPERAND (orig_var
, 0);
4568 tree d
= OMP_CLAUSE_DECL (c
);
4569 tree type
= TREE_TYPE (d
);
4570 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4571 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4572 const char *name
= get_name (orig_var
);
4573 if (TREE_CONSTANT (v
))
4575 x
= create_tmp_var_raw (type
, name
);
4576 gimple_add_tmp_var (x
);
4577 TREE_ADDRESSABLE (x
) = 1;
4578 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4583 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4584 tree t
= maybe_lookup_decl (v
, ctx
);
4588 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4589 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4590 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4592 build_int_cst (TREE_TYPE (v
), 1));
4593 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4595 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4596 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4597 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4600 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4601 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4602 tree y
= create_tmp_var (ptype
, name
);
4603 gimplify_assign (y
, x
, ilist
);
4607 if (!integer_zerop (bias
))
4609 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4611 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4613 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4614 pointer_sized_int_node
, yb
, bias
);
4615 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4616 yb
= create_tmp_var (ptype
, name
);
4617 gimplify_assign (yb
, x
, ilist
);
4621 d
= TREE_OPERAND (d
, 0);
4622 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4623 d
= TREE_OPERAND (d
, 0);
4624 if (TREE_CODE (d
) == ADDR_EXPR
)
4626 if (orig_var
!= var
)
4628 gcc_assert (is_variable_sized (orig_var
));
4629 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4631 gimplify_assign (new_var
, x
, ilist
);
4632 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4633 tree t
= build_fold_indirect_ref (new_var
);
4634 DECL_IGNORED_P (new_var
) = 0;
4635 TREE_THIS_NOTRAP (t
);
4636 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4637 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4641 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4642 build_int_cst (ptype
, 0));
4643 SET_DECL_VALUE_EXPR (new_var
, x
);
4644 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4649 gcc_assert (orig_var
== var
);
4650 if (TREE_CODE (d
) == INDIRECT_REF
)
4652 x
= create_tmp_var (ptype
, name
);
4653 TREE_ADDRESSABLE (x
) = 1;
4654 gimplify_assign (x
, yb
, ilist
);
4655 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4657 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4658 gimplify_assign (new_var
, x
, ilist
);
4660 tree y1
= create_tmp_var (ptype
, NULL
);
4661 gimplify_assign (y1
, y
, ilist
);
4662 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4663 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4664 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4665 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4667 y2
= create_tmp_var (ptype
, NULL
);
4668 gimplify_assign (y2
, y
, ilist
);
4669 tree ref
= build_outer_var_ref (var
, ctx
);
4670 /* For ref build_outer_var_ref already performs this. */
4671 if (TREE_CODE (d
) == INDIRECT_REF
)
4672 gcc_assert (is_reference (var
));
4673 else if (TREE_CODE (d
) == ADDR_EXPR
)
4674 ref
= build_fold_addr_expr (ref
);
4675 else if (is_reference (var
))
4676 ref
= build_fold_addr_expr (ref
);
4677 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4678 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4679 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4681 y3
= create_tmp_var (ptype
, NULL
);
4682 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4686 y4
= create_tmp_var (ptype
, NULL
);
4687 gimplify_assign (y4
, ref
, dlist
);
4690 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4691 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4692 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4693 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4694 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4697 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4698 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4699 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4700 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4701 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4703 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4705 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4706 tree decl_placeholder
4707 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4708 SET_DECL_VALUE_EXPR (decl_placeholder
,
4709 build_simple_mem_ref (y1
));
4710 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4711 SET_DECL_VALUE_EXPR (placeholder
,
4712 y3
? build_simple_mem_ref (y3
)
4714 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4715 x
= lang_hooks
.decls
.omp_clause_default_ctor
4716 (c
, build_simple_mem_ref (y1
),
4717 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4719 gimplify_and_add (x
, ilist
);
4720 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4722 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4723 lower_omp (&tseq
, ctx
);
4724 gimple_seq_add_seq (ilist
, tseq
);
4726 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4729 SET_DECL_VALUE_EXPR (decl_placeholder
,
4730 build_simple_mem_ref (y2
));
4731 SET_DECL_VALUE_EXPR (placeholder
,
4732 build_simple_mem_ref (y4
));
4733 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4734 lower_omp (&tseq
, ctx
);
4735 gimple_seq_add_seq (dlist
, tseq
);
4736 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4738 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4739 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4740 x
= lang_hooks
.decls
.omp_clause_dtor
4741 (c
, build_simple_mem_ref (y2
));
4744 gimple_seq tseq
= NULL
;
4746 gimplify_stmt (&dtor
, &tseq
);
4747 gimple_seq_add_seq (dlist
, tseq
);
4752 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4753 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4755 /* reduction(-:var) sums up the partial results, so it
4756 acts identically to reduction(+:var). */
4757 if (code
== MINUS_EXPR
)
4760 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4763 x
= build2 (code
, TREE_TYPE (type
),
4764 build_simple_mem_ref (y4
),
4765 build_simple_mem_ref (y2
));
4766 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4770 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4771 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4772 gimple_seq_add_stmt (ilist
, g
);
4775 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4776 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4777 gimple_seq_add_stmt (ilist
, g
);
4779 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4780 build_int_cst (TREE_TYPE (i
), 1));
4781 gimple_seq_add_stmt (ilist
, g
);
4782 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4783 gimple_seq_add_stmt (ilist
, g
);
4784 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4787 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4788 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4789 gimple_seq_add_stmt (dlist
, g
);
4792 g
= gimple_build_assign
4793 (y4
, POINTER_PLUS_EXPR
, y4
,
4794 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4795 gimple_seq_add_stmt (dlist
, g
);
4797 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4798 build_int_cst (TREE_TYPE (i2
), 1));
4799 gimple_seq_add_stmt (dlist
, g
);
4800 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4801 gimple_seq_add_stmt (dlist
, g
);
4802 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4806 else if (is_variable_sized (var
))
4808 /* For variable sized types, we need to allocate the
4809 actual storage here. Call alloca and store the
4810 result in the pointer decl that we created elsewhere. */
4814 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4819 ptr
= DECL_VALUE_EXPR (new_var
);
4820 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4821 ptr
= TREE_OPERAND (ptr
, 0);
4822 gcc_assert (DECL_P (ptr
));
4823 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4825 /* void *tmp = __builtin_alloca */
4826 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4827 stmt
= gimple_build_call (atmp
, 2, x
,
4828 size_int (DECL_ALIGN (var
)));
4829 tmp
= create_tmp_var_raw (ptr_type_node
);
4830 gimple_add_tmp_var (tmp
);
4831 gimple_call_set_lhs (stmt
, tmp
);
4833 gimple_seq_add_stmt (ilist
, stmt
);
4835 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4836 gimplify_assign (ptr
, x
, ilist
);
4839 else if (is_reference (var
))
4841 /* For references that are being privatized for Fortran,
4842 allocate new backing storage for the new pointer
4843 variable. This allows us to avoid changing all the
4844 code that expects a pointer to something that expects
4845 a direct variable. */
4849 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4850 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4852 x
= build_receiver_ref (var
, false, ctx
);
4853 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4855 else if (TREE_CONSTANT (x
))
4857 /* For reduction in SIMD loop, defer adding the
4858 initialization of the reference, because if we decide
4859 to use SIMD array for it, the initilization could cause
4861 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4865 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4867 gimple_add_tmp_var (x
);
4868 TREE_ADDRESSABLE (x
) = 1;
4869 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4875 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4876 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4877 tree al
= size_int (TYPE_ALIGN (rtype
));
4878 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4883 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4884 gimplify_assign (new_var
, x
, ilist
);
4887 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4889 else if (c_kind
== OMP_CLAUSE_REDUCTION
4890 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4898 switch (OMP_CLAUSE_CODE (c
))
4900 case OMP_CLAUSE_SHARED
:
4901 /* Ignore shared directives in teams construct. */
4902 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4904 /* Shared global vars are just accessed directly. */
4905 if (is_global_var (new_var
))
4907 /* For taskloop firstprivate/lastprivate, represented
4908 as firstprivate and shared clause on the task, new_var
4909 is the firstprivate var. */
4910 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4912 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4913 needs to be delayed until after fixup_child_record_type so
4914 that we get the correct type during the dereference. */
4915 by_ref
= use_pointer_for_field (var
, ctx
);
4916 x
= build_receiver_ref (var
, by_ref
, ctx
);
4917 SET_DECL_VALUE_EXPR (new_var
, x
);
4918 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4920 /* ??? If VAR is not passed by reference, and the variable
4921 hasn't been initialized yet, then we'll get a warning for
4922 the store into the omp_data_s structure. Ideally, we'd be
4923 able to notice this and not store anything at all, but
4924 we're generating code too early. Suppress the warning. */
4926 TREE_NO_WARNING (var
) = 1;
4929 case OMP_CLAUSE_LASTPRIVATE
:
4930 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4934 case OMP_CLAUSE_PRIVATE
:
4935 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4936 x
= build_outer_var_ref (var
, ctx
);
4937 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4939 if (is_task_ctx (ctx
))
4940 x
= build_receiver_ref (var
, false, ctx
);
4942 x
= build_outer_var_ref (var
, ctx
);
4948 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4949 (c
, unshare_expr (new_var
), x
);
4952 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4953 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4954 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4955 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4956 idx
, lane
, ivar
, lvar
))
4959 x
= lang_hooks
.decls
.omp_clause_default_ctor
4960 (c
, unshare_expr (ivar
), x
);
4962 gimplify_and_add (x
, &llist
[0]);
4965 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4968 gimple_seq tseq
= NULL
;
4971 gimplify_stmt (&dtor
, &tseq
);
4972 gimple_seq_add_seq (&llist
[1], tseq
);
4979 gimplify_and_add (nx
, ilist
);
4983 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4986 gimple_seq tseq
= NULL
;
4989 gimplify_stmt (&dtor
, &tseq
);
4990 gimple_seq_add_seq (dlist
, tseq
);
4994 case OMP_CLAUSE_LINEAR
:
4995 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4996 goto do_firstprivate
;
4997 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
5000 x
= build_outer_var_ref (var
, ctx
);
5003 case OMP_CLAUSE_FIRSTPRIVATE
:
5004 if (is_task_ctx (ctx
))
5006 if (is_reference (var
) || is_variable_sized (var
))
5008 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
5010 || use_pointer_for_field (var
, NULL
))
5012 x
= build_receiver_ref (var
, false, ctx
);
5013 SET_DECL_VALUE_EXPR (new_var
, x
);
5014 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5019 x
= build_outer_var_ref (var
, ctx
);
5022 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5023 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5025 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5026 tree stept
= TREE_TYPE (t
);
5027 tree ct
= find_omp_clause (clauses
,
5028 OMP_CLAUSE__LOOPTEMP_
);
5030 tree l
= OMP_CLAUSE_DECL (ct
);
5031 tree n1
= fd
->loop
.n1
;
5032 tree step
= fd
->loop
.step
;
5033 tree itype
= TREE_TYPE (l
);
5034 if (POINTER_TYPE_P (itype
))
5035 itype
= signed_type_for (itype
);
5036 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5037 if (TYPE_UNSIGNED (itype
)
5038 && fd
->loop
.cond_code
== GT_EXPR
)
5039 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5040 fold_build1 (NEGATE_EXPR
, itype
, l
),
5041 fold_build1 (NEGATE_EXPR
,
5044 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5045 t
= fold_build2 (MULT_EXPR
, stept
,
5046 fold_convert (stept
, l
), t
);
5048 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5050 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5052 gimplify_and_add (x
, ilist
);
5056 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5057 x
= fold_build2 (POINTER_PLUS_EXPR
,
5058 TREE_TYPE (x
), x
, t
);
5060 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5063 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5064 || TREE_ADDRESSABLE (new_var
))
5065 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5066 idx
, lane
, ivar
, lvar
))
5068 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5070 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5071 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5072 gimplify_and_add (x
, ilist
);
5073 gimple_stmt_iterator gsi
5074 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5076 = gimple_build_assign (unshare_expr (lvar
), iv
);
5077 gsi_insert_before_without_update (&gsi
, g
,
5079 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5080 enum tree_code code
= PLUS_EXPR
;
5081 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5082 code
= POINTER_PLUS_EXPR
;
5083 g
= gimple_build_assign (iv
, code
, iv
, t
);
5084 gsi_insert_before_without_update (&gsi
, g
,
5088 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5089 (c
, unshare_expr (ivar
), x
);
5090 gimplify_and_add (x
, &llist
[0]);
5091 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5094 gimple_seq tseq
= NULL
;
5097 gimplify_stmt (&dtor
, &tseq
);
5098 gimple_seq_add_seq (&llist
[1], tseq
);
5103 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5104 (c
, unshare_expr (new_var
), x
);
5105 gimplify_and_add (x
, ilist
);
5108 case OMP_CLAUSE__LOOPTEMP_
:
5109 gcc_assert (is_taskreg_ctx (ctx
));
5110 x
= build_outer_var_ref (var
, ctx
);
5111 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5112 gimplify_and_add (x
, ilist
);
5115 case OMP_CLAUSE_COPYIN
:
5116 by_ref
= use_pointer_for_field (var
, NULL
);
5117 x
= build_receiver_ref (var
, by_ref
, ctx
);
5118 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5119 append_to_statement_list (x
, ©in_seq
);
5120 copyin_by_ref
|= by_ref
;
5123 case OMP_CLAUSE_REDUCTION
:
5124 /* OpenACC reductions are initialized using the
5125 GOACC_REDUCTION internal function. */
5126 if (is_gimple_omp_oacc (ctx
->stmt
))
5128 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5130 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5132 x
= build_outer_var_ref (var
, ctx
);
5134 if (is_reference (var
)
5135 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5137 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5138 SET_DECL_VALUE_EXPR (placeholder
, x
);
5139 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5140 tree new_vard
= new_var
;
5141 if (is_reference (var
))
5143 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5144 new_vard
= TREE_OPERAND (new_var
, 0);
5145 gcc_assert (DECL_P (new_vard
));
5148 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5149 idx
, lane
, ivar
, lvar
))
5151 if (new_vard
== new_var
)
5153 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5154 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5158 SET_DECL_VALUE_EXPR (new_vard
,
5159 build_fold_addr_expr (ivar
));
5160 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5162 x
= lang_hooks
.decls
.omp_clause_default_ctor
5163 (c
, unshare_expr (ivar
),
5164 build_outer_var_ref (var
, ctx
));
5166 gimplify_and_add (x
, &llist
[0]);
5167 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5169 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5170 lower_omp (&tseq
, ctx
);
5171 gimple_seq_add_seq (&llist
[0], tseq
);
5173 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5174 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5175 lower_omp (&tseq
, ctx
);
5176 gimple_seq_add_seq (&llist
[1], tseq
);
5177 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5178 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5179 if (new_vard
== new_var
)
5180 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5182 SET_DECL_VALUE_EXPR (new_vard
,
5183 build_fold_addr_expr (lvar
));
5184 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5189 gimplify_stmt (&dtor
, &tseq
);
5190 gimple_seq_add_seq (&llist
[1], tseq
);
5194 /* If this is a reference to constant size reduction var
5195 with placeholder, we haven't emitted the initializer
5196 for it because it is undesirable if SIMD arrays are used.
5197 But if they aren't used, we need to emit the deferred
5198 initialization now. */
5199 else if (is_reference (var
) && is_simd
)
5200 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5201 x
= lang_hooks
.decls
.omp_clause_default_ctor
5202 (c
, unshare_expr (new_var
),
5203 build_outer_var_ref (var
, ctx
));
5205 gimplify_and_add (x
, ilist
);
5206 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5208 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5209 lower_omp (&tseq
, ctx
);
5210 gimple_seq_add_seq (ilist
, tseq
);
5212 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5215 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5216 lower_omp (&tseq
, ctx
);
5217 gimple_seq_add_seq (dlist
, tseq
);
5218 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5220 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5225 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5226 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5227 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5229 /* reduction(-:var) sums up the partial results, so it
5230 acts identically to reduction(+:var). */
5231 if (code
== MINUS_EXPR
)
5234 tree new_vard
= new_var
;
5235 if (is_simd
&& is_reference (var
))
5237 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5238 new_vard
= TREE_OPERAND (new_var
, 0);
5239 gcc_assert (DECL_P (new_vard
));
5242 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5243 idx
, lane
, ivar
, lvar
))
5245 tree ref
= build_outer_var_ref (var
, ctx
);
5247 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5249 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5250 ref
= build_outer_var_ref (var
, ctx
);
5251 gimplify_assign (ref
, x
, &llist
[1]);
5253 if (new_vard
!= new_var
)
5255 SET_DECL_VALUE_EXPR (new_vard
,
5256 build_fold_addr_expr (lvar
));
5257 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5262 if (is_reference (var
) && is_simd
)
5263 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5264 gimplify_assign (new_var
, x
, ilist
);
5267 tree ref
= build_outer_var_ref (var
, ctx
);
5269 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5270 ref
= build_outer_var_ref (var
, ctx
);
5271 gimplify_assign (ref
, x
, dlist
);
5285 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5286 /* Don't want uninit warnings on simduid, it is always uninitialized,
5287 but we use it not for the value, but for the DECL_UID only. */
5288 TREE_NO_WARNING (uid
) = 1;
5290 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5291 gimple_call_set_lhs (g
, lane
);
5292 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5293 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5294 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5295 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5296 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5297 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5298 g
= gimple_build_assign (lane
, INTEGER_CST
,
5299 build_int_cst (unsigned_type_node
, 0));
5300 gimple_seq_add_stmt (ilist
, g
);
5301 for (int i
= 0; i
< 2; i
++)
5304 tree vf
= create_tmp_var (unsigned_type_node
);
5305 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5306 gimple_call_set_lhs (g
, vf
);
5307 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5308 gimple_seq_add_stmt (seq
, g
);
5309 tree t
= build_int_cst (unsigned_type_node
, 0);
5310 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5311 gimple_seq_add_stmt (seq
, g
);
5312 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5313 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5314 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5315 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5316 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5317 gimple_seq_add_seq (seq
, llist
[i
]);
5318 t
= build_int_cst (unsigned_type_node
, 1);
5319 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5320 gimple_seq_add_stmt (seq
, g
);
5321 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5322 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5323 gimple_seq_add_stmt (seq
, g
);
5324 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5328 /* The copyin sequence is not to be executed by the main thread, since
5329 that would result in self-copies. Perhaps not visible to scalars,
5330 but it certainly is to C++ operator=. */
5333 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5335 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5336 build_int_cst (TREE_TYPE (x
), 0));
5337 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5338 gimplify_and_add (x
, ilist
);
5341 /* If any copyin variable is passed by reference, we must ensure the
5342 master thread doesn't modify it before it is copied over in all
5343 threads. Similarly for variables in both firstprivate and
5344 lastprivate clauses we need to ensure the lastprivate copying
5345 happens after firstprivate copying in all threads. And similarly
5346 for UDRs if initializer expression refers to omp_orig. */
5347 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5349 /* Don't add any barrier for #pragma omp simd or
5350 #pragma omp distribute. */
5351 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5352 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5353 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5356 /* If max_vf is non-zero, then we can use only a vectorization factor
5357 up to the max_vf we chose. So stick it into the safelen clause. */
5360 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5361 OMP_CLAUSE_SAFELEN
);
5363 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5364 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5367 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5368 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5370 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5371 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5377 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5378 both parallel and workshare constructs. PREDICATE may be NULL if it's
5382 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5385 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5386 bool par_clauses
= false;
5387 tree simduid
= NULL
, lastlane
= NULL
;
5389 /* Early exit if there are no lastprivate or linear clauses. */
5390 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5391 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5392 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5393 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5395 if (clauses
== NULL
)
5397 /* If this was a workshare clause, see if it had been combined
5398 with its parallel. In that case, look for the clauses on the
5399 parallel statement itself. */
5400 if (is_parallel_ctx (ctx
))
5404 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5407 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5408 OMP_CLAUSE_LASTPRIVATE
);
5409 if (clauses
== NULL
)
5417 tree label_true
, arm1
, arm2
;
5419 label
= create_artificial_label (UNKNOWN_LOCATION
);
5420 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5421 arm1
= TREE_OPERAND (predicate
, 0);
5422 arm2
= TREE_OPERAND (predicate
, 1);
5423 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5424 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5425 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5427 gimple_seq_add_stmt (stmt_list
, stmt
);
5428 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5431 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5432 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5434 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5436 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5439 for (c
= clauses
; c
;)
5442 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5444 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5445 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5446 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5448 var
= OMP_CLAUSE_DECL (c
);
5449 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5450 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5451 && is_taskloop_ctx (ctx
))
5453 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5454 new_var
= lookup_decl (var
, ctx
->outer
);
5457 new_var
= lookup_decl (var
, ctx
);
5459 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5461 tree val
= DECL_VALUE_EXPR (new_var
);
5462 if (TREE_CODE (val
) == ARRAY_REF
5463 && VAR_P (TREE_OPERAND (val
, 0))
5464 && lookup_attribute ("omp simd array",
5465 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5468 if (lastlane
== NULL
)
5470 lastlane
= create_tmp_var (unsigned_type_node
);
5472 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5474 TREE_OPERAND (val
, 1));
5475 gimple_call_set_lhs (g
, lastlane
);
5476 gimple_seq_add_stmt (stmt_list
, g
);
5478 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5479 TREE_OPERAND (val
, 0), lastlane
,
5480 NULL_TREE
, NULL_TREE
);
5484 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5485 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5487 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5488 gimple_seq_add_seq (stmt_list
,
5489 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5490 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5492 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5493 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5495 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5496 gimple_seq_add_seq (stmt_list
,
5497 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5498 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5502 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5503 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5505 gcc_checking_assert (is_taskloop_ctx (ctx
));
5506 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5508 if (is_global_var (ovar
))
5512 x
= build_outer_var_ref (var
, ctx
, true);
5513 if (is_reference (var
))
5514 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5515 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5516 gimplify_and_add (x
, stmt_list
);
5518 c
= OMP_CLAUSE_CHAIN (c
);
5519 if (c
== NULL
&& !par_clauses
)
5521 /* If this was a workshare clause, see if it had been combined
5522 with its parallel. In that case, continue looking for the
5523 clauses also on the parallel statement itself. */
5524 if (is_parallel_ctx (ctx
))
5528 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5531 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5532 OMP_CLAUSE_LASTPRIVATE
);
5538 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5541 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5542 (which might be a placeholder). INNER is true if this is an inner
5543 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5544 join markers. Generate the before-loop forking sequence in
5545 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5546 general form of these sequences is
5548 GOACC_REDUCTION_SETUP
5550 GOACC_REDUCTION_INIT
5552 GOACC_REDUCTION_FINI
5554 GOACC_REDUCTION_TEARDOWN. */
5557 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5558 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5559 gimple_seq
*join_seq
, omp_context
*ctx
)
5561 gimple_seq before_fork
= NULL
;
5562 gimple_seq after_fork
= NULL
;
5563 gimple_seq before_join
= NULL
;
5564 gimple_seq after_join
= NULL
;
5565 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5566 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5567 unsigned offset
= 0;
5569 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5570 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5572 tree orig
= OMP_CLAUSE_DECL (c
);
5573 tree var
= maybe_lookup_decl (orig
, ctx
);
5574 tree ref_to_res
= NULL_TREE
;
5575 tree incoming
, outgoing
, v1
, v2
, v3
;
5576 bool is_private
= false;
5578 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5579 if (rcode
== MINUS_EXPR
)
5581 else if (rcode
== TRUTH_ANDIF_EXPR
)
5582 rcode
= BIT_AND_EXPR
;
5583 else if (rcode
== TRUTH_ORIF_EXPR
)
5584 rcode
= BIT_IOR_EXPR
;
5585 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5590 incoming
= outgoing
= var
;
5594 /* See if an outer construct also reduces this variable. */
5595 omp_context
*outer
= ctx
;
5597 while (omp_context
*probe
= outer
->outer
)
5599 enum gimple_code type
= gimple_code (probe
->stmt
);
5604 case GIMPLE_OMP_FOR
:
5605 cls
= gimple_omp_for_clauses (probe
->stmt
);
5608 case GIMPLE_OMP_TARGET
:
5609 if (gimple_omp_target_kind (probe
->stmt
)
5610 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5613 cls
= gimple_omp_target_clauses (probe
->stmt
);
5621 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5622 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5623 && orig
== OMP_CLAUSE_DECL (cls
))
5625 incoming
= outgoing
= lookup_decl (orig
, probe
);
5626 goto has_outer_reduction
;
5628 else if ((OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_FIRSTPRIVATE
5629 || OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_PRIVATE
)
5630 && orig
== OMP_CLAUSE_DECL (cls
))
5638 /* This is the outermost construct with this reduction,
5639 see if there's a mapping for it. */
5640 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5641 && maybe_lookup_field (orig
, outer
) && !is_private
)
5643 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5644 if (is_reference (orig
))
5645 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5647 tree type
= TREE_TYPE (var
);
5648 if (POINTER_TYPE_P (type
))
5649 type
= TREE_TYPE (type
);
5652 incoming
= omp_reduction_init_op (loc
, rcode
, type
);
5654 else if (ctx
->outer
)
5655 incoming
= outgoing
= lookup_decl (orig
, ctx
->outer
);
5657 incoming
= outgoing
= orig
;
5659 has_outer_reduction
:;
5663 ref_to_res
= integer_zero_node
;
5665 if (is_reference (orig
))
5667 tree type
= TREE_TYPE (var
);
5668 const char *id
= IDENTIFIER_POINTER (DECL_NAME (var
));
5672 tree x
= create_tmp_var (TREE_TYPE (type
), id
);
5673 gimplify_assign (var
, build_fold_addr_expr (x
), fork_seq
);
5676 v1
= create_tmp_var (type
, id
);
5677 v2
= create_tmp_var (type
, id
);
5678 v3
= create_tmp_var (type
, id
);
5680 gimplify_assign (v1
, var
, fork_seq
);
5681 gimplify_assign (v2
, var
, fork_seq
);
5682 gimplify_assign (v3
, var
, fork_seq
);
5684 var
= build_simple_mem_ref (var
);
5685 v1
= build_simple_mem_ref (v1
);
5686 v2
= build_simple_mem_ref (v2
);
5687 v3
= build_simple_mem_ref (v3
);
5688 outgoing
= build_simple_mem_ref (outgoing
);
5690 if (!TREE_CONSTANT (incoming
))
5691 incoming
= build_simple_mem_ref (incoming
);
5696 /* Determine position in reduction buffer, which may be used
5698 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5699 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5700 offset
= (offset
+ align
- 1) & ~(align
- 1);
5701 tree off
= build_int_cst (sizetype
, offset
);
5702 offset
+= GET_MODE_SIZE (mode
);
5706 init_code
= build_int_cst (integer_type_node
,
5707 IFN_GOACC_REDUCTION_INIT
);
5708 fini_code
= build_int_cst (integer_type_node
,
5709 IFN_GOACC_REDUCTION_FINI
);
5710 setup_code
= build_int_cst (integer_type_node
,
5711 IFN_GOACC_REDUCTION_SETUP
);
5712 teardown_code
= build_int_cst (integer_type_node
,
5713 IFN_GOACC_REDUCTION_TEARDOWN
);
5717 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5718 TREE_TYPE (var
), 6, setup_code
,
5719 unshare_expr (ref_to_res
),
5720 incoming
, level
, op
, off
);
5722 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5723 TREE_TYPE (var
), 6, init_code
,
5724 unshare_expr (ref_to_res
),
5725 v1
, level
, op
, off
);
5727 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5728 TREE_TYPE (var
), 6, fini_code
,
5729 unshare_expr (ref_to_res
),
5730 v2
, level
, op
, off
);
5732 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5733 TREE_TYPE (var
), 6, teardown_code
,
5734 ref_to_res
, v3
, level
, op
, off
);
5736 gimplify_assign (v1
, setup_call
, &before_fork
);
5737 gimplify_assign (v2
, init_call
, &after_fork
);
5738 gimplify_assign (v3
, fini_call
, &before_join
);
5739 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5742 /* Now stitch things together. */
5743 gimple_seq_add_seq (fork_seq
, before_fork
);
5745 gimple_seq_add_stmt (fork_seq
, fork
);
5746 gimple_seq_add_seq (fork_seq
, after_fork
);
5748 gimple_seq_add_seq (join_seq
, before_join
);
5750 gimple_seq_add_stmt (join_seq
, join
);
5751 gimple_seq_add_seq (join_seq
, after_join
);
5754 /* Generate code to implement the REDUCTION clauses. */
5757 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5759 gimple_seq sub_seq
= NULL
;
5764 /* OpenACC loop reductions are handled elsewhere. */
5765 if (is_gimple_omp_oacc (ctx
->stmt
))
5768 /* SIMD reductions are handled in lower_rec_input_clauses. */
5769 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5770 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5773 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5774 update in that case, otherwise use a lock. */
5775 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5776 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5778 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5779 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5781 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5791 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5793 tree var
, ref
, new_var
, orig_var
;
5794 enum tree_code code
;
5795 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5797 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5800 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5801 if (TREE_CODE (var
) == MEM_REF
)
5803 var
= TREE_OPERAND (var
, 0);
5804 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5805 var
= TREE_OPERAND (var
, 0);
5806 if (TREE_CODE (var
) == INDIRECT_REF
5807 || TREE_CODE (var
) == ADDR_EXPR
)
5808 var
= TREE_OPERAND (var
, 0);
5810 if (is_variable_sized (var
))
5812 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5813 var
= DECL_VALUE_EXPR (var
);
5814 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5815 var
= TREE_OPERAND (var
, 0);
5816 gcc_assert (DECL_P (var
));
5819 new_var
= lookup_decl (var
, ctx
);
5820 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5821 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5822 ref
= build_outer_var_ref (var
, ctx
);
5823 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5825 /* reduction(-:var) sums up the partial results, so it acts
5826 identically to reduction(+:var). */
5827 if (code
== MINUS_EXPR
)
5832 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5834 addr
= save_expr (addr
);
5835 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5836 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5837 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5838 gimplify_and_add (x
, stmt_seqp
);
5841 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5843 tree d
= OMP_CLAUSE_DECL (c
);
5844 tree type
= TREE_TYPE (d
);
5845 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5846 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5847 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5848 tree bias
= TREE_OPERAND (d
, 1);
5849 d
= TREE_OPERAND (d
, 0);
5850 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5852 tree b
= TREE_OPERAND (d
, 1);
5853 b
= maybe_lookup_decl (b
, ctx
);
5856 b
= TREE_OPERAND (d
, 1);
5857 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5859 if (integer_zerop (bias
))
5863 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5864 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5865 TREE_TYPE (b
), b
, bias
);
5867 d
= TREE_OPERAND (d
, 0);
5869 /* For ref build_outer_var_ref already performs this, so
5870 only new_var needs a dereference. */
5871 if (TREE_CODE (d
) == INDIRECT_REF
)
5873 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5874 gcc_assert (is_reference (var
) && var
== orig_var
);
5876 else if (TREE_CODE (d
) == ADDR_EXPR
)
5878 if (orig_var
== var
)
5880 new_var
= build_fold_addr_expr (new_var
);
5881 ref
= build_fold_addr_expr (ref
);
5886 gcc_assert (orig_var
== var
);
5887 if (is_reference (var
))
5888 ref
= build_fold_addr_expr (ref
);
5892 tree t
= maybe_lookup_decl (v
, ctx
);
5896 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5897 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5899 if (!integer_zerop (bias
))
5901 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5902 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5903 TREE_TYPE (new_var
), new_var
,
5904 unshare_expr (bias
));
5905 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5906 TREE_TYPE (ref
), ref
, bias
);
5908 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5909 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5910 tree m
= create_tmp_var (ptype
, NULL
);
5911 gimplify_assign (m
, new_var
, stmt_seqp
);
5913 m
= create_tmp_var (ptype
, NULL
);
5914 gimplify_assign (m
, ref
, stmt_seqp
);
5916 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5917 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5918 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5919 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5920 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5921 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5922 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5924 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5925 tree decl_placeholder
5926 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5927 SET_DECL_VALUE_EXPR (placeholder
, out
);
5928 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5929 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5930 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5931 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5932 gimple_seq_add_seq (&sub_seq
,
5933 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5934 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5935 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5936 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5940 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5941 out
= unshare_expr (out
);
5942 gimplify_assign (out
, x
, &sub_seq
);
5944 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5945 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5946 gimple_seq_add_stmt (&sub_seq
, g
);
5947 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5948 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5949 gimple_seq_add_stmt (&sub_seq
, g
);
5950 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5951 build_int_cst (TREE_TYPE (i
), 1));
5952 gimple_seq_add_stmt (&sub_seq
, g
);
5953 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5954 gimple_seq_add_stmt (&sub_seq
, g
);
5955 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5957 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5959 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5961 if (is_reference (var
)
5962 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5964 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5965 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5966 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5967 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5968 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5969 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5970 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5974 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5975 ref
= build_outer_var_ref (var
, ctx
);
5976 gimplify_assign (ref
, x
, &sub_seq
);
5980 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5982 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5984 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5986 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5988 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5992 /* Generate code to implement the COPYPRIVATE clauses. */
5995 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
6000 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6002 tree var
, new_var
, ref
, x
;
6004 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6006 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
6009 var
= OMP_CLAUSE_DECL (c
);
6010 by_ref
= use_pointer_for_field (var
, NULL
);
6012 ref
= build_sender_ref (var
, ctx
);
6013 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
6016 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
6017 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
6019 gimplify_assign (ref
, x
, slist
);
6021 ref
= build_receiver_ref (var
, false, ctx
);
6024 ref
= fold_convert_loc (clause_loc
,
6025 build_pointer_type (TREE_TYPE (new_var
)),
6027 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
6029 if (is_reference (var
))
6031 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
6032 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
6033 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
6035 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
6036 gimplify_and_add (x
, rlist
);
6041 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
6042 and REDUCTION from the sender (aka parent) side. */
6045 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
6049 int ignored_looptemp
= 0;
6050 bool is_taskloop
= false;
6052 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
6053 by GOMP_taskloop. */
6054 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
6056 ignored_looptemp
= 2;
6060 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6062 tree val
, ref
, x
, var
;
6063 bool by_ref
, do_in
= false, do_out
= false;
6064 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6066 switch (OMP_CLAUSE_CODE (c
))
6068 case OMP_CLAUSE_PRIVATE
:
6069 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
6072 case OMP_CLAUSE_FIRSTPRIVATE
:
6073 case OMP_CLAUSE_COPYIN
:
6074 case OMP_CLAUSE_LASTPRIVATE
:
6075 case OMP_CLAUSE_REDUCTION
:
6077 case OMP_CLAUSE_SHARED
:
6078 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6081 case OMP_CLAUSE__LOOPTEMP_
:
6082 if (ignored_looptemp
)
6092 val
= OMP_CLAUSE_DECL (c
);
6093 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6094 && TREE_CODE (val
) == MEM_REF
)
6096 val
= TREE_OPERAND (val
, 0);
6097 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
6098 val
= TREE_OPERAND (val
, 0);
6099 if (TREE_CODE (val
) == INDIRECT_REF
6100 || TREE_CODE (val
) == ADDR_EXPR
)
6101 val
= TREE_OPERAND (val
, 0);
6102 if (is_variable_sized (val
))
6106 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6107 outer taskloop region. */
6108 omp_context
*ctx_for_o
= ctx
;
6110 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
6111 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6112 ctx_for_o
= ctx
->outer
;
6114 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
6116 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
6117 && is_global_var (var
))
6120 t
= omp_member_access_dummy_var (var
);
6123 var
= DECL_VALUE_EXPR (var
);
6124 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
6126 var
= unshare_and_remap (var
, t
, o
);
6128 var
= unshare_expr (var
);
6131 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
6133 /* Handle taskloop firstprivate/lastprivate, where the
6134 lastprivate on GIMPLE_OMP_TASK is represented as
6135 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6136 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
6137 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
6138 if (use_pointer_for_field (val
, ctx
))
6139 var
= build_fold_addr_expr (var
);
6140 gimplify_assign (x
, var
, ilist
);
6141 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
6145 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6146 || val
== OMP_CLAUSE_DECL (c
))
6147 && is_variable_sized (val
))
6149 by_ref
= use_pointer_for_field (val
, NULL
);
6151 switch (OMP_CLAUSE_CODE (c
))
6153 case OMP_CLAUSE_FIRSTPRIVATE
:
6154 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
)
6156 && is_task_ctx (ctx
))
6157 TREE_NO_WARNING (var
) = 1;
6161 case OMP_CLAUSE_PRIVATE
:
6162 case OMP_CLAUSE_COPYIN
:
6163 case OMP_CLAUSE__LOOPTEMP_
:
6167 case OMP_CLAUSE_LASTPRIVATE
:
6168 if (by_ref
|| is_reference (val
))
6170 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
6177 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
6182 case OMP_CLAUSE_REDUCTION
:
6184 if (val
== OMP_CLAUSE_DECL (c
))
6185 do_out
= !(by_ref
|| is_reference (val
));
6187 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
6196 ref
= build_sender_ref (val
, ctx
);
6197 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
6198 gimplify_assign (ref
, x
, ilist
);
6199 if (is_task_ctx (ctx
))
6200 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
6205 ref
= build_sender_ref (val
, ctx
);
6206 gimplify_assign (var
, ref
, olist
);
6211 /* Generate code to implement SHARED from the sender (aka parent)
6212 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6213 list things that got automatically shared. */
6216 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6218 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6220 if (ctx
->record_type
== NULL
)
6223 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6224 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6226 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6227 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6230 nvar
= maybe_lookup_decl (ovar
, ctx
);
6231 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6234 /* If CTX is a nested parallel directive. Find the immediately
6235 enclosing parallel or workshare construct that contains a
6236 mapping for OVAR. */
6237 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6239 t
= omp_member_access_dummy_var (var
);
6242 var
= DECL_VALUE_EXPR (var
);
6243 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6245 var
= unshare_and_remap (var
, t
, o
);
6247 var
= unshare_expr (var
);
6250 if (use_pointer_for_field (ovar
, ctx
))
6252 x
= build_sender_ref (ovar
, ctx
);
6253 var
= build_fold_addr_expr (var
);
6254 gimplify_assign (x
, var
, ilist
);
6258 x
= build_sender_ref (ovar
, ctx
);
6259 gimplify_assign (x
, var
, ilist
);
6261 if (!TREE_READONLY (var
)
6262 /* We don't need to receive a new reference to a result
6263 or parm decl. In fact we may not store to it as we will
6264 invalidate any pending RSO and generate wrong gimple
6266 && !((TREE_CODE (var
) == RESULT_DECL
6267 || TREE_CODE (var
) == PARM_DECL
)
6268 && DECL_BY_REFERENCE (var
)))
6270 x
= build_sender_ref (ovar
, ctx
);
6271 gimplify_assign (var
, x
, olist
);
6277 /* Emit an OpenACC head marker call, encapulating the partitioning and
6278 other information that must be processed by the target compiler.
6279 Return the maximum number of dimensions the associated loop might
6280 be partitioned over. */
6283 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6284 gimple_seq
*seq
, omp_context
*ctx
)
6286 unsigned levels
= 0;
6288 tree gang_static
= NULL_TREE
;
6289 auto_vec
<tree
, 5> args
;
6291 args
.quick_push (build_int_cst
6292 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6293 args
.quick_push (ddvar
);
6294 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6296 switch (OMP_CLAUSE_CODE (c
))
6298 case OMP_CLAUSE_GANG
:
6299 tag
|= OLF_DIM_GANG
;
6300 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6301 /* static:* is represented by -1, and we can ignore it, as
6302 scheduling is always static. */
6303 if (gang_static
&& integer_minus_onep (gang_static
))
6304 gang_static
= NULL_TREE
;
6308 case OMP_CLAUSE_WORKER
:
6309 tag
|= OLF_DIM_WORKER
;
6313 case OMP_CLAUSE_VECTOR
:
6314 tag
|= OLF_DIM_VECTOR
;
6318 case OMP_CLAUSE_SEQ
:
6322 case OMP_CLAUSE_AUTO
:
6326 case OMP_CLAUSE_INDEPENDENT
:
6327 tag
|= OLF_INDEPENDENT
;
6337 if (DECL_P (gang_static
))
6338 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6339 tag
|= OLF_GANG_STATIC
;
6342 /* In a parallel region, loops are implicitly INDEPENDENT. */
6343 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6344 if (!tgt
|| is_oacc_parallel (tgt
))
6345 tag
|= OLF_INDEPENDENT
;
6347 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6348 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6352 /* Ensure at least one level. */
6356 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6357 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6359 args
.quick_push (gang_static
);
6361 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6362 gimple_set_location (call
, loc
);
6363 gimple_set_lhs (call
, ddvar
);
6364 gimple_seq_add_stmt (seq
, call
);
6369 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6370 partitioning level of the enclosed region. */
6373 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6374 tree tofollow
, gimple_seq
*seq
)
6376 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6377 : IFN_UNIQUE_OACC_TAIL_MARK
);
6378 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6379 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6380 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6381 marker
, ddvar
, tofollow
);
6382 gimple_set_location (call
, loc
);
6383 gimple_set_lhs (call
, ddvar
);
6384 gimple_seq_add_stmt (seq
, call
);
6387 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6388 the loop clauses, from which we extract reductions. Initialize
6392 lower_oacc_head_tail (location_t loc
, tree clauses
,
6393 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6396 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6397 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6399 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6400 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6401 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6404 for (unsigned done
= 1; count
; count
--, done
++)
6406 gimple_seq fork_seq
= NULL
;
6407 gimple_seq join_seq
= NULL
;
6409 tree place
= build_int_cst (integer_type_node
, -1);
6410 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6411 fork_kind
, ddvar
, place
);
6412 gimple_set_location (fork
, loc
);
6413 gimple_set_lhs (fork
, ddvar
);
6415 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6416 join_kind
, ddvar
, place
);
6417 gimple_set_location (join
, loc
);
6418 gimple_set_lhs (join
, ddvar
);
6420 /* Mark the beginning of this level sequence. */
6422 lower_oacc_loop_marker (loc
, ddvar
, true,
6423 build_int_cst (integer_type_node
, count
),
6425 lower_oacc_loop_marker (loc
, ddvar
, false,
6426 build_int_cst (integer_type_node
, done
),
6429 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6430 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6432 /* Append this level to head. */
6433 gimple_seq_add_seq (head
, fork_seq
);
6434 /* Prepend it to tail. */
6435 gimple_seq_add_seq (&join_seq
, *tail
);
6441 /* Mark the end of the sequence. */
6442 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6443 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6446 /* A convenience function to build an empty GIMPLE_COND with just the
6450 gimple_build_cond_empty (tree cond
)
6452 enum tree_code pred_code
;
6455 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6456 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6459 /* Return true if a parallel REGION is within a declare target function or
6460 within a target region and is not a part of a gridified target. */
6463 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
6465 bool indirect
= false;
6466 for (region
= region
->outer
; region
; region
= region
->outer
)
6468 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6470 else if (region
->type
== GIMPLE_OMP_TARGET
)
6472 gomp_target
*tgt_stmt
6473 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
6475 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
6476 OMP_CLAUSE__GRIDDIM_
))
6483 if (lookup_attribute ("omp declare target",
6484 DECL_ATTRIBUTES (current_function_decl
)))
6490 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6493 /* Build the function calls to GOMP_parallel_start etc to actually
6494 generate the parallel operation. REGION is the parallel region
6495 being expanded. BB is the block where to insert the code. WS_ARGS
6496 will be set if this is a call to a combined parallel+workshare
6497 construct, it contains the list of additional arguments needed by
6498 the workshare construct. */
6501 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6502 gomp_parallel
*entry_stmt
,
6503 vec
<tree
, va_gc
> *ws_args
)
6505 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6506 gimple_stmt_iterator gsi
;
6508 enum built_in_function start_ix
;
6510 location_t clause_loc
;
6511 vec
<tree
, va_gc
> *args
;
6513 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6515 /* Determine what flavor of GOMP_parallel we will be
6517 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6518 if (is_combined_parallel (region
))
6520 switch (region
->inner
->type
)
6522 case GIMPLE_OMP_FOR
:
6523 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6524 switch (region
->inner
->sched_kind
)
6526 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6529 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6530 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6531 if (region
->inner
->sched_modifiers
6532 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6534 start_ix2
= 3 + region
->inner
->sched_kind
;
6539 start_ix2
= region
->inner
->sched_kind
;
6542 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6543 start_ix
= (enum built_in_function
) start_ix2
;
6545 case GIMPLE_OMP_SECTIONS
:
6546 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6553 /* By default, the value of NUM_THREADS is zero (selected at run time)
6554 and there is no conditional. */
6556 val
= build_int_cst (unsigned_type_node
, 0);
6557 flags
= build_int_cst (unsigned_type_node
, 0);
6559 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6561 cond
= OMP_CLAUSE_IF_EXPR (c
);
6563 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6566 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6567 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6570 clause_loc
= gimple_location (entry_stmt
);
6572 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6574 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6576 /* Ensure 'val' is of the correct type. */
6577 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6579 /* If we found the clause 'if (cond)', build either
6580 (cond != 0) or (cond ? val : 1u). */
6583 cond
= gimple_boolify (cond
);
6585 if (integer_zerop (val
))
6586 val
= fold_build2_loc (clause_loc
,
6587 EQ_EXPR
, unsigned_type_node
, cond
,
6588 build_int_cst (TREE_TYPE (cond
), 0));
6591 basic_block cond_bb
, then_bb
, else_bb
;
6592 edge e
, e_then
, e_else
;
6593 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6595 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6596 if (gimple_in_ssa_p (cfun
))
6598 tmp_then
= make_ssa_name (tmp_var
);
6599 tmp_else
= make_ssa_name (tmp_var
);
6600 tmp_join
= make_ssa_name (tmp_var
);
6609 e
= split_block_after_labels (bb
);
6614 then_bb
= create_empty_bb (cond_bb
);
6615 else_bb
= create_empty_bb (then_bb
);
6616 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6617 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6619 stmt
= gimple_build_cond_empty (cond
);
6620 gsi
= gsi_start_bb (cond_bb
);
6621 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6623 gsi
= gsi_start_bb (then_bb
);
6624 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6626 gsi
= gsi_start_bb (else_bb
);
6627 expand_omp_build_assign (&gsi
, tmp_else
,
6628 build_int_cst (unsigned_type_node
, 1),
6631 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6632 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6633 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6634 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6635 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6636 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6638 if (gimple_in_ssa_p (cfun
))
6640 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6641 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6642 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6648 gsi
= gsi_start_bb (bb
);
6649 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6650 false, GSI_CONTINUE_LINKING
);
6653 gsi
= gsi_last_bb (bb
);
6654 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6656 t1
= null_pointer_node
;
6658 t1
= build_fold_addr_expr (t
);
6659 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
6660 t2
= build_fold_addr_expr (child_fndecl
);
6662 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6663 args
->quick_push (t2
);
6664 args
->quick_push (t1
);
6665 args
->quick_push (val
);
6667 args
->splice (*ws_args
);
6668 args
->quick_push (flags
);
6670 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6671 builtin_decl_explicit (start_ix
), args
);
6673 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6674 false, GSI_CONTINUE_LINKING
);
6676 if (hsa_gen_requested_p ()
6677 && parallel_needs_hsa_kernel_p (region
))
6679 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
6680 hsa_register_kernel (child_cnode
);
6684 /* Insert a function call whose name is FUNC_NAME with the information from
6685 ENTRY_STMT into the basic_block BB. */
6688 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6689 vec
<tree
, va_gc
> *ws_args
)
6692 gimple_stmt_iterator gsi
;
6693 vec
<tree
, va_gc
> *args
;
6695 gcc_assert (vec_safe_length (ws_args
) == 2);
6696 tree func_name
= (*ws_args
)[0];
6697 tree grain
= (*ws_args
)[1];
6699 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6700 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6701 gcc_assert (count
!= NULL_TREE
);
6702 count
= OMP_CLAUSE_OPERAND (count
, 0);
6704 gsi
= gsi_last_bb (bb
);
6705 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6707 t1
= null_pointer_node
;
6709 t1
= build_fold_addr_expr (t
);
6710 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6712 vec_alloc (args
, 4);
6713 args
->quick_push (t2
);
6714 args
->quick_push (t1
);
6715 args
->quick_push (count
);
6716 args
->quick_push (grain
);
6717 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6719 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6720 GSI_CONTINUE_LINKING
);
6723 /* Build the function call to GOMP_task to actually
6724 generate the task operation. BB is the block where to insert the code. */
6727 expand_task_call (struct omp_region
*region
, basic_block bb
,
6728 gomp_task
*entry_stmt
)
6731 gimple_stmt_iterator gsi
;
6732 location_t loc
= gimple_location (entry_stmt
);
6734 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6736 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6737 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6738 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6739 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6740 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6741 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6744 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6745 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6746 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6748 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6749 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6750 tree num_tasks
= NULL_TREE
;
6754 gimple
*g
= last_stmt (region
->outer
->entry
);
6755 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6756 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6757 struct omp_for_data fd
;
6758 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6759 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6760 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6761 OMP_CLAUSE__LOOPTEMP_
);
6762 startvar
= OMP_CLAUSE_DECL (startvar
);
6763 endvar
= OMP_CLAUSE_DECL (endvar
);
6764 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6765 if (fd
.loop
.cond_code
== LT_EXPR
)
6766 iflags
|= GOMP_TASK_FLAG_UP
;
6767 tree tclauses
= gimple_omp_for_clauses (g
);
6768 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6770 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6773 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6776 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6777 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6780 num_tasks
= integer_zero_node
;
6782 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6783 if (ifc
== NULL_TREE
)
6784 iflags
|= GOMP_TASK_FLAG_IF
;
6785 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6786 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6787 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6790 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6792 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6794 tree cond
= boolean_true_node
;
6799 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6800 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6801 build_int_cst (unsigned_type_node
,
6803 build_int_cst (unsigned_type_node
, 0));
6804 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6808 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6813 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6814 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6815 build_int_cst (unsigned_type_node
,
6816 GOMP_TASK_FLAG_FINAL
),
6817 build_int_cst (unsigned_type_node
, 0));
6818 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6821 depend
= OMP_CLAUSE_DECL (depend
);
6823 depend
= build_int_cst (ptr_type_node
, 0);
6825 priority
= fold_convert (integer_type_node
,
6826 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6828 priority
= integer_zero_node
;
6830 gsi
= gsi_last_bb (bb
);
6831 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6833 t2
= null_pointer_node
;
6835 t2
= build_fold_addr_expr_loc (loc
, t
);
6836 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6837 t
= gimple_omp_task_copy_fn (entry_stmt
);
6839 t3
= null_pointer_node
;
6841 t3
= build_fold_addr_expr_loc (loc
, t
);
6844 t
= build_call_expr (ull
6845 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6846 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6848 gimple_omp_task_arg_size (entry_stmt
),
6849 gimple_omp_task_arg_align (entry_stmt
), flags
,
6850 num_tasks
, priority
, startvar
, endvar
, step
);
6852 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6854 gimple_omp_task_arg_size (entry_stmt
),
6855 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6858 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6859 false, GSI_CONTINUE_LINKING
);
6863 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6864 catch handler and return it. This prevents programs from violating the
6865 structured block semantics with throws. */
6868 maybe_catch_exception (gimple_seq body
)
6873 if (!flag_exceptions
)
6876 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6877 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6879 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6881 g
= gimple_build_eh_must_not_throw (decl
);
6882 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6885 return gimple_seq_alloc_with_stmt (g
);
6888 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6891 vec2chain (vec
<tree
, va_gc
> *v
)
6893 tree chain
= NULL_TREE
, t
;
6896 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6898 DECL_CHAIN (t
) = chain
;
6906 /* Remove barriers in REGION->EXIT's block. Note that this is only
6907 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6908 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6909 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6913 remove_exit_barrier (struct omp_region
*region
)
6915 gimple_stmt_iterator gsi
;
6916 basic_block exit_bb
;
6920 int any_addressable_vars
= -1;
6922 exit_bb
= region
->exit
;
6924 /* If the parallel region doesn't return, we don't have REGION->EXIT
6929 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6930 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6931 statements that can appear in between are extremely limited -- no
6932 memory operations at all. Here, we allow nothing at all, so the
6933 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6934 gsi
= gsi_last_bb (exit_bb
);
6935 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6937 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6940 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6942 gsi
= gsi_last_bb (e
->src
);
6943 if (gsi_end_p (gsi
))
6945 stmt
= gsi_stmt (gsi
);
6946 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6947 && !gimple_omp_return_nowait_p (stmt
))
6949 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6950 in many cases. If there could be tasks queued, the barrier
6951 might be needed to let the tasks run before some local
6952 variable of the parallel that the task uses as shared
6953 runs out of scope. The task can be spawned either
6954 from within current function (this would be easy to check)
6955 or from some function it calls and gets passed an address
6956 of such a variable. */
6957 if (any_addressable_vars
< 0)
6959 gomp_parallel
*parallel_stmt
6960 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6961 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6962 tree local_decls
, block
, decl
;
6965 any_addressable_vars
= 0;
6966 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6967 if (TREE_ADDRESSABLE (decl
))
6969 any_addressable_vars
= 1;
6972 for (block
= gimple_block (stmt
);
6973 !any_addressable_vars
6975 && TREE_CODE (block
) == BLOCK
;
6976 block
= BLOCK_SUPERCONTEXT (block
))
6978 for (local_decls
= BLOCK_VARS (block
);
6980 local_decls
= DECL_CHAIN (local_decls
))
6981 if (TREE_ADDRESSABLE (local_decls
))
6983 any_addressable_vars
= 1;
6986 if (block
== gimple_block (parallel_stmt
))
6990 if (!any_addressable_vars
)
6991 gimple_omp_return_set_nowait (stmt
);
6997 remove_exit_barriers (struct omp_region
*region
)
6999 if (region
->type
== GIMPLE_OMP_PARALLEL
)
7000 remove_exit_barrier (region
);
7004 region
= region
->inner
;
7005 remove_exit_barriers (region
);
7006 while (region
->next
)
7008 region
= region
->next
;
7009 remove_exit_barriers (region
);
7014 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
7015 calls. These can't be declared as const functions, but
7016 within one parallel body they are constant, so they can be
7017 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
7018 which are declared const. Similarly for task body, except
7019 that in untied task omp_get_thread_num () can change at any task
7020 scheduling point. */
7023 optimize_omp_library_calls (gimple
*entry_stmt
)
7026 gimple_stmt_iterator gsi
;
7027 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7028 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
7029 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7030 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
7031 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
7032 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
7033 OMP_CLAUSE_UNTIED
) != NULL
);
7035 FOR_EACH_BB_FN (bb
, cfun
)
7036 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
7038 gimple
*call
= gsi_stmt (gsi
);
7041 if (is_gimple_call (call
)
7042 && (decl
= gimple_call_fndecl (call
))
7043 && DECL_EXTERNAL (decl
)
7044 && TREE_PUBLIC (decl
)
7045 && DECL_INITIAL (decl
) == NULL
)
7049 if (DECL_NAME (decl
) == thr_num_id
)
7051 /* In #pragma omp task untied omp_get_thread_num () can change
7052 during the execution of the task region. */
7055 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7057 else if (DECL_NAME (decl
) == num_thr_id
)
7058 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7062 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
7063 || gimple_call_num_args (call
) != 0)
7066 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
7069 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
7070 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
7071 TREE_TYPE (TREE_TYPE (built_in
))))
7074 gimple_call_set_fndecl (call
, built_in
);
7079 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7083 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
7087 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7088 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
7091 if (TREE_CODE (t
) == ADDR_EXPR
)
7092 recompute_tree_invariant_for_addr_expr (t
);
7094 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
7098 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7101 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
7104 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
7105 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
7106 !after
, after
? GSI_CONTINUE_LINKING
7108 gimple
*stmt
= gimple_build_assign (to
, from
);
7110 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
7112 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
7113 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
7114 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
7116 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
7117 gimple_regimplify_operands (stmt
, &gsi
);
7121 /* Expand the OpenMP parallel or task directive starting at REGION. */
7124 expand_omp_taskreg (struct omp_region
*region
)
7126 basic_block entry_bb
, exit_bb
, new_bb
;
7127 struct function
*child_cfun
;
7128 tree child_fn
, block
, t
;
7129 gimple_stmt_iterator gsi
;
7130 gimple
*entry_stmt
, *stmt
;
7132 vec
<tree
, va_gc
> *ws_args
;
7134 entry_stmt
= last_stmt (region
->entry
);
7135 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
7136 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7138 entry_bb
= region
->entry
;
7139 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
7140 exit_bb
= region
->cont
;
7142 exit_bb
= region
->exit
;
7146 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
7147 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
7148 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
7151 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7152 and the inner statement contains the name of the built-in function
7154 ws_args
= region
->inner
->ws_args
;
7155 else if (is_combined_parallel (region
))
7156 ws_args
= region
->ws_args
;
7160 if (child_cfun
->cfg
)
7162 /* Due to inlining, it may happen that we have already outlined
7163 the region, in which case all we need to do is make the
7164 sub-graph unreachable and emit the parallel call. */
7165 edge entry_succ_e
, exit_succ_e
;
7167 entry_succ_e
= single_succ_edge (entry_bb
);
7169 gsi
= gsi_last_bb (entry_bb
);
7170 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
7171 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
7172 gsi_remove (&gsi
, true);
7177 exit_succ_e
= single_succ_edge (exit_bb
);
7178 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
7180 remove_edge_and_dominated_blocks (entry_succ_e
);
7184 unsigned srcidx
, dstidx
, num
;
7186 /* If the parallel region needs data sent from the parent
7187 function, then the very first statement (except possible
7188 tree profile counter updates) of the parallel body
7189 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7190 &.OMP_DATA_O is passed as an argument to the child function,
7191 we need to replace it with the argument as seen by the child
7194 In most cases, this will end up being the identity assignment
7195 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7196 a function call that has been inlined, the original PARM_DECL
7197 .OMP_DATA_I may have been converted into a different local
7198 variable. In which case, we need to keep the assignment. */
7199 if (gimple_omp_taskreg_data_arg (entry_stmt
))
7201 basic_block entry_succ_bb
7202 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
7203 : FALLTHRU_EDGE (entry_bb
)->dest
;
7205 gimple
*parcopy_stmt
= NULL
;
7207 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7211 gcc_assert (!gsi_end_p (gsi
));
7212 stmt
= gsi_stmt (gsi
);
7213 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7216 if (gimple_num_ops (stmt
) == 2)
7218 tree arg
= gimple_assign_rhs1 (stmt
);
7220 /* We're ignore the subcode because we're
7221 effectively doing a STRIP_NOPS. */
7223 if (TREE_CODE (arg
) == ADDR_EXPR
7224 && TREE_OPERAND (arg
, 0)
7225 == gimple_omp_taskreg_data_arg (entry_stmt
))
7227 parcopy_stmt
= stmt
;
7233 gcc_assert (parcopy_stmt
!= NULL
);
7234 arg
= DECL_ARGUMENTS (child_fn
);
7236 if (!gimple_in_ssa_p (cfun
))
7238 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
7239 gsi_remove (&gsi
, true);
7242 /* ?? Is setting the subcode really necessary ?? */
7243 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
7244 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7249 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
7250 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
7251 /* We'd like to set the rhs to the default def in the child_fn,
7252 but it's too early to create ssa names in the child_fn.
7253 Instead, we set the rhs to the parm. In
7254 move_sese_region_to_fn, we introduce a default def for the
7255 parm, map the parm to it's default def, and once we encounter
7256 this stmt, replace the parm with the default def. */
7257 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7258 update_stmt (parcopy_stmt
);
7262 /* Declare local variables needed in CHILD_CFUN. */
7263 block
= DECL_INITIAL (child_fn
);
7264 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7265 /* The gimplifier could record temporaries in parallel/task block
7266 rather than in containing function's local_decls chain,
7267 which would mean cgraph missed finalizing them. Do it now. */
7268 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7269 if (TREE_CODE (t
) == VAR_DECL
7271 && !DECL_EXTERNAL (t
))
7272 varpool_node::finalize_decl (t
);
7273 DECL_SAVED_TREE (child_fn
) = NULL
;
7274 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7275 gimple_set_body (child_fn
, NULL
);
7276 TREE_USED (block
) = 1;
7278 /* Reset DECL_CONTEXT on function arguments. */
7279 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7280 DECL_CONTEXT (t
) = child_fn
;
7282 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7283 so that it can be moved to the child function. */
7284 gsi
= gsi_last_bb (entry_bb
);
7285 stmt
= gsi_stmt (gsi
);
7286 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7287 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7288 e
= split_block (entry_bb
, stmt
);
7289 gsi_remove (&gsi
, true);
7292 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7293 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7296 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7297 gcc_assert (e2
->dest
== region
->exit
);
7298 remove_edge (BRANCH_EDGE (entry_bb
));
7299 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7300 gsi
= gsi_last_bb (region
->exit
);
7301 gcc_assert (!gsi_end_p (gsi
)
7302 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7303 gsi_remove (&gsi
, true);
7306 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7309 gsi
= gsi_last_bb (exit_bb
);
7310 gcc_assert (!gsi_end_p (gsi
)
7311 && (gimple_code (gsi_stmt (gsi
))
7312 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7313 stmt
= gimple_build_return (NULL
);
7314 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7315 gsi_remove (&gsi
, true);
7318 /* Move the parallel region into CHILD_CFUN. */
7320 if (gimple_in_ssa_p (cfun
))
7322 init_tree_ssa (child_cfun
);
7323 init_ssa_operands (child_cfun
);
7324 child_cfun
->gimple_df
->in_ssa_p
= true;
7328 block
= gimple_block (entry_stmt
);
7330 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7332 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7335 basic_block dest_bb
= e2
->dest
;
7337 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7339 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7341 /* When the OMP expansion process cannot guarantee an up-to-date
7342 loop tree arrange for the child function to fixup loops. */
7343 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7344 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7346 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7347 num
= vec_safe_length (child_cfun
->local_decls
);
7348 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7350 t
= (*child_cfun
->local_decls
)[srcidx
];
7351 if (DECL_CONTEXT (t
) == cfun
->decl
)
7353 if (srcidx
!= dstidx
)
7354 (*child_cfun
->local_decls
)[dstidx
] = t
;
7358 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7360 /* Inform the callgraph about the new function. */
7361 child_cfun
->curr_properties
= cfun
->curr_properties
;
7362 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7363 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7364 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7365 node
->parallelized_function
= 1;
7366 cgraph_node::add_new_function (child_fn
, true);
7368 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7369 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7371 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7372 fixed in a following pass. */
7373 push_cfun (child_cfun
);
7375 assign_assembler_name_if_neeeded (child_fn
);
7378 optimize_omp_library_calls (entry_stmt
);
7379 cgraph_edge::rebuild_edges ();
7381 /* Some EH regions might become dead, see PR34608. If
7382 pass_cleanup_cfg isn't the first pass to happen with the
7383 new child, these dead EH edges might cause problems.
7384 Clean them up now. */
7385 if (flag_exceptions
)
7388 bool changed
= false;
7390 FOR_EACH_BB_FN (bb
, cfun
)
7391 changed
|= gimple_purge_dead_eh_edges (bb
);
7393 cleanup_tree_cfg ();
7395 if (gimple_in_ssa_p (cfun
))
7396 update_ssa (TODO_update_ssa
);
7397 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7398 verify_loop_structure ();
7401 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7403 omp_any_child_fn_dumped
= true;
7404 dump_function_header (dump_file
, child_fn
, dump_flags
);
7405 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7409 /* Emit a library call to launch the children threads. */
7411 expand_cilk_for_call (new_bb
,
7412 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7413 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7414 expand_parallel_call (region
, new_bb
,
7415 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7417 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7418 if (gimple_in_ssa_p (cfun
))
7419 update_ssa (TODO_update_ssa_only_virtuals
);
7422 /* Information about members of an OpenACC collapsed loop nest. */
7424 struct oacc_collapse
7426 tree base
; /* Base value. */
7427 tree iters
; /* Number of steps. */
7428 tree step
; /* step size. */
7431 /* Helper for expand_oacc_for. Determine collapsed loop information.
7432 Fill in COUNTS array. Emit any initialization code before GSI.
7433 Return the calculated outer loop bound of BOUND_TYPE. */
7436 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7437 gimple_stmt_iterator
*gsi
,
7438 oacc_collapse
*counts
, tree bound_type
)
7440 tree total
= build_int_cst (bound_type
, 1);
7443 gcc_assert (integer_onep (fd
->loop
.step
));
7444 gcc_assert (integer_zerop (fd
->loop
.n1
));
7446 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7448 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7450 tree iter_type
= TREE_TYPE (loop
->v
);
7451 tree diff_type
= iter_type
;
7452 tree plus_type
= iter_type
;
7454 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7456 if (POINTER_TYPE_P (iter_type
))
7457 plus_type
= sizetype
;
7458 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7459 diff_type
= signed_type_for (diff_type
);
7463 tree s
= loop
->step
;
7464 bool up
= loop
->cond_code
== LT_EXPR
;
7465 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7469 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7470 true, GSI_SAME_STMT
);
7471 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7472 true, GSI_SAME_STMT
);
7474 /* Convert the step, avoiding possible unsigned->signed overflow. */
7475 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7477 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7478 s
= fold_convert (diff_type
, s
);
7480 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7481 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7482 true, GSI_SAME_STMT
);
7484 /* Determine the range, avoiding possible unsigned->signed overflow. */
7485 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7486 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7487 fold_convert (plus_type
, negating
? b
: e
),
7488 fold_convert (plus_type
, negating
? e
: b
));
7489 expr
= fold_convert (diff_type
, expr
);
7491 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7492 tree range
= force_gimple_operand_gsi
7493 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7495 /* Determine number of iterations. */
7496 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7497 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7498 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7500 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7501 true, GSI_SAME_STMT
);
7503 counts
[ix
].base
= b
;
7504 counts
[ix
].iters
= iters
;
7505 counts
[ix
].step
= s
;
7507 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7508 fold_convert (bound_type
, iters
));
7514 /* Emit initializers for collapsed loop members. IVAR is the outer
7515 loop iteration variable, from which collapsed loop iteration values
7516 are calculated. COUNTS array has been initialized by
7517 expand_oacc_collapse_inits. */
7520 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7521 gimple_stmt_iterator
*gsi
,
7522 const oacc_collapse
*counts
, tree ivar
)
7524 tree ivar_type
= TREE_TYPE (ivar
);
7526 /* The most rapidly changing iteration variable is the innermost
7528 for (int ix
= fd
->collapse
; ix
--;)
7530 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7531 const oacc_collapse
*collapse
= &counts
[ix
];
7532 tree iter_type
= TREE_TYPE (loop
->v
);
7533 tree diff_type
= TREE_TYPE (collapse
->step
);
7534 tree plus_type
= iter_type
;
7535 enum tree_code plus_code
= PLUS_EXPR
;
7538 if (POINTER_TYPE_P (iter_type
))
7540 plus_code
= POINTER_PLUS_EXPR
;
7541 plus_type
= sizetype
;
7544 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7545 fold_convert (ivar_type
, collapse
->iters
));
7546 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7548 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7549 fold_convert (plus_type
, expr
));
7550 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7551 true, GSI_SAME_STMT
);
7552 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7553 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7557 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7558 fold_convert (ivar_type
, collapse
->iters
));
7559 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7560 true, GSI_SAME_STMT
);
7566 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7567 of the combined collapse > 1 loop constructs, generate code like:
7568 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7573 count3 = (adj + N32 - N31) / STEP3;
7574 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7579 count2 = (adj + N22 - N21) / STEP2;
7580 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7585 count1 = (adj + N12 - N11) / STEP1;
7586 count = count1 * count2 * count3;
7587 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7589 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7590 of the combined loop constructs, just initialize COUNTS array
7591 from the _looptemp_ clauses. */
7593 /* NOTE: It *could* be better to moosh all of the BBs together,
7594 creating one larger BB with all the computation and the unexpected
7595 jump at the end. I.e.
7597 bool zero3, zero2, zero1, zero;
7600 count3 = (N32 - N31) /[cl] STEP3;
7602 count2 = (N22 - N21) /[cl] STEP2;
7604 count1 = (N12 - N11) /[cl] STEP1;
7605 zero = zero3 || zero2 || zero1;
7606 count = count1 * count2 * count3;
7607 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7609 After all, we expect the zero=false, and thus we expect to have to
7610 evaluate all of the comparison expressions, so short-circuiting
7611 oughtn't be a win. Since the condition isn't protecting a
7612 denominator, we're not concerned about divide-by-zero, so we can
7613 fully evaluate count even if a numerator turned out to be wrong.
7615 It seems like putting this all together would create much better
7616 scheduling opportunities, and less pressure on the chip's branch
7620 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7621 basic_block
&entry_bb
, tree
*counts
,
7622 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7623 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7624 basic_block
&l2_dom_bb
)
7626 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7630 /* Collapsed loops need work for expansion into SSA form. */
7631 gcc_assert (!gimple_in_ssa_p (cfun
));
7633 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7634 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7636 gcc_assert (fd
->ordered
== 0);
7637 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7638 isn't supposed to be handled, as the inner loop doesn't
7640 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7641 OMP_CLAUSE__LOOPTEMP_
);
7642 gcc_assert (innerc
);
7643 for (i
= 0; i
< fd
->collapse
; i
++)
7645 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7646 OMP_CLAUSE__LOOPTEMP_
);
7647 gcc_assert (innerc
);
7649 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7651 counts
[0] = NULL_TREE
;
7656 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7658 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7659 counts
[i
] = NULL_TREE
;
7660 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7661 fold_convert (itype
, fd
->loops
[i
].n1
),
7662 fold_convert (itype
, fd
->loops
[i
].n2
));
7663 if (t
&& integer_zerop (t
))
7665 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7666 counts
[i
] = build_int_cst (type
, 0);
7670 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7672 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7674 if (i
>= fd
->collapse
&& counts
[i
])
7676 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7677 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7678 fold_convert (itype
, fd
->loops
[i
].n1
),
7679 fold_convert (itype
, fd
->loops
[i
].n2
)))
7680 == NULL_TREE
|| !integer_onep (t
)))
7684 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7685 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7686 true, GSI_SAME_STMT
);
7687 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7688 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7689 true, GSI_SAME_STMT
);
7690 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7691 NULL_TREE
, NULL_TREE
);
7692 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7693 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7694 expand_omp_regimplify_p
, NULL
, NULL
)
7695 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7696 expand_omp_regimplify_p
, NULL
, NULL
))
7698 *gsi
= gsi_for_stmt (cond_stmt
);
7699 gimple_regimplify_operands (cond_stmt
, gsi
);
7701 e
= split_block (entry_bb
, cond_stmt
);
7702 basic_block
&zero_iter_bb
7703 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7704 int &first_zero_iter
7705 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7706 if (zero_iter_bb
== NULL
)
7708 gassign
*assign_stmt
;
7709 first_zero_iter
= i
;
7710 zero_iter_bb
= create_empty_bb (entry_bb
);
7711 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7712 *gsi
= gsi_after_labels (zero_iter_bb
);
7713 if (i
< fd
->collapse
)
7714 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7715 build_zero_cst (type
));
7718 counts
[i
] = create_tmp_reg (type
, ".count");
7720 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7722 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7723 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7726 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7727 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7728 e
->flags
= EDGE_TRUE_VALUE
;
7729 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7730 if (l2_dom_bb
== NULL
)
7731 l2_dom_bb
= entry_bb
;
7733 *gsi
= gsi_last_bb (entry_bb
);
7736 if (POINTER_TYPE_P (itype
))
7737 itype
= signed_type_for (itype
);
7738 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7740 t
= fold_build2 (PLUS_EXPR
, itype
,
7741 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7742 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7743 fold_convert (itype
, fd
->loops
[i
].n2
));
7744 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7745 fold_convert (itype
, fd
->loops
[i
].n1
));
7746 /* ?? We could probably use CEIL_DIV_EXPR instead of
7747 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7748 generate the same code in the end because generically we
7749 don't know that the values involved must be negative for
7751 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7752 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7753 fold_build1 (NEGATE_EXPR
, itype
, t
),
7754 fold_build1 (NEGATE_EXPR
, itype
,
7755 fold_convert (itype
,
7756 fd
->loops
[i
].step
)));
7758 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7759 fold_convert (itype
, fd
->loops
[i
].step
));
7760 t
= fold_convert (type
, t
);
7761 if (TREE_CODE (t
) == INTEGER_CST
)
7765 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7766 counts
[i
] = create_tmp_reg (type
, ".count");
7767 expand_omp_build_assign (gsi
, counts
[i
], t
);
7769 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7774 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7775 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7781 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7783 V3 = N31 + (T % count3) * STEP3;
7785 V2 = N21 + (T % count2) * STEP2;
7787 V1 = N11 + T * STEP1;
7788 if this loop doesn't have an inner loop construct combined with it.
7789 If it does have an inner loop construct combined with it and the
7790 iteration count isn't known constant, store values from counts array
7791 into its _looptemp_ temporaries instead. */
7794 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7795 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7798 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7800 /* If fd->loop.n2 is constant, then no propagation of the counts
7801 is needed, they are constant. */
7802 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7805 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7806 ? gimple_omp_taskreg_clauses (inner_stmt
)
7807 : gimple_omp_for_clauses (inner_stmt
);
7808 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7809 isn't supposed to be handled, as the inner loop doesn't
7811 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7812 gcc_assert (innerc
);
7813 for (i
= 0; i
< fd
->collapse
; i
++)
7815 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7816 OMP_CLAUSE__LOOPTEMP_
);
7817 gcc_assert (innerc
);
7820 tree tem
= OMP_CLAUSE_DECL (innerc
);
7821 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7822 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7823 false, GSI_CONTINUE_LINKING
);
7824 gassign
*stmt
= gimple_build_assign (tem
, t
);
7825 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7831 tree type
= TREE_TYPE (fd
->loop
.v
);
7832 tree tem
= create_tmp_reg (type
, ".tem");
7833 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7834 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7836 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7838 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7840 if (POINTER_TYPE_P (vtype
))
7841 itype
= signed_type_for (vtype
);
7843 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7846 t
= fold_convert (itype
, t
);
7847 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7848 fold_convert (itype
, fd
->loops
[i
].step
));
7849 if (POINTER_TYPE_P (vtype
))
7850 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7852 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7853 t
= force_gimple_operand_gsi (gsi
, t
,
7854 DECL_P (fd
->loops
[i
].v
)
7855 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7857 GSI_CONTINUE_LINKING
);
7858 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7859 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7862 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7863 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7864 false, GSI_CONTINUE_LINKING
);
7865 stmt
= gimple_build_assign (tem
, t
);
7866 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7872 /* Helper function for expand_omp_for_*. Generate code like:
7875 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7879 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7886 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7887 basic_block body_bb
)
7889 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7891 gimple_stmt_iterator gsi
;
7897 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7899 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7901 bb
= create_empty_bb (last_bb
);
7902 add_bb_to_loop (bb
, last_bb
->loop_father
);
7903 gsi
= gsi_start_bb (bb
);
7905 if (i
< fd
->collapse
- 1)
7907 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7908 e
->probability
= REG_BR_PROB_BASE
/ 8;
7910 t
= fd
->loops
[i
+ 1].n1
;
7911 t
= force_gimple_operand_gsi (&gsi
, t
,
7912 DECL_P (fd
->loops
[i
+ 1].v
)
7913 && TREE_ADDRESSABLE (fd
->loops
[i
7916 GSI_CONTINUE_LINKING
);
7917 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7918 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7923 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7925 if (POINTER_TYPE_P (vtype
))
7926 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7928 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7929 t
= force_gimple_operand_gsi (&gsi
, t
,
7930 DECL_P (fd
->loops
[i
].v
)
7931 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7932 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7933 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7934 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7938 t
= fd
->loops
[i
].n2
;
7939 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7940 false, GSI_CONTINUE_LINKING
);
7941 tree v
= fd
->loops
[i
].v
;
7942 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7943 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7944 false, GSI_CONTINUE_LINKING
);
7945 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7946 stmt
= gimple_build_cond_empty (t
);
7947 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7948 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7949 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7952 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7960 /* Expand #pragma omp ordered depend(source). */
7963 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7964 tree
*counts
, location_t loc
)
7966 enum built_in_function source_ix
7967 = fd
->iter_type
== long_integer_type_node
7968 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7970 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7971 build_fold_addr_expr (counts
[fd
->ordered
]));
7972 gimple_set_location (g
, loc
);
7973 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7976 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7979 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7980 tree
*counts
, tree c
, location_t loc
)
7982 auto_vec
<tree
, 10> args
;
7983 enum built_in_function sink_ix
7984 = fd
->iter_type
== long_integer_type_node
7985 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7986 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7988 gimple_stmt_iterator gsi2
= *gsi
;
7989 bool warned_step
= false;
7991 for (i
= 0; i
< fd
->ordered
; i
++)
7993 off
= TREE_PURPOSE (deps
);
7994 if (!integer_zerop (off
))
7996 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7997 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7998 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7999 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8000 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
8001 "lexically later iteration");
8004 deps
= TREE_CHAIN (deps
);
8006 /* If all offsets corresponding to the collapsed loops are zero,
8007 this depend clause can be ignored. FIXME: but there is still a
8008 flush needed. We need to emit one __sync_synchronize () for it
8009 though (perhaps conditionally)? Solve this together with the
8010 conservative dependence folding optimization.
8011 if (i >= fd->collapse)
8014 deps
= OMP_CLAUSE_DECL (c
);
8016 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
8017 edge e2
= split_block_after_labels (e1
->dest
);
8019 *gsi
= gsi_after_labels (e1
->dest
);
8020 for (i
= 0; i
< fd
->ordered
; i
++)
8022 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
8023 if (POINTER_TYPE_P (itype
))
8026 deps
= TREE_CHAIN (deps
);
8027 off
= TREE_PURPOSE (deps
);
8028 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
8030 if (integer_zerop (off
))
8031 t
= boolean_true_node
;
8035 tree co
= fold_convert_loc (loc
, itype
, off
);
8036 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
8038 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8039 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
8040 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
8041 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
8044 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8045 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8046 fd
->loops
[i
].v
, co
);
8048 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8049 fd
->loops
[i
].v
, co
);
8050 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
8052 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8053 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
8056 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
8059 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8060 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
8063 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
8067 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
8071 off
= fold_convert_loc (loc
, itype
, off
);
8073 if (fd
->loops
[i
].cond_code
== LT_EXPR
8074 ? !integer_onep (fd
->loops
[i
].step
)
8075 : !integer_minus_onep (fd
->loops
[i
].step
))
8077 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8078 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
8079 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8082 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
8083 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
8084 build_int_cst (itype
, 0));
8085 if (integer_zerop (t
) && !warned_step
)
8087 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
8088 "in the iteration space");
8091 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
8095 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8101 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8102 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8103 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
8105 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8106 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
8107 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8110 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
8111 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8112 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
8113 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
8114 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8117 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
8119 if (i
< fd
->collapse
- 1)
8121 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
8126 off
= unshare_expr (off
);
8127 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
8128 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
8129 true, GSI_SAME_STMT
);
8132 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
8133 gimple_set_location (g
, loc
);
8134 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
8136 *gsi
= gsi_last_bb (e1
->src
);
8137 cond
= unshare_expr (cond
);
8138 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
8139 GSI_CONTINUE_LINKING
);
8140 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
8141 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
8142 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8143 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8144 e1
->flags
= EDGE_TRUE_VALUE
;
8145 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
8147 *gsi
= gsi_after_labels (e2
->dest
);
8150 /* Expand all #pragma omp ordered depend(source) and
8151 #pragma omp ordered depend(sink:...) constructs in the current
8152 #pragma omp for ordered(n) region. */
8155 expand_omp_ordered_source_sink (struct omp_region
*region
,
8156 struct omp_for_data
*fd
, tree
*counts
,
8157 basic_block cont_bb
)
8159 struct omp_region
*inner
;
8161 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
8162 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
8163 counts
[i
] = NULL_TREE
;
8164 else if (i
>= fd
->collapse
&& !cont_bb
)
8165 counts
[i
] = build_zero_cst (fd
->iter_type
);
8166 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
8167 && integer_onep (fd
->loops
[i
].step
))
8168 counts
[i
] = NULL_TREE
;
8170 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
8172 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
8173 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
8174 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
8176 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
8177 if (inner
->type
== GIMPLE_OMP_ORDERED
)
8179 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
8180 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
8181 location_t loc
= gimple_location (ord_stmt
);
8183 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8184 c
; c
= OMP_CLAUSE_CHAIN (c
))
8185 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
8188 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
8189 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8190 c
; c
= OMP_CLAUSE_CHAIN (c
))
8191 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8192 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
8193 gsi_remove (&gsi
, true);
8197 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8201 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
8202 basic_block cont_bb
, basic_block body_bb
,
8203 bool ordered_lastprivate
)
8205 if (fd
->ordered
== fd
->collapse
)
8210 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8211 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8213 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8214 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
8215 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
8216 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8217 size_int (i
- fd
->collapse
+ 1),
8218 NULL_TREE
, NULL_TREE
);
8219 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8224 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
8226 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
8227 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8228 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8229 fold_convert (type
, fd
->loops
[i
].n1
));
8231 expand_omp_build_assign (&gsi
, counts
[i
],
8232 build_zero_cst (fd
->iter_type
));
8233 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8234 size_int (i
- fd
->collapse
+ 1),
8235 NULL_TREE
, NULL_TREE
);
8236 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8237 if (!gsi_end_p (gsi
))
8240 gsi
= gsi_last_bb (body_bb
);
8241 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
8242 basic_block new_body
= e1
->dest
;
8243 if (body_bb
== cont_bb
)
8246 basic_block new_header
;
8247 if (EDGE_COUNT (cont_bb
->preds
) > 0)
8249 gsi
= gsi_last_bb (cont_bb
);
8250 if (POINTER_TYPE_P (type
))
8251 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8252 fold_convert (sizetype
,
8253 fd
->loops
[i
].step
));
8255 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8256 fold_convert (type
, fd
->loops
[i
].step
));
8257 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8260 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8261 build_int_cst (fd
->iter_type
, 1));
8262 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8267 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8268 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8269 t
= fold_convert (fd
->iter_type
, t
);
8270 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8271 true, GSI_SAME_STMT
);
8273 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8274 size_int (i
- fd
->collapse
+ 1),
8275 NULL_TREE
, NULL_TREE
);
8276 expand_omp_build_assign (&gsi
, aref
, t
);
8278 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8279 new_header
= e2
->dest
;
8282 new_header
= cont_bb
;
8283 gsi
= gsi_after_labels (new_header
);
8284 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8285 true, GSI_SAME_STMT
);
8287 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8288 true, NULL_TREE
, true, GSI_SAME_STMT
);
8289 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8290 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8291 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8294 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8295 e3
->flags
= EDGE_FALSE_VALUE
;
8296 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8297 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8298 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8300 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8301 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8305 struct loop
*loop
= alloc_loop ();
8306 loop
->header
= new_header
;
8307 loop
->latch
= e2
->src
;
8308 add_loop (loop
, body_bb
->loop_father
);
8312 /* If there are any lastprivate clauses and it is possible some loops
8313 might have zero iterations, ensure all the decls are initialized,
8314 otherwise we could crash evaluating C++ class iterators with lastprivate
8316 bool need_inits
= false;
8317 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8320 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8321 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8322 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8323 fold_convert (type
, fd
->loops
[i
].n1
));
8327 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8328 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8330 fold_convert (type
, fd
->loops
[i
].n1
),
8331 fold_convert (type
, fd
->loops
[i
].n2
));
8332 if (!integer_onep (this_cond
))
8340 /* A subroutine of expand_omp_for. Generate code for a parallel
8341 loop with any schedule. Given parameters:
8343 for (V = N1; V cond N2; V += STEP) BODY;
8345 where COND is "<" or ">", we generate pseudocode
8347 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8348 if (more) goto L0; else goto L3;
8355 if (V cond iend) goto L1; else goto L2;
8357 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8360 If this is a combined omp parallel loop, instead of the call to
8361 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8362 If this is gimple_omp_for_combined_p loop, then instead of assigning
8363 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8364 inner GIMPLE_OMP_FOR and V += STEP; and
8365 if (V cond iend) goto L1; else goto L2; are removed.
8367 For collapsed loops, given parameters:
8369 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8370 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8371 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8374 we generate pseudocode
8376 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8381 count3 = (adj + N32 - N31) / STEP3;
8382 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8387 count2 = (adj + N22 - N21) / STEP2;
8388 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8393 count1 = (adj + N12 - N11) / STEP1;
8394 count = count1 * count2 * count3;
8399 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8400 if (more) goto L0; else goto L3;
8404 V3 = N31 + (T % count3) * STEP3;
8406 V2 = N21 + (T % count2) * STEP2;
8408 V1 = N11 + T * STEP1;
8413 if (V < iend) goto L10; else goto L2;
8416 if (V3 cond3 N32) goto L1; else goto L11;
8420 if (V2 cond2 N22) goto L1; else goto L12;
8426 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8432 expand_omp_for_generic (struct omp_region
*region
,
8433 struct omp_for_data
*fd
,
8434 enum built_in_function start_fn
,
8435 enum built_in_function next_fn
,
8438 tree type
, istart0
, iend0
, iend
;
8439 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8440 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8441 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8442 gimple_stmt_iterator gsi
;
8443 gassign
*assign_stmt
;
8444 bool in_combined_parallel
= is_combined_parallel (region
);
8445 bool broken_loop
= region
->cont
== NULL
;
8447 tree
*counts
= NULL
;
8449 bool ordered_lastprivate
= false;
8451 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8452 gcc_assert (fd
->iter_type
== long_integer_type_node
8453 || !in_combined_parallel
);
8455 entry_bb
= region
->entry
;
8456 cont_bb
= region
->cont
;
8458 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8459 gcc_assert (broken_loop
8460 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8461 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8462 l1_bb
= single_succ (l0_bb
);
8465 l2_bb
= create_empty_bb (cont_bb
);
8466 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8467 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8469 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8473 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8474 exit_bb
= region
->exit
;
8476 gsi
= gsi_last_bb (entry_bb
);
8478 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8480 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8481 OMP_CLAUSE_LASTPRIVATE
))
8482 ordered_lastprivate
= false;
8483 if (fd
->collapse
> 1 || fd
->ordered
)
8485 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8486 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8488 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8489 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8490 zero_iter1_bb
, first_zero_iter1
,
8491 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8495 /* Some counts[i] vars might be uninitialized if
8496 some loop has zero iterations. But the body shouldn't
8497 be executed in that case, so just avoid uninit warnings. */
8498 for (i
= first_zero_iter1
;
8499 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8500 if (SSA_VAR_P (counts
[i
]))
8501 TREE_NO_WARNING (counts
[i
]) = 1;
8503 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8505 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8506 gsi
= gsi_last_bb (entry_bb
);
8507 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8508 get_immediate_dominator (CDI_DOMINATORS
,
8513 /* Some counts[i] vars might be uninitialized if
8514 some loop has zero iterations. But the body shouldn't
8515 be executed in that case, so just avoid uninit warnings. */
8516 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8517 if (SSA_VAR_P (counts
[i
]))
8518 TREE_NO_WARNING (counts
[i
]) = 1;
8520 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8524 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8526 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8527 gsi
= gsi_last_bb (entry_bb
);
8528 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8529 get_immediate_dominator
8530 (CDI_DOMINATORS
, zero_iter2_bb
));
8533 if (fd
->collapse
== 1)
8535 counts
[0] = fd
->loop
.n2
;
8536 fd
->loop
= fd
->loops
[0];
8540 type
= TREE_TYPE (fd
->loop
.v
);
8541 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8542 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8543 TREE_ADDRESSABLE (istart0
) = 1;
8544 TREE_ADDRESSABLE (iend0
) = 1;
8546 /* See if we need to bias by LLONG_MIN. */
8547 if (fd
->iter_type
== long_long_unsigned_type_node
8548 && TREE_CODE (type
) == INTEGER_TYPE
8549 && !TYPE_UNSIGNED (type
)
8550 && fd
->ordered
== 0)
8554 if (fd
->loop
.cond_code
== LT_EXPR
)
8557 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8561 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8564 if (TREE_CODE (n1
) != INTEGER_CST
8565 || TREE_CODE (n2
) != INTEGER_CST
8566 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8567 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8570 gimple_stmt_iterator gsif
= gsi
;
8573 tree arr
= NULL_TREE
;
8574 if (in_combined_parallel
)
8576 gcc_assert (fd
->ordered
== 0);
8577 /* In a combined parallel loop, emit a call to
8578 GOMP_loop_foo_next. */
8579 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8580 build_fold_addr_expr (istart0
),
8581 build_fold_addr_expr (iend0
));
8585 tree t0
, t1
, t2
, t3
, t4
;
8586 /* If this is not a combined parallel loop, emit a call to
8587 GOMP_loop_foo_start in ENTRY_BB. */
8588 t4
= build_fold_addr_expr (iend0
);
8589 t3
= build_fold_addr_expr (istart0
);
8592 t0
= build_int_cst (unsigned_type_node
,
8593 fd
->ordered
- fd
->collapse
+ 1);
8594 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8596 - fd
->collapse
+ 1),
8598 DECL_NAMELESS (arr
) = 1;
8599 TREE_ADDRESSABLE (arr
) = 1;
8600 TREE_STATIC (arr
) = 1;
8601 vec
<constructor_elt
, va_gc
> *v
;
8602 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8605 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8608 if (idx
== 0 && fd
->collapse
> 1)
8611 c
= counts
[idx
+ fd
->collapse
- 1];
8612 tree purpose
= size_int (idx
);
8613 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8614 if (TREE_CODE (c
) != INTEGER_CST
)
8615 TREE_STATIC (arr
) = 0;
8618 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8619 if (!TREE_STATIC (arr
))
8620 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8621 void_type_node
, arr
),
8622 true, NULL_TREE
, true, GSI_SAME_STMT
);
8623 t1
= build_fold_addr_expr (arr
);
8628 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8631 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8634 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8635 OMP_CLAUSE__LOOPTEMP_
);
8636 gcc_assert (innerc
);
8637 t0
= OMP_CLAUSE_DECL (innerc
);
8638 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8639 OMP_CLAUSE__LOOPTEMP_
);
8640 gcc_assert (innerc
);
8641 t1
= OMP_CLAUSE_DECL (innerc
);
8643 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8644 && TYPE_PRECISION (TREE_TYPE (t0
))
8645 != TYPE_PRECISION (fd
->iter_type
))
8647 /* Avoid casting pointers to integer of a different size. */
8648 tree itype
= signed_type_for (type
);
8649 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8650 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8654 t1
= fold_convert (fd
->iter_type
, t1
);
8655 t0
= fold_convert (fd
->iter_type
, t0
);
8659 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8660 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8663 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8667 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8668 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8670 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8671 5, t0
, t1
, t
, t3
, t4
);
8673 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8674 6, t0
, t1
, t2
, t
, t3
, t4
);
8676 else if (fd
->ordered
)
8677 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8680 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8681 5, t0
, t1
, t2
, t3
, t4
);
8689 /* The GOMP_loop_ull_*start functions have additional boolean
8690 argument, true for < loops and false for > loops.
8691 In Fortran, the C bool type can be different from
8692 boolean_type_node. */
8693 bfn_decl
= builtin_decl_explicit (start_fn
);
8694 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8695 t5
= build_int_cst (c_bool_type
,
8696 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8699 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8700 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8701 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8702 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8705 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8706 6, t5
, t0
, t1
, t2
, t3
, t4
);
8709 if (TREE_TYPE (t
) != boolean_type_node
)
8710 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8711 t
, build_int_cst (TREE_TYPE (t
), 0));
8712 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8713 true, GSI_SAME_STMT
);
8714 if (arr
&& !TREE_STATIC (arr
))
8716 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8717 TREE_THIS_VOLATILE (clobber
) = 1;
8718 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8721 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8723 /* Remove the GIMPLE_OMP_FOR statement. */
8724 gsi_remove (&gsi
, true);
8726 if (gsi_end_p (gsif
))
8727 gsif
= gsi_after_labels (gsi_bb (gsif
));
8730 /* Iteration setup for sequential loop goes in L0_BB. */
8731 tree startvar
= fd
->loop
.v
;
8732 tree endvar
= NULL_TREE
;
8734 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8736 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8737 && gimple_omp_for_kind (inner_stmt
)
8738 == GF_OMP_FOR_KIND_SIMD
);
8739 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8740 OMP_CLAUSE__LOOPTEMP_
);
8741 gcc_assert (innerc
);
8742 startvar
= OMP_CLAUSE_DECL (innerc
);
8743 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8744 OMP_CLAUSE__LOOPTEMP_
);
8745 gcc_assert (innerc
);
8746 endvar
= OMP_CLAUSE_DECL (innerc
);
8749 gsi
= gsi_start_bb (l0_bb
);
8751 if (fd
->ordered
&& fd
->collapse
== 1)
8752 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8753 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8755 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8756 if (fd
->ordered
&& fd
->collapse
== 1)
8758 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8759 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8760 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8763 t
= fold_convert (TREE_TYPE (startvar
), t
);
8764 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8770 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8771 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8772 t
= fold_convert (TREE_TYPE (startvar
), t
);
8774 t
= force_gimple_operand_gsi (&gsi
, t
,
8776 && TREE_ADDRESSABLE (startvar
),
8777 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8778 assign_stmt
= gimple_build_assign (startvar
, t
);
8779 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8782 if (fd
->ordered
&& fd
->collapse
== 1)
8783 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8784 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8786 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8787 if (fd
->ordered
&& fd
->collapse
== 1)
8789 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8790 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8791 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8794 t
= fold_convert (TREE_TYPE (startvar
), t
);
8795 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8801 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8802 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8803 t
= fold_convert (TREE_TYPE (startvar
), t
);
8805 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8806 false, GSI_CONTINUE_LINKING
);
8809 assign_stmt
= gimple_build_assign (endvar
, iend
);
8810 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8811 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8812 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8814 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8815 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8817 /* Handle linear clause adjustments. */
8818 tree itercnt
= NULL_TREE
;
8819 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8820 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8821 c
; c
= OMP_CLAUSE_CHAIN (c
))
8822 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8823 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8825 tree d
= OMP_CLAUSE_DECL (c
);
8826 bool is_ref
= is_reference (d
);
8827 tree t
= d
, a
, dest
;
8829 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8830 tree type
= TREE_TYPE (t
);
8831 if (POINTER_TYPE_P (type
))
8833 dest
= unshare_expr (t
);
8834 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8835 expand_omp_build_assign (&gsif
, v
, t
);
8836 if (itercnt
== NULL_TREE
)
8839 tree n1
= fd
->loop
.n1
;
8840 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8843 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8845 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8847 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8849 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8850 itercnt
, fd
->loop
.step
);
8851 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8853 GSI_CONTINUE_LINKING
);
8855 a
= fold_build2 (MULT_EXPR
, type
,
8856 fold_convert (type
, itercnt
),
8857 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8858 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8859 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8860 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8861 false, GSI_CONTINUE_LINKING
);
8862 assign_stmt
= gimple_build_assign (dest
, t
);
8863 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8865 if (fd
->collapse
> 1)
8866 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8870 /* Until now, counts array contained number of iterations or
8871 variable containing it for ith loop. From now on, we need
8872 those counts only for collapsed loops, and only for the 2nd
8873 till the last collapsed one. Move those one element earlier,
8874 we'll use counts[fd->collapse - 1] for the first source/sink
8875 iteration counter and so on and counts[fd->ordered]
8876 as the array holding the current counter values for
8878 if (fd
->collapse
> 1)
8879 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8883 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8885 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8887 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8888 fold_convert (type
, fd
->loops
[i
].n1
),
8889 fold_convert (type
, fd
->loops
[i
].n2
));
8890 if (!integer_onep (this_cond
))
8893 if (i
< fd
->ordered
)
8896 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8897 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8898 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8899 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8900 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8901 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8902 make_edge (cont_bb
, l1_bb
, 0);
8903 l2_bb
= create_empty_bb (cont_bb
);
8904 broken_loop
= false;
8907 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8908 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8909 ordered_lastprivate
);
8910 if (counts
[fd
->collapse
- 1])
8912 gcc_assert (fd
->collapse
== 1);
8913 gsi
= gsi_last_bb (l0_bb
);
8914 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8916 gsi
= gsi_last_bb (cont_bb
);
8917 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8918 build_int_cst (fd
->iter_type
, 1));
8919 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8920 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8921 size_zero_node
, NULL_TREE
, NULL_TREE
);
8922 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8923 t
= counts
[fd
->collapse
- 1];
8925 else if (fd
->collapse
> 1)
8929 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8930 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8931 t
= fold_convert (fd
->iter_type
, t
);
8933 gsi
= gsi_last_bb (l0_bb
);
8934 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8935 size_zero_node
, NULL_TREE
, NULL_TREE
);
8936 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8937 false, GSI_CONTINUE_LINKING
);
8938 expand_omp_build_assign (&gsi
, aref
, t
, true);
8943 /* Code to control the increment and predicate for the sequential
8944 loop goes in the CONT_BB. */
8945 gsi
= gsi_last_bb (cont_bb
);
8946 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8947 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8948 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8949 vback
= gimple_omp_continue_control_def (cont_stmt
);
8951 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8953 if (POINTER_TYPE_P (type
))
8954 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8956 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8957 t
= force_gimple_operand_gsi (&gsi
, t
,
8959 && TREE_ADDRESSABLE (vback
),
8960 NULL_TREE
, true, GSI_SAME_STMT
);
8961 assign_stmt
= gimple_build_assign (vback
, t
);
8962 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8964 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8966 if (fd
->collapse
> 1)
8970 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8971 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8972 t
= fold_convert (fd
->iter_type
, t
);
8974 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8975 counts
[fd
->ordered
], size_zero_node
,
8976 NULL_TREE
, NULL_TREE
);
8977 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8978 true, GSI_SAME_STMT
);
8979 expand_omp_build_assign (&gsi
, aref
, t
);
8982 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8983 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8985 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8986 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8989 /* Remove GIMPLE_OMP_CONTINUE. */
8990 gsi_remove (&gsi
, true);
8992 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8993 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8995 /* Emit code to get the next parallel iteration in L2_BB. */
8996 gsi
= gsi_start_bb (l2_bb
);
8998 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8999 build_fold_addr_expr (istart0
),
9000 build_fold_addr_expr (iend0
));
9001 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9002 false, GSI_CONTINUE_LINKING
);
9003 if (TREE_TYPE (t
) != boolean_type_node
)
9004 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
9005 t
, build_int_cst (TREE_TYPE (t
), 0));
9006 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9007 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
9010 /* Add the loop cleanup function. */
9011 gsi
= gsi_last_bb (exit_bb
);
9012 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9013 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
9014 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9015 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
9017 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
9018 gcall
*call_stmt
= gimple_build_call (t
, 0);
9019 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9020 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
9021 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
9024 tree arr
= counts
[fd
->ordered
];
9025 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
9026 TREE_THIS_VOLATILE (clobber
) = 1;
9027 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
9030 gsi_remove (&gsi
, true);
9032 /* Connect the new blocks. */
9033 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
9034 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
9040 e
= find_edge (cont_bb
, l3_bb
);
9041 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
9043 phis
= phi_nodes (l3_bb
);
9044 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9046 gimple
*phi
= gsi_stmt (gsi
);
9047 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
9048 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
9052 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
9053 e
= find_edge (cont_bb
, l1_bb
);
9056 e
= BRANCH_EDGE (cont_bb
);
9057 gcc_assert (single_succ (e
->dest
) == l1_bb
);
9059 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9064 else if (fd
->collapse
> 1)
9067 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9070 e
->flags
= EDGE_TRUE_VALUE
;
9073 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9074 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
9078 e
= find_edge (cont_bb
, l2_bb
);
9079 e
->flags
= EDGE_FALLTHRU
;
9081 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9083 if (gimple_in_ssa_p (cfun
))
9085 /* Add phis to the outer loop that connect to the phis in the inner,
9086 original loop, and move the loop entry value of the inner phi to
9087 the loop entry value of the outer phi. */
9089 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
9091 source_location locus
;
9093 gphi
*exit_phi
= psi
.phi ();
9095 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
9096 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
9098 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
9099 edge latch_to_l1
= find_edge (latch
, l1_bb
);
9101 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
9103 tree t
= gimple_phi_result (exit_phi
);
9104 tree new_res
= copy_ssa_name (t
, NULL
);
9105 nphi
= create_phi_node (new_res
, l0_bb
);
9107 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
9108 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
9109 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
9110 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
9111 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
9113 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
9114 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
9116 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
9120 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
9121 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
9122 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
9123 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
9124 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
9125 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
9126 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
9127 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
9129 /* We enter expand_omp_for_generic with a loop. This original loop may
9130 have its own loop struct, or it may be part of an outer loop struct
9131 (which may be the fake loop). */
9132 struct loop
*outer_loop
= entry_bb
->loop_father
;
9133 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
9135 add_bb_to_loop (l2_bb
, outer_loop
);
9137 /* We've added a new loop around the original loop. Allocate the
9138 corresponding loop struct. */
9139 struct loop
*new_loop
= alloc_loop ();
9140 new_loop
->header
= l0_bb
;
9141 new_loop
->latch
= l2_bb
;
9142 add_loop (new_loop
, outer_loop
);
9144 /* Allocate a loop structure for the original loop unless we already
9146 if (!orig_loop_has_loop_struct
9147 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9149 struct loop
*orig_loop
= alloc_loop ();
9150 orig_loop
->header
= l1_bb
;
9151 /* The loop may have multiple latches. */
9152 add_loop (orig_loop
, new_loop
);
9158 /* A subroutine of expand_omp_for. Generate code for a parallel
9159 loop with static schedule and no specified chunk size. Given
9162 for (V = N1; V cond N2; V += STEP) BODY;
9164 where COND is "<" or ">", we generate pseudocode
9166 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9171 if ((__typeof (V)) -1 > 0 && cond is >)
9172 n = -(adj + N2 - N1) / -STEP;
9174 n = (adj + N2 - N1) / STEP;
9177 if (threadid < tt) goto L3; else goto L4;
9182 s0 = q * threadid + tt;
9185 if (s0 >= e0) goto L2; else goto L0;
9191 if (V cond e) goto L1;
9196 expand_omp_for_static_nochunk (struct omp_region
*region
,
9197 struct omp_for_data
*fd
,
9200 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
9201 tree type
, itype
, vmain
, vback
;
9202 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
9203 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
9205 gimple_stmt_iterator gsi
;
9207 bool broken_loop
= region
->cont
== NULL
;
9208 tree
*counts
= NULL
;
9211 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9212 if (POINTER_TYPE_P (type
))
9213 itype
= signed_type_for (type
);
9215 entry_bb
= region
->entry
;
9216 cont_bb
= region
->cont
;
9217 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9218 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9219 gcc_assert (broken_loop
9220 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
9221 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
9222 body_bb
= single_succ (seq_start_bb
);
9225 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9226 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9227 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9229 exit_bb
= region
->exit
;
9231 /* Iteration space partitioning goes in ENTRY_BB. */
9232 gsi
= gsi_last_bb (entry_bb
);
9233 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9235 if (fd
->collapse
> 1)
9237 int first_zero_iter
= -1, dummy
= -1;
9238 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9240 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9241 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9242 fin_bb
, first_zero_iter
,
9243 dummy_bb
, dummy
, l2_dom_bb
);
9246 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9247 t
= integer_one_node
;
9249 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9250 fold_convert (type
, fd
->loop
.n1
),
9251 fold_convert (type
, fd
->loop
.n2
));
9252 if (fd
->collapse
== 1
9253 && TYPE_UNSIGNED (type
)
9254 && (t
== NULL_TREE
|| !integer_onep (t
)))
9256 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9257 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9258 true, GSI_SAME_STMT
);
9259 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9260 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9261 true, GSI_SAME_STMT
);
9262 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9263 NULL_TREE
, NULL_TREE
);
9264 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9265 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9266 expand_omp_regimplify_p
, NULL
, NULL
)
9267 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9268 expand_omp_regimplify_p
, NULL
, NULL
))
9270 gsi
= gsi_for_stmt (cond_stmt
);
9271 gimple_regimplify_operands (cond_stmt
, &gsi
);
9273 ep
= split_block (entry_bb
, cond_stmt
);
9274 ep
->flags
= EDGE_TRUE_VALUE
;
9275 entry_bb
= ep
->dest
;
9276 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9277 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9278 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9279 if (gimple_in_ssa_p (cfun
))
9281 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9282 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9283 !gsi_end_p (gpi
); gsi_next (&gpi
))
9285 gphi
*phi
= gpi
.phi ();
9286 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9287 ep
, UNKNOWN_LOCATION
);
9290 gsi
= gsi_last_bb (entry_bb
);
9293 switch (gimple_omp_for_kind (fd
->for_stmt
))
9295 case GF_OMP_FOR_KIND_FOR
:
9296 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9297 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9299 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9300 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9301 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9306 nthreads
= build_call_expr (nthreads
, 0);
9307 nthreads
= fold_convert (itype
, nthreads
);
9308 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9309 true, GSI_SAME_STMT
);
9310 threadid
= build_call_expr (threadid
, 0);
9311 threadid
= fold_convert (itype
, threadid
);
9312 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9313 true, GSI_SAME_STMT
);
9317 step
= fd
->loop
.step
;
9318 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9320 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9321 OMP_CLAUSE__LOOPTEMP_
);
9322 gcc_assert (innerc
);
9323 n1
= OMP_CLAUSE_DECL (innerc
);
9324 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9325 OMP_CLAUSE__LOOPTEMP_
);
9326 gcc_assert (innerc
);
9327 n2
= OMP_CLAUSE_DECL (innerc
);
9329 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9330 true, NULL_TREE
, true, GSI_SAME_STMT
);
9331 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9332 true, NULL_TREE
, true, GSI_SAME_STMT
);
9333 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9334 true, NULL_TREE
, true, GSI_SAME_STMT
);
9336 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9337 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9338 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9339 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9340 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9341 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9342 fold_build1 (NEGATE_EXPR
, itype
, t
),
9343 fold_build1 (NEGATE_EXPR
, itype
, step
));
9345 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9346 t
= fold_convert (itype
, t
);
9347 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9349 q
= create_tmp_reg (itype
, "q");
9350 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9351 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9352 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9354 tt
= create_tmp_reg (itype
, "tt");
9355 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9356 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9357 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9359 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9360 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9361 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9363 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9364 gsi
= gsi_last_bb (second_bb
);
9365 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9367 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9369 gassign
*assign_stmt
9370 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9371 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9373 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9374 gsi
= gsi_last_bb (third_bb
);
9375 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9377 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9378 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9379 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9381 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9382 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9384 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9385 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9387 /* Remove the GIMPLE_OMP_FOR statement. */
9388 gsi_remove (&gsi
, true);
9390 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9391 gsi
= gsi_start_bb (seq_start_bb
);
9393 tree startvar
= fd
->loop
.v
;
9394 tree endvar
= NULL_TREE
;
9396 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9398 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9399 ? gimple_omp_parallel_clauses (inner_stmt
)
9400 : gimple_omp_for_clauses (inner_stmt
);
9401 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9402 gcc_assert (innerc
);
9403 startvar
= OMP_CLAUSE_DECL (innerc
);
9404 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9405 OMP_CLAUSE__LOOPTEMP_
);
9406 gcc_assert (innerc
);
9407 endvar
= OMP_CLAUSE_DECL (innerc
);
9408 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9409 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9412 for (i
= 1; i
< fd
->collapse
; i
++)
9414 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9415 OMP_CLAUSE__LOOPTEMP_
);
9416 gcc_assert (innerc
);
9418 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9419 OMP_CLAUSE__LOOPTEMP_
);
9422 /* If needed (distribute parallel for with lastprivate),
9423 propagate down the total number of iterations. */
9424 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9426 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9427 GSI_CONTINUE_LINKING
);
9428 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9429 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9433 t
= fold_convert (itype
, s0
);
9434 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9435 if (POINTER_TYPE_P (type
))
9436 t
= fold_build_pointer_plus (n1
, t
);
9438 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9439 t
= fold_convert (TREE_TYPE (startvar
), t
);
9440 t
= force_gimple_operand_gsi (&gsi
, t
,
9442 && TREE_ADDRESSABLE (startvar
),
9443 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9444 assign_stmt
= gimple_build_assign (startvar
, t
);
9445 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9447 t
= fold_convert (itype
, e0
);
9448 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9449 if (POINTER_TYPE_P (type
))
9450 t
= fold_build_pointer_plus (n1
, t
);
9452 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9453 t
= fold_convert (TREE_TYPE (startvar
), t
);
9454 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9455 false, GSI_CONTINUE_LINKING
);
9458 assign_stmt
= gimple_build_assign (endvar
, e
);
9459 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9460 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9461 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9463 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9464 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9466 /* Handle linear clause adjustments. */
9467 tree itercnt
= NULL_TREE
;
9468 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9469 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9470 c
; c
= OMP_CLAUSE_CHAIN (c
))
9471 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9472 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9474 tree d
= OMP_CLAUSE_DECL (c
);
9475 bool is_ref
= is_reference (d
);
9476 tree t
= d
, a
, dest
;
9478 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9479 if (itercnt
== NULL_TREE
)
9481 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9483 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9484 fold_convert (itype
, n1
),
9485 fold_convert (itype
, fd
->loop
.n1
));
9486 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9487 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9488 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9490 GSI_CONTINUE_LINKING
);
9495 tree type
= TREE_TYPE (t
);
9496 if (POINTER_TYPE_P (type
))
9498 a
= fold_build2 (MULT_EXPR
, type
,
9499 fold_convert (type
, itercnt
),
9500 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9501 dest
= unshare_expr (t
);
9502 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9503 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9504 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9505 false, GSI_CONTINUE_LINKING
);
9506 assign_stmt
= gimple_build_assign (dest
, t
);
9507 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9509 if (fd
->collapse
> 1)
9510 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9514 /* The code controlling the sequential loop replaces the
9515 GIMPLE_OMP_CONTINUE. */
9516 gsi
= gsi_last_bb (cont_bb
);
9517 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9518 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9519 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9520 vback
= gimple_omp_continue_control_def (cont_stmt
);
9522 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9524 if (POINTER_TYPE_P (type
))
9525 t
= fold_build_pointer_plus (vmain
, step
);
9527 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9528 t
= force_gimple_operand_gsi (&gsi
, t
,
9530 && TREE_ADDRESSABLE (vback
),
9531 NULL_TREE
, true, GSI_SAME_STMT
);
9532 assign_stmt
= gimple_build_assign (vback
, t
);
9533 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9535 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9536 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9538 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9541 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9542 gsi_remove (&gsi
, true);
9544 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9545 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9548 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9549 gsi
= gsi_last_bb (exit_bb
);
9550 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9552 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9553 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9555 gsi_remove (&gsi
, true);
9557 /* Connect all the blocks. */
9558 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9559 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9560 ep
= find_edge (entry_bb
, second_bb
);
9561 ep
->flags
= EDGE_TRUE_VALUE
;
9562 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9563 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9564 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9568 ep
= find_edge (cont_bb
, body_bb
);
9571 ep
= BRANCH_EDGE (cont_bb
);
9572 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9574 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9579 else if (fd
->collapse
> 1)
9582 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9585 ep
->flags
= EDGE_TRUE_VALUE
;
9586 find_edge (cont_bb
, fin_bb
)->flags
9587 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9590 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9591 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9592 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9594 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9595 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9596 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9597 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9599 struct loop
*loop
= body_bb
->loop_father
;
9600 if (loop
!= entry_bb
->loop_father
)
9602 gcc_assert (loop
->header
== body_bb
);
9603 gcc_assert (broken_loop
9604 || loop
->latch
== region
->cont
9605 || single_pred (loop
->latch
) == region
->cont
);
9609 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9611 loop
= alloc_loop ();
9612 loop
->header
= body_bb
;
9613 if (collapse_bb
== NULL
)
9614 loop
->latch
= cont_bb
;
9615 add_loop (loop
, body_bb
->loop_father
);
9619 /* Return phi in E->DEST with ARG on edge E. */
9622 find_phi_with_arg_on_edge (tree arg
, edge e
)
9624 basic_block bb
= e
->dest
;
9626 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9630 gphi
*phi
= gpi
.phi ();
9631 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9638 /* A subroutine of expand_omp_for. Generate code for a parallel
9639 loop with static schedule and a specified chunk size. Given
9642 for (V = N1; V cond N2; V += STEP) BODY;
9644 where COND is "<" or ">", we generate pseudocode
9646 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9651 if ((__typeof (V)) -1 > 0 && cond is >)
9652 n = -(adj + N2 - N1) / -STEP;
9654 n = (adj + N2 - N1) / STEP;
9656 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9657 here so that V is defined
9658 if the loop is not entered
9660 s0 = (trip * nthreads + threadid) * CHUNK;
9661 e0 = min(s0 + CHUNK, n);
9662 if (s0 < n) goto L1; else goto L4;
9669 if (V cond e) goto L2; else goto L3;
9677 expand_omp_for_static_chunk (struct omp_region
*region
,
9678 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9680 tree n
, s0
, e0
, e
, t
;
9681 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9682 tree type
, itype
, vmain
, vback
, vextra
;
9683 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9684 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9685 gimple_stmt_iterator gsi
;
9687 bool broken_loop
= region
->cont
== NULL
;
9688 tree
*counts
= NULL
;
9691 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9692 if (POINTER_TYPE_P (type
))
9693 itype
= signed_type_for (type
);
9695 entry_bb
= region
->entry
;
9696 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9698 iter_part_bb
= se
->dest
;
9699 cont_bb
= region
->cont
;
9700 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9701 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9702 gcc_assert (broken_loop
9703 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9704 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9705 body_bb
= single_succ (seq_start_bb
);
9708 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9709 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9710 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9711 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9713 exit_bb
= region
->exit
;
9715 /* Trip and adjustment setup goes in ENTRY_BB. */
9716 gsi
= gsi_last_bb (entry_bb
);
9717 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9719 if (fd
->collapse
> 1)
9721 int first_zero_iter
= -1, dummy
= -1;
9722 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9724 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9725 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9726 fin_bb
, first_zero_iter
,
9727 dummy_bb
, dummy
, l2_dom_bb
);
9730 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9731 t
= integer_one_node
;
9733 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9734 fold_convert (type
, fd
->loop
.n1
),
9735 fold_convert (type
, fd
->loop
.n2
));
9736 if (fd
->collapse
== 1
9737 && TYPE_UNSIGNED (type
)
9738 && (t
== NULL_TREE
|| !integer_onep (t
)))
9740 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9741 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9742 true, GSI_SAME_STMT
);
9743 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9744 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9745 true, GSI_SAME_STMT
);
9746 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9747 NULL_TREE
, NULL_TREE
);
9748 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9749 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9750 expand_omp_regimplify_p
, NULL
, NULL
)
9751 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9752 expand_omp_regimplify_p
, NULL
, NULL
))
9754 gsi
= gsi_for_stmt (cond_stmt
);
9755 gimple_regimplify_operands (cond_stmt
, &gsi
);
9757 se
= split_block (entry_bb
, cond_stmt
);
9758 se
->flags
= EDGE_TRUE_VALUE
;
9759 entry_bb
= se
->dest
;
9760 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9761 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9762 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9763 if (gimple_in_ssa_p (cfun
))
9765 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9766 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9767 !gsi_end_p (gpi
); gsi_next (&gpi
))
9769 gphi
*phi
= gpi
.phi ();
9770 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9771 se
, UNKNOWN_LOCATION
);
9774 gsi
= gsi_last_bb (entry_bb
);
9777 switch (gimple_omp_for_kind (fd
->for_stmt
))
9779 case GF_OMP_FOR_KIND_FOR
:
9780 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9781 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9783 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9784 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9785 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9790 nthreads
= build_call_expr (nthreads
, 0);
9791 nthreads
= fold_convert (itype
, nthreads
);
9792 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9793 true, GSI_SAME_STMT
);
9794 threadid
= build_call_expr (threadid
, 0);
9795 threadid
= fold_convert (itype
, threadid
);
9796 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9797 true, GSI_SAME_STMT
);
9801 step
= fd
->loop
.step
;
9802 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9804 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9805 OMP_CLAUSE__LOOPTEMP_
);
9806 gcc_assert (innerc
);
9807 n1
= OMP_CLAUSE_DECL (innerc
);
9808 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9809 OMP_CLAUSE__LOOPTEMP_
);
9810 gcc_assert (innerc
);
9811 n2
= OMP_CLAUSE_DECL (innerc
);
9813 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9814 true, NULL_TREE
, true, GSI_SAME_STMT
);
9815 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9816 true, NULL_TREE
, true, GSI_SAME_STMT
);
9817 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9818 true, NULL_TREE
, true, GSI_SAME_STMT
);
9819 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9820 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9822 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9825 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9826 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9827 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9828 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9829 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9830 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9831 fold_build1 (NEGATE_EXPR
, itype
, t
),
9832 fold_build1 (NEGATE_EXPR
, itype
, step
));
9834 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9835 t
= fold_convert (itype
, t
);
9836 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9837 true, GSI_SAME_STMT
);
9839 trip_var
= create_tmp_reg (itype
, ".trip");
9840 if (gimple_in_ssa_p (cfun
))
9842 trip_init
= make_ssa_name (trip_var
);
9843 trip_main
= make_ssa_name (trip_var
);
9844 trip_back
= make_ssa_name (trip_var
);
9848 trip_init
= trip_var
;
9849 trip_main
= trip_var
;
9850 trip_back
= trip_var
;
9853 gassign
*assign_stmt
9854 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9855 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9857 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9858 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9859 if (POINTER_TYPE_P (type
))
9860 t
= fold_build_pointer_plus (n1
, t
);
9862 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9863 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9864 true, GSI_SAME_STMT
);
9866 /* Remove the GIMPLE_OMP_FOR. */
9867 gsi_remove (&gsi
, true);
9869 gimple_stmt_iterator gsif
= gsi
;
9871 /* Iteration space partitioning goes in ITER_PART_BB. */
9872 gsi
= gsi_last_bb (iter_part_bb
);
9874 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9875 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9876 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9877 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9878 false, GSI_CONTINUE_LINKING
);
9880 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9881 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9882 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9883 false, GSI_CONTINUE_LINKING
);
9885 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9886 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9888 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9889 gsi
= gsi_start_bb (seq_start_bb
);
9891 tree startvar
= fd
->loop
.v
;
9892 tree endvar
= NULL_TREE
;
9894 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9896 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9897 ? gimple_omp_parallel_clauses (inner_stmt
)
9898 : gimple_omp_for_clauses (inner_stmt
);
9899 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9900 gcc_assert (innerc
);
9901 startvar
= OMP_CLAUSE_DECL (innerc
);
9902 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9903 OMP_CLAUSE__LOOPTEMP_
);
9904 gcc_assert (innerc
);
9905 endvar
= OMP_CLAUSE_DECL (innerc
);
9906 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9907 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9910 for (i
= 1; i
< fd
->collapse
; i
++)
9912 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9913 OMP_CLAUSE__LOOPTEMP_
);
9914 gcc_assert (innerc
);
9916 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9917 OMP_CLAUSE__LOOPTEMP_
);
9920 /* If needed (distribute parallel for with lastprivate),
9921 propagate down the total number of iterations. */
9922 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9924 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9925 GSI_CONTINUE_LINKING
);
9926 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9927 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9932 t
= fold_convert (itype
, s0
);
9933 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9934 if (POINTER_TYPE_P (type
))
9935 t
= fold_build_pointer_plus (n1
, t
);
9937 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9938 t
= fold_convert (TREE_TYPE (startvar
), t
);
9939 t
= force_gimple_operand_gsi (&gsi
, t
,
9941 && TREE_ADDRESSABLE (startvar
),
9942 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9943 assign_stmt
= gimple_build_assign (startvar
, t
);
9944 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9946 t
= fold_convert (itype
, e0
);
9947 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9948 if (POINTER_TYPE_P (type
))
9949 t
= fold_build_pointer_plus (n1
, t
);
9951 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9952 t
= fold_convert (TREE_TYPE (startvar
), t
);
9953 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9954 false, GSI_CONTINUE_LINKING
);
9957 assign_stmt
= gimple_build_assign (endvar
, e
);
9958 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9959 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9960 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9962 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9963 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9965 /* Handle linear clause adjustments. */
9966 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9967 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9968 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9969 c
; c
= OMP_CLAUSE_CHAIN (c
))
9970 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9971 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9973 tree d
= OMP_CLAUSE_DECL (c
);
9974 bool is_ref
= is_reference (d
);
9975 tree t
= d
, a
, dest
;
9977 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9978 tree type
= TREE_TYPE (t
);
9979 if (POINTER_TYPE_P (type
))
9981 dest
= unshare_expr (t
);
9982 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9983 expand_omp_build_assign (&gsif
, v
, t
);
9984 if (itercnt
== NULL_TREE
)
9986 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9989 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9990 fold_convert (itype
, fd
->loop
.n1
));
9991 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9994 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9997 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9998 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
10000 GSI_CONTINUE_LINKING
);
10005 a
= fold_build2 (MULT_EXPR
, type
,
10006 fold_convert (type
, itercnt
),
10007 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
10008 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
10009 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
10010 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10011 false, GSI_CONTINUE_LINKING
);
10012 assign_stmt
= gimple_build_assign (dest
, t
);
10013 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10015 if (fd
->collapse
> 1)
10016 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10020 /* The code controlling the sequential loop goes in CONT_BB,
10021 replacing the GIMPLE_OMP_CONTINUE. */
10022 gsi
= gsi_last_bb (cont_bb
);
10023 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10024 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10025 vback
= gimple_omp_continue_control_def (cont_stmt
);
10027 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10029 if (POINTER_TYPE_P (type
))
10030 t
= fold_build_pointer_plus (vmain
, step
);
10032 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10033 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
10034 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10035 true, GSI_SAME_STMT
);
10036 assign_stmt
= gimple_build_assign (vback
, t
);
10037 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10039 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
10040 t
= build2 (EQ_EXPR
, boolean_type_node
,
10041 build_int_cst (itype
, 0),
10042 build_int_cst (itype
, 1));
10044 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10045 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10047 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10050 /* Remove GIMPLE_OMP_CONTINUE. */
10051 gsi_remove (&gsi
, true);
10053 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10054 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10056 /* Trip update code goes into TRIP_UPDATE_BB. */
10057 gsi
= gsi_start_bb (trip_update_bb
);
10059 t
= build_int_cst (itype
, 1);
10060 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
10061 assign_stmt
= gimple_build_assign (trip_back
, t
);
10062 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10065 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10066 gsi
= gsi_last_bb (exit_bb
);
10067 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
10069 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
10070 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
10072 gsi_remove (&gsi
, true);
10074 /* Connect the new blocks. */
10075 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
10076 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
10080 se
= find_edge (cont_bb
, body_bb
);
10083 se
= BRANCH_EDGE (cont_bb
);
10084 gcc_assert (single_succ (se
->dest
) == body_bb
);
10086 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10091 else if (fd
->collapse
> 1)
10094 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10097 se
->flags
= EDGE_TRUE_VALUE
;
10098 find_edge (cont_bb
, trip_update_bb
)->flags
10099 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10101 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
10104 if (gimple_in_ssa_p (cfun
))
10112 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
10114 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10115 remove arguments of the phi nodes in fin_bb. We need to create
10116 appropriate phi nodes in iter_part_bb instead. */
10117 se
= find_edge (iter_part_bb
, fin_bb
);
10118 re
= single_succ_edge (trip_update_bb
);
10119 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
10120 ene
= single_succ_edge (entry_bb
);
10122 psi
= gsi_start_phis (fin_bb
);
10123 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
10124 gsi_next (&psi
), ++i
)
10127 source_location locus
;
10130 t
= gimple_phi_result (phi
);
10131 gcc_assert (t
== redirect_edge_var_map_result (vm
));
10133 if (!single_pred_p (fin_bb
))
10134 t
= copy_ssa_name (t
, phi
);
10136 nphi
= create_phi_node (t
, iter_part_bb
);
10138 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
10139 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
10141 /* A special case -- fd->loop.v is not yet computed in
10142 iter_part_bb, we need to use vextra instead. */
10143 if (t
== fd
->loop
.v
)
10145 add_phi_arg (nphi
, t
, ene
, locus
);
10146 locus
= redirect_edge_var_map_location (vm
);
10147 tree back_arg
= redirect_edge_var_map_def (vm
);
10148 add_phi_arg (nphi
, back_arg
, re
, locus
);
10149 edge ce
= find_edge (cont_bb
, body_bb
);
10152 ce
= BRANCH_EDGE (cont_bb
);
10153 gcc_assert (single_succ (ce
->dest
) == body_bb
);
10154 ce
= single_succ_edge (ce
->dest
);
10156 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
10157 gcc_assert (inner_loop_phi
!= NULL
);
10158 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
10159 find_edge (seq_start_bb
, body_bb
), locus
);
10161 if (!single_pred_p (fin_bb
))
10162 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
10164 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
10165 redirect_edge_var_map_clear (re
);
10166 if (single_pred_p (fin_bb
))
10169 psi
= gsi_start_phis (fin_bb
);
10170 if (gsi_end_p (psi
))
10172 remove_phi_node (&psi
, false);
10175 /* Make phi node for trip. */
10176 phi
= create_phi_node (trip_main
, iter_part_bb
);
10177 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
10179 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
10184 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
10185 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
10186 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
10187 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10188 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10189 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
10190 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
10191 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10192 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10196 struct loop
*loop
= body_bb
->loop_father
;
10197 struct loop
*trip_loop
= alloc_loop ();
10198 trip_loop
->header
= iter_part_bb
;
10199 trip_loop
->latch
= trip_update_bb
;
10200 add_loop (trip_loop
, iter_part_bb
->loop_father
);
10202 if (loop
!= entry_bb
->loop_father
)
10204 gcc_assert (loop
->header
== body_bb
);
10205 gcc_assert (loop
->latch
== region
->cont
10206 || single_pred (loop
->latch
) == region
->cont
);
10207 trip_loop
->inner
= loop
;
10211 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10213 loop
= alloc_loop ();
10214 loop
->header
= body_bb
;
10215 if (collapse_bb
== NULL
)
10216 loop
->latch
= cont_bb
;
10217 add_loop (loop
, trip_loop
);
10222 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10224 for (V = N1; V cond N2; V += STEP) BODY;
10226 where COND is "<" or ">" or "!=", we generate pseudocode
10228 for (ind_var = low; ind_var < high; ind_var++)
10230 V = n1 + (ind_var * STEP)
10235 In the above pseudocode, low and high are function parameters of the
10236 child function. In the function below, we are inserting a temp.
10237 variable that will be making a call to two OMP functions that will not be
10238 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10239 with _Cilk_for). These functions are replaced with low and high
10240 by the function that handles taskreg. */
10244 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10246 bool broken_loop
= region
->cont
== NULL
;
10247 basic_block entry_bb
= region
->entry
;
10248 basic_block cont_bb
= region
->cont
;
10250 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10251 gcc_assert (broken_loop
10252 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10253 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10254 basic_block l1_bb
, l2_bb
;
10258 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10259 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10260 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10261 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10265 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10266 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10267 l2_bb
= single_succ (l1_bb
);
10269 basic_block exit_bb
= region
->exit
;
10270 basic_block l2_dom_bb
= NULL
;
10272 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10274 /* Below statements until the "tree high_val = ..." are pseudo statements
10275 used to pass information to be used by expand_omp_taskreg.
10276 low_val and high_val will be replaced by the __low and __high
10277 parameter from the child function.
10279 The call_exprs part is a place-holder, it is mainly used
10280 to distinctly identify to the top-level part that this is
10281 where we should put low and high (reasoning given in header
10285 = gimple_omp_parallel_child_fn (
10286 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10287 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10288 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10290 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10292 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10295 gcc_assert (low_val
&& high_val
);
10297 tree type
= TREE_TYPE (low_val
);
10298 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10299 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10301 /* Not needed in SSA form right now. */
10302 gcc_assert (!gimple_in_ssa_p (cfun
));
10303 if (l2_dom_bb
== NULL
)
10307 tree n2
= high_val
;
10309 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10311 /* Replace the GIMPLE_OMP_FOR statement. */
10312 gsi_replace (&gsi
, stmt
, true);
10316 /* Code to control the increment goes in the CONT_BB. */
10317 gsi
= gsi_last_bb (cont_bb
);
10318 stmt
= gsi_stmt (gsi
);
10319 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10320 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10321 build_one_cst (type
));
10323 /* Replace GIMPLE_OMP_CONTINUE. */
10324 gsi_replace (&gsi
, stmt
, true);
10327 /* Emit the condition in L1_BB. */
10328 gsi
= gsi_after_labels (l1_bb
);
10329 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10330 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10332 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10333 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10334 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10336 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10337 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10338 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10339 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10341 /* The condition is always '<' since the runtime will fill in the low
10342 and high values. */
10343 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10344 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10346 /* Remove GIMPLE_OMP_RETURN. */
10347 gsi
= gsi_last_bb (exit_bb
);
10348 gsi_remove (&gsi
, true);
10350 /* Connect the new blocks. */
10351 remove_edge (FALLTHRU_EDGE (entry_bb
));
10356 remove_edge (BRANCH_EDGE (entry_bb
));
10357 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10359 e
= BRANCH_EDGE (l1_bb
);
10360 ne
= FALLTHRU_EDGE (l1_bb
);
10361 e
->flags
= EDGE_TRUE_VALUE
;
10365 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10367 ne
= single_succ_edge (l1_bb
);
10368 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10371 ne
->flags
= EDGE_FALSE_VALUE
;
10372 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10373 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10375 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10376 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10377 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10381 struct loop
*loop
= alloc_loop ();
10382 loop
->header
= l1_bb
;
10383 loop
->latch
= cont_bb
;
10384 add_loop (loop
, l1_bb
->loop_father
);
10385 loop
->safelen
= INT_MAX
;
10388 /* Pick the correct library function based on the precision of the
10389 induction variable type. */
10390 tree lib_fun
= NULL_TREE
;
10391 if (TYPE_PRECISION (type
) == 32)
10392 lib_fun
= cilk_for_32_fndecl
;
10393 else if (TYPE_PRECISION (type
) == 64)
10394 lib_fun
= cilk_for_64_fndecl
;
10396 gcc_unreachable ();
10398 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10400 /* WS_ARGS contains the library function flavor to call:
10401 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10402 user-defined grain value. If the user does not define one, then zero
10403 is passed in by the parser. */
10404 vec_alloc (region
->ws_args
, 2);
10405 region
->ws_args
->quick_push (lib_fun
);
10406 region
->ws_args
->quick_push (fd
->chunk_size
);
10409 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10410 loop. Given parameters:
10412 for (V = N1; V cond N2; V += STEP) BODY;
10414 where COND is "<" or ">", we generate pseudocode
10422 if (V cond N2) goto L0; else goto L2;
10425 For collapsed loops, given parameters:
10427 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10428 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10429 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10432 we generate pseudocode
10438 count3 = (adj + N32 - N31) / STEP3;
10443 count2 = (adj + N22 - N21) / STEP2;
10448 count1 = (adj + N12 - N11) / STEP1;
10449 count = count1 * count2 * count3;
10459 V2 += (V3 cond3 N32) ? 0 : STEP2;
10460 V3 = (V3 cond3 N32) ? V3 : N31;
10461 V1 += (V2 cond2 N22) ? 0 : STEP1;
10462 V2 = (V2 cond2 N22) ? V2 : N21;
10464 if (V < count) goto L0; else goto L2;
10470 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10473 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10474 gimple_stmt_iterator gsi
;
10477 bool broken_loop
= region
->cont
== NULL
;
10479 tree
*counts
= NULL
;
10481 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10482 OMP_CLAUSE_SAFELEN
);
10483 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10484 OMP_CLAUSE__SIMDUID_
);
10487 type
= TREE_TYPE (fd
->loop
.v
);
10488 entry_bb
= region
->entry
;
10489 cont_bb
= region
->cont
;
10490 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10491 gcc_assert (broken_loop
10492 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10493 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10496 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10497 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10498 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10499 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10503 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10504 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10505 l2_bb
= single_succ (l1_bb
);
10507 exit_bb
= region
->exit
;
10510 gsi
= gsi_last_bb (entry_bb
);
10512 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10513 /* Not needed in SSA form right now. */
10514 gcc_assert (!gimple_in_ssa_p (cfun
));
10515 if (fd
->collapse
> 1)
10517 int first_zero_iter
= -1, dummy
= -1;
10518 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10520 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10521 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10522 zero_iter_bb
, first_zero_iter
,
10523 dummy_bb
, dummy
, l2_dom_bb
);
10525 if (l2_dom_bb
== NULL
)
10530 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10532 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10533 OMP_CLAUSE__LOOPTEMP_
);
10534 gcc_assert (innerc
);
10535 n1
= OMP_CLAUSE_DECL (innerc
);
10536 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10537 OMP_CLAUSE__LOOPTEMP_
);
10538 gcc_assert (innerc
);
10539 n2
= OMP_CLAUSE_DECL (innerc
);
10540 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10541 fold_convert (type
, n1
));
10542 if (fd
->collapse
> 1)
10545 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10551 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10552 fold_convert (type
, fd
->loop
.n1
));
10553 if (fd
->collapse
> 1)
10554 for (i
= 0; i
< fd
->collapse
; i
++)
10556 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10557 if (POINTER_TYPE_P (itype
))
10558 itype
= signed_type_for (itype
);
10559 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10560 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10564 /* Remove the GIMPLE_OMP_FOR statement. */
10565 gsi_remove (&gsi
, true);
10569 /* Code to control the increment goes in the CONT_BB. */
10570 gsi
= gsi_last_bb (cont_bb
);
10571 stmt
= gsi_stmt (gsi
);
10572 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10574 if (POINTER_TYPE_P (type
))
10575 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10577 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10578 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10580 if (fd
->collapse
> 1)
10582 i
= fd
->collapse
- 1;
10583 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10585 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10586 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10590 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10591 fd
->loops
[i
].step
);
10592 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10593 fd
->loops
[i
].v
, t
);
10595 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10597 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10599 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10600 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10601 if (POINTER_TYPE_P (itype2
))
10602 itype2
= signed_type_for (itype2
);
10603 t
= build3 (COND_EXPR
, itype2
,
10604 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10606 fold_convert (itype
, fd
->loops
[i
].n2
)),
10607 build_int_cst (itype2
, 0),
10608 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10609 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10610 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10612 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10613 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10615 t
= build3 (COND_EXPR
, itype
,
10616 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10618 fold_convert (itype
, fd
->loops
[i
].n2
)),
10620 fold_convert (itype
, fd
->loops
[i
].n1
));
10621 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10625 /* Remove GIMPLE_OMP_CONTINUE. */
10626 gsi_remove (&gsi
, true);
10629 /* Emit the condition in L1_BB. */
10630 gsi
= gsi_start_bb (l1_bb
);
10632 t
= fold_convert (type
, n2
);
10633 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10634 false, GSI_CONTINUE_LINKING
);
10635 tree v
= fd
->loop
.v
;
10636 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10637 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10638 false, GSI_CONTINUE_LINKING
);
10639 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10640 cond_stmt
= gimple_build_cond_empty (t
);
10641 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10642 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10644 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10647 gsi
= gsi_for_stmt (cond_stmt
);
10648 gimple_regimplify_operands (cond_stmt
, &gsi
);
10651 /* Remove GIMPLE_OMP_RETURN. */
10652 gsi
= gsi_last_bb (exit_bb
);
10653 gsi_remove (&gsi
, true);
10655 /* Connect the new blocks. */
10656 remove_edge (FALLTHRU_EDGE (entry_bb
));
10660 remove_edge (BRANCH_EDGE (entry_bb
));
10661 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10663 e
= BRANCH_EDGE (l1_bb
);
10664 ne
= FALLTHRU_EDGE (l1_bb
);
10665 e
->flags
= EDGE_TRUE_VALUE
;
10669 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10671 ne
= single_succ_edge (l1_bb
);
10672 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10675 ne
->flags
= EDGE_FALSE_VALUE
;
10676 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10677 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10679 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10680 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10681 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10685 struct loop
*loop
= alloc_loop ();
10686 loop
->header
= l1_bb
;
10687 loop
->latch
= cont_bb
;
10688 add_loop (loop
, l1_bb
->loop_father
);
10689 if (safelen
== NULL_TREE
)
10690 loop
->safelen
= INT_MAX
;
10693 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10694 if (TREE_CODE (safelen
) != INTEGER_CST
)
10696 else if (!tree_fits_uhwi_p (safelen
)
10697 || tree_to_uhwi (safelen
) > INT_MAX
)
10698 loop
->safelen
= INT_MAX
;
10700 loop
->safelen
= tree_to_uhwi (safelen
);
10701 if (loop
->safelen
== 1)
10706 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10707 cfun
->has_simduid_loops
= true;
10709 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10711 if ((flag_tree_loop_vectorize
10712 || (!global_options_set
.x_flag_tree_loop_vectorize
10713 && !global_options_set
.x_flag_tree_vectorize
))
10714 && flag_tree_loop_optimize
10715 && loop
->safelen
> 1)
10717 loop
->force_vectorize
= true;
10718 cfun
->has_force_vectorize_loops
= true;
10722 cfun
->has_simduid_loops
= true;
10725 /* Taskloop construct is represented after gimplification with
10726 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10727 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10728 which should just compute all the needed loop temporaries
10729 for GIMPLE_OMP_TASK. */
10732 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10733 struct omp_for_data
*fd
,
10734 gimple
*inner_stmt
)
10736 tree type
, bias
= NULL_TREE
;
10737 basic_block entry_bb
, cont_bb
, exit_bb
;
10738 gimple_stmt_iterator gsi
;
10739 gassign
*assign_stmt
;
10740 tree
*counts
= NULL
;
10743 gcc_assert (inner_stmt
);
10744 gcc_assert (region
->cont
);
10745 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10746 && gimple_omp_task_taskloop_p (inner_stmt
));
10747 type
= TREE_TYPE (fd
->loop
.v
);
10749 /* See if we need to bias by LLONG_MIN. */
10750 if (fd
->iter_type
== long_long_unsigned_type_node
10751 && TREE_CODE (type
) == INTEGER_TYPE
10752 && !TYPE_UNSIGNED (type
))
10756 if (fd
->loop
.cond_code
== LT_EXPR
)
10759 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10763 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10766 if (TREE_CODE (n1
) != INTEGER_CST
10767 || TREE_CODE (n2
) != INTEGER_CST
10768 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10769 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10772 entry_bb
= region
->entry
;
10773 cont_bb
= region
->cont
;
10774 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10775 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10776 exit_bb
= region
->exit
;
10778 gsi
= gsi_last_bb (entry_bb
);
10779 gimple
*for_stmt
= gsi_stmt (gsi
);
10780 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10781 if (fd
->collapse
> 1)
10783 int first_zero_iter
= -1, dummy
= -1;
10784 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10786 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10787 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10788 zero_iter_bb
, first_zero_iter
,
10789 dummy_bb
, dummy
, l2_dom_bb
);
10793 /* Some counts[i] vars might be uninitialized if
10794 some loop has zero iterations. But the body shouldn't
10795 be executed in that case, so just avoid uninit warnings. */
10796 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10797 if (SSA_VAR_P (counts
[i
]))
10798 TREE_NO_WARNING (counts
[i
]) = 1;
10800 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10801 entry_bb
= e
->dest
;
10802 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10803 gsi
= gsi_last_bb (entry_bb
);
10804 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10805 get_immediate_dominator (CDI_DOMINATORS
,
10813 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10814 && TYPE_PRECISION (TREE_TYPE (t0
))
10815 != TYPE_PRECISION (fd
->iter_type
))
10817 /* Avoid casting pointers to integer of a different size. */
10818 tree itype
= signed_type_for (type
);
10819 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10820 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10824 t1
= fold_convert (fd
->iter_type
, t1
);
10825 t0
= fold_convert (fd
->iter_type
, t0
);
10829 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10830 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10833 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10834 OMP_CLAUSE__LOOPTEMP_
);
10835 gcc_assert (innerc
);
10836 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10837 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10838 gcc_assert (innerc
);
10839 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10840 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10842 gcc_assert (innerc
);
10843 for (i
= 1; i
< fd
->collapse
; i
++)
10845 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10846 OMP_CLAUSE__LOOPTEMP_
);
10847 gcc_assert (innerc
);
10849 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10850 OMP_CLAUSE__LOOPTEMP_
);
10853 /* If needed (inner taskloop has lastprivate clause), propagate
10854 down the total number of iterations. */
10855 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10857 GSI_CONTINUE_LINKING
);
10858 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10859 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10863 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10864 GSI_CONTINUE_LINKING
);
10865 assign_stmt
= gimple_build_assign (startvar
, t0
);
10866 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10868 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10869 GSI_CONTINUE_LINKING
);
10870 assign_stmt
= gimple_build_assign (endvar
, t1
);
10871 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10872 if (fd
->collapse
> 1)
10873 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10875 /* Remove the GIMPLE_OMP_FOR statement. */
10876 gsi
= gsi_for_stmt (for_stmt
);
10877 gsi_remove (&gsi
, true);
10879 gsi
= gsi_last_bb (cont_bb
);
10880 gsi_remove (&gsi
, true);
10882 gsi
= gsi_last_bb (exit_bb
);
10883 gsi_remove (&gsi
, true);
10885 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10886 remove_edge (BRANCH_EDGE (entry_bb
));
10887 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10888 remove_edge (BRANCH_EDGE (cont_bb
));
10889 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10890 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10891 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10894 /* Taskloop construct is represented after gimplification with
10895 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10896 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10897 GOMP_taskloop{,_ull} function arranges for each task to be given just
10898 a single range of iterations. */
10901 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10902 struct omp_for_data
*fd
,
10903 gimple
*inner_stmt
)
10905 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10906 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10907 basic_block fin_bb
;
10908 gimple_stmt_iterator gsi
;
10910 bool broken_loop
= region
->cont
== NULL
;
10911 tree
*counts
= NULL
;
10914 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10915 if (POINTER_TYPE_P (type
))
10916 itype
= signed_type_for (type
);
10918 /* See if we need to bias by LLONG_MIN. */
10919 if (fd
->iter_type
== long_long_unsigned_type_node
10920 && TREE_CODE (type
) == INTEGER_TYPE
10921 && !TYPE_UNSIGNED (type
))
10925 if (fd
->loop
.cond_code
== LT_EXPR
)
10928 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10932 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10935 if (TREE_CODE (n1
) != INTEGER_CST
10936 || TREE_CODE (n2
) != INTEGER_CST
10937 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10938 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10941 entry_bb
= region
->entry
;
10942 cont_bb
= region
->cont
;
10943 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10944 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10945 gcc_assert (broken_loop
10946 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10947 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10950 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10951 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10953 exit_bb
= region
->exit
;
10955 /* Iteration space partitioning goes in ENTRY_BB. */
10956 gsi
= gsi_last_bb (entry_bb
);
10957 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10959 if (fd
->collapse
> 1)
10961 int first_zero_iter
= -1, dummy
= -1;
10962 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10964 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10965 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10966 fin_bb
, first_zero_iter
,
10967 dummy_bb
, dummy
, l2_dom_bb
);
10971 t
= integer_one_node
;
10973 step
= fd
->loop
.step
;
10974 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10975 OMP_CLAUSE__LOOPTEMP_
);
10976 gcc_assert (innerc
);
10977 n1
= OMP_CLAUSE_DECL (innerc
);
10978 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10979 gcc_assert (innerc
);
10980 n2
= OMP_CLAUSE_DECL (innerc
);
10983 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10984 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10986 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10987 true, NULL_TREE
, true, GSI_SAME_STMT
);
10988 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10989 true, NULL_TREE
, true, GSI_SAME_STMT
);
10990 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10991 true, NULL_TREE
, true, GSI_SAME_STMT
);
10993 tree startvar
= fd
->loop
.v
;
10994 tree endvar
= NULL_TREE
;
10996 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10998 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10999 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
11000 gcc_assert (innerc
);
11001 startvar
= OMP_CLAUSE_DECL (innerc
);
11002 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
11003 OMP_CLAUSE__LOOPTEMP_
);
11004 gcc_assert (innerc
);
11005 endvar
= OMP_CLAUSE_DECL (innerc
);
11007 t
= fold_convert (TREE_TYPE (startvar
), n1
);
11008 t
= force_gimple_operand_gsi (&gsi
, t
,
11010 && TREE_ADDRESSABLE (startvar
),
11011 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
11012 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
11013 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11015 t
= fold_convert (TREE_TYPE (startvar
), n2
);
11016 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
11017 false, GSI_CONTINUE_LINKING
);
11020 assign_stmt
= gimple_build_assign (endvar
, e
);
11021 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11022 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
11023 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
11025 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
11026 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11028 if (fd
->collapse
> 1)
11029 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
11033 /* The code controlling the sequential loop replaces the
11034 GIMPLE_OMP_CONTINUE. */
11035 gsi
= gsi_last_bb (cont_bb
);
11036 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11037 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
11038 vmain
= gimple_omp_continue_control_use (cont_stmt
);
11039 vback
= gimple_omp_continue_control_def (cont_stmt
);
11041 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
11043 if (POINTER_TYPE_P (type
))
11044 t
= fold_build_pointer_plus (vmain
, step
);
11046 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
11047 t
= force_gimple_operand_gsi (&gsi
, t
,
11049 && TREE_ADDRESSABLE (vback
),
11050 NULL_TREE
, true, GSI_SAME_STMT
);
11051 assign_stmt
= gimple_build_assign (vback
, t
);
11052 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
11054 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
11055 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
11057 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
11060 /* Remove the GIMPLE_OMP_CONTINUE statement. */
11061 gsi_remove (&gsi
, true);
11063 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
11064 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
11067 /* Remove the GIMPLE_OMP_FOR statement. */
11068 gsi
= gsi_for_stmt (fd
->for_stmt
);
11069 gsi_remove (&gsi
, true);
11071 /* Remove the GIMPLE_OMP_RETURN statement. */
11072 gsi
= gsi_last_bb (exit_bb
);
11073 gsi_remove (&gsi
, true);
11075 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
11077 remove_edge (BRANCH_EDGE (entry_bb
));
11080 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
11081 region
->outer
->cont
= NULL
;
11084 /* Connect all the blocks. */
11087 ep
= find_edge (cont_bb
, body_bb
);
11088 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11093 else if (fd
->collapse
> 1)
11096 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
11099 ep
->flags
= EDGE_TRUE_VALUE
;
11100 find_edge (cont_bb
, fin_bb
)->flags
11101 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
11104 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
11105 recompute_dominator (CDI_DOMINATORS
, body_bb
));
11107 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
11108 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
11110 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
11112 struct loop
*loop
= alloc_loop ();
11113 loop
->header
= body_bb
;
11114 if (collapse_bb
== NULL
)
11115 loop
->latch
= cont_bb
;
11116 add_loop (loop
, body_bb
->loop_father
);
11120 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11121 partitioned loop. The lowering here is abstracted, in that the
11122 loop parameters are passed through internal functions, which are
11123 further lowered by oacc_device_lower, once we get to the target
11124 compiler. The loop is of the form:
11126 for (V = B; V LTGT E; V += S) {BODY}
11128 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11129 (constant 0 for no chunking) and we will have a GWV partitioning
11130 mask, specifying dimensions over which the loop is to be
11131 partitioned (see note below). We generate code that looks like:
11133 <entry_bb> [incoming FALL->body, BRANCH->exit]
11134 typedef signedintify (typeof (V)) T; // underlying signed integral type
11137 T DIR = LTGT == '<' ? +1 : -1;
11138 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11139 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11141 <head_bb> [created by splitting end of entry_bb]
11142 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11143 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11144 if (!(offset LTGT bound)) goto bottom_bb;
11146 <body_bb> [incoming]
11150 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11152 if (offset LTGT bound) goto body_bb; [*]
11154 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11156 if (chunk < chunk_max) goto head_bb;
11158 <exit_bb> [incoming]
11159 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11161 [*] Needed if V live at end of loop
11163 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11164 transition, and will be specified by a more general mechanism shortly.
11168 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
11170 tree v
= fd
->loop
.v
;
11171 enum tree_code cond_code
= fd
->loop
.cond_code
;
11172 enum tree_code plus_code
= PLUS_EXPR
;
11174 tree chunk_size
= integer_minus_one_node
;
11175 tree gwv
= integer_zero_node
;
11176 tree iter_type
= TREE_TYPE (v
);
11177 tree diff_type
= iter_type
;
11178 tree plus_type
= iter_type
;
11179 struct oacc_collapse
*counts
= NULL
;
11181 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
11182 == GF_OMP_FOR_KIND_OACC_LOOP
);
11183 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
11184 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
11186 if (POINTER_TYPE_P (iter_type
))
11188 plus_code
= POINTER_PLUS_EXPR
;
11189 plus_type
= sizetype
;
11191 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
11192 diff_type
= signed_type_for (diff_type
);
11194 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
11195 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
11196 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
11197 basic_block bottom_bb
= NULL
;
11199 /* entry_bb has two sucessors; the branch edge is to the exit
11200 block, fallthrough edge to body. */
11201 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
11202 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
11204 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11205 body_bb, or to a block whose only successor is the body_bb. Its
11206 fallthrough successor is the final block (same as the branch
11207 successor of the entry_bb). */
11210 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
11211 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
11213 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
11214 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
11217 gcc_assert (!gimple_in_ssa_p (cfun
));
11219 /* The exit block only has entry_bb and cont_bb as predecessors. */
11220 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
11223 tree chunk_max
= NULL_TREE
;
11224 tree bound
, offset
;
11225 tree step
= create_tmp_var (diff_type
, ".step");
11226 bool up
= cond_code
== LT_EXPR
;
11227 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
11228 bool chunking
= !gimple_in_ssa_p (cfun
);;
11231 /* SSA instances. */
11232 tree offset_incr
= NULL_TREE
;
11233 tree offset_init
= NULL_TREE
;
11235 gimple_stmt_iterator gsi
;
11241 edge split
, be
, fte
;
11243 /* Split the end of entry_bb to create head_bb. */
11244 split
= split_block (entry_bb
, last_stmt (entry_bb
));
11245 basic_block head_bb
= split
->dest
;
11246 entry_bb
= split
->src
;
11248 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11249 gsi
= gsi_last_bb (entry_bb
);
11250 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
11251 loc
= gimple_location (for_stmt
);
11253 if (gimple_in_ssa_p (cfun
))
11255 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11256 gcc_assert (integer_zerop (fd
->loop
.n1
));
11257 /* The SSA parallelizer does gang parallelism. */
11258 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11261 if (fd
->collapse
> 1)
11263 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11264 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11265 TREE_TYPE (fd
->loop
.n2
));
11267 if (SSA_VAR_P (fd
->loop
.n2
))
11269 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11270 true, GSI_SAME_STMT
);
11271 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11272 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11277 tree b
= fd
->loop
.n1
;
11278 tree e
= fd
->loop
.n2
;
11279 tree s
= fd
->loop
.step
;
11281 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11282 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11284 /* Convert the step, avoiding possible unsigned->signed overflow. */
11285 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11287 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11288 s
= fold_convert (diff_type
, s
);
11290 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11291 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11294 chunk_size
= integer_zero_node
;
11295 expr
= fold_convert (diff_type
, chunk_size
);
11296 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11297 NULL_TREE
, true, GSI_SAME_STMT
);
11298 /* Determine the range, avoiding possible unsigned->signed overflow. */
11299 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11300 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11301 fold_convert (plus_type
, negating
? b
: e
),
11302 fold_convert (plus_type
, negating
? e
: b
));
11303 expr
= fold_convert (diff_type
, expr
);
11305 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11306 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11307 NULL_TREE
, true, GSI_SAME_STMT
);
11309 chunk_no
= build_int_cst (diff_type
, 0);
11312 gcc_assert (!gimple_in_ssa_p (cfun
));
11315 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11316 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11318 ass
= gimple_build_assign (chunk_no
, expr
);
11319 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11321 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11322 build_int_cst (integer_type_node
,
11323 IFN_GOACC_LOOP_CHUNKS
),
11324 dir
, range
, s
, chunk_size
, gwv
);
11325 gimple_call_set_lhs (call
, chunk_max
);
11326 gimple_set_location (call
, loc
);
11327 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11330 chunk_size
= chunk_no
;
11332 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11333 build_int_cst (integer_type_node
,
11334 IFN_GOACC_LOOP_STEP
),
11335 dir
, range
, s
, chunk_size
, gwv
);
11336 gimple_call_set_lhs (call
, step
);
11337 gimple_set_location (call
, loc
);
11338 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11340 /* Remove the GIMPLE_OMP_FOR. */
11341 gsi_remove (&gsi
, true);
11343 /* Fixup edges from head_bb */
11344 be
= BRANCH_EDGE (head_bb
);
11345 fte
= FALLTHRU_EDGE (head_bb
);
11346 be
->flags
|= EDGE_FALSE_VALUE
;
11347 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11349 basic_block body_bb
= fte
->dest
;
11351 if (gimple_in_ssa_p (cfun
))
11353 gsi
= gsi_last_bb (cont_bb
);
11354 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11356 offset
= gimple_omp_continue_control_use (cont_stmt
);
11357 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11361 offset
= create_tmp_var (diff_type
, ".offset");
11362 offset_init
= offset_incr
= offset
;
11364 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11366 /* Loop offset & bound go into head_bb. */
11367 gsi
= gsi_start_bb (head_bb
);
11369 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11370 build_int_cst (integer_type_node
,
11371 IFN_GOACC_LOOP_OFFSET
),
11373 chunk_size
, gwv
, chunk_no
);
11374 gimple_call_set_lhs (call
, offset_init
);
11375 gimple_set_location (call
, loc
);
11376 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11378 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11379 build_int_cst (integer_type_node
,
11380 IFN_GOACC_LOOP_BOUND
),
11382 chunk_size
, gwv
, offset_init
);
11383 gimple_call_set_lhs (call
, bound
);
11384 gimple_set_location (call
, loc
);
11385 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11387 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11388 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11389 GSI_CONTINUE_LINKING
);
11391 /* V assignment goes into body_bb. */
11392 if (!gimple_in_ssa_p (cfun
))
11394 gsi
= gsi_start_bb (body_bb
);
11396 expr
= build2 (plus_code
, iter_type
, b
,
11397 fold_convert (plus_type
, offset
));
11398 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11399 true, GSI_SAME_STMT
);
11400 ass
= gimple_build_assign (v
, expr
);
11401 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11402 if (fd
->collapse
> 1)
11403 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11406 /* Loop increment goes into cont_bb. If this is not a loop, we
11407 will have spawned threads as if it was, and each one will
11408 execute one iteration. The specification is not explicit about
11409 whether such constructs are ill-formed or not, and they can
11410 occur, especially when noreturn routines are involved. */
11413 gsi
= gsi_last_bb (cont_bb
);
11414 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11415 loc
= gimple_location (cont_stmt
);
11417 /* Increment offset. */
11418 if (gimple_in_ssa_p (cfun
))
11419 expr
= build2 (plus_code
, iter_type
, offset
,
11420 fold_convert (plus_type
, step
));
11422 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11423 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11424 true, GSI_SAME_STMT
);
11425 ass
= gimple_build_assign (offset_incr
, expr
);
11426 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11427 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11428 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11430 /* Remove the GIMPLE_OMP_CONTINUE. */
11431 gsi_remove (&gsi
, true);
11433 /* Fixup edges from cont_bb */
11434 be
= BRANCH_EDGE (cont_bb
);
11435 fte
= FALLTHRU_EDGE (cont_bb
);
11436 be
->flags
|= EDGE_TRUE_VALUE
;
11437 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11441 /* Split the beginning of exit_bb to make bottom_bb. We
11442 need to insert a nop at the start, because splitting is
11443 after a stmt, not before. */
11444 gsi
= gsi_start_bb (exit_bb
);
11445 stmt
= gimple_build_nop ();
11446 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11447 split
= split_block (exit_bb
, stmt
);
11448 bottom_bb
= split
->src
;
11449 exit_bb
= split
->dest
;
11450 gsi
= gsi_last_bb (bottom_bb
);
11452 /* Chunk increment and test goes into bottom_bb. */
11453 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11454 build_int_cst (diff_type
, 1));
11455 ass
= gimple_build_assign (chunk_no
, expr
);
11456 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11458 /* Chunk test at end of bottom_bb. */
11459 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11460 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11461 GSI_CONTINUE_LINKING
);
11463 /* Fixup edges from bottom_bb. */
11464 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11465 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11469 gsi
= gsi_last_bb (exit_bb
);
11470 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11471 loc
= gimple_location (gsi_stmt (gsi
));
11473 if (!gimple_in_ssa_p (cfun
))
11475 /* Insert the final value of V, in case it is live. This is the
11476 value for the only thread that survives past the join. */
11477 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11478 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11479 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11480 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11481 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11482 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11483 true, GSI_SAME_STMT
);
11484 ass
= gimple_build_assign (v
, expr
);
11485 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11488 /* Remove the OMP_RETURN. */
11489 gsi_remove (&gsi
, true);
11493 /* We now have one or two nested loops. Update the loop
11495 struct loop
*parent
= entry_bb
->loop_father
;
11496 struct loop
*body
= body_bb
->loop_father
;
11500 struct loop
*chunk_loop
= alloc_loop ();
11501 chunk_loop
->header
= head_bb
;
11502 chunk_loop
->latch
= bottom_bb
;
11503 add_loop (chunk_loop
, parent
);
11504 parent
= chunk_loop
;
11506 else if (parent
!= body
)
11508 gcc_assert (body
->header
== body_bb
);
11509 gcc_assert (body
->latch
== cont_bb
11510 || single_pred (body
->latch
) == cont_bb
);
11516 struct loop
*body_loop
= alloc_loop ();
11517 body_loop
->header
= body_bb
;
11518 body_loop
->latch
= cont_bb
;
11519 add_loop (body_loop
, parent
);
11524 /* Expand the OMP loop defined by REGION. */
11527 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11529 struct omp_for_data fd
;
11530 struct omp_for_data_loop
*loops
;
11533 = (struct omp_for_data_loop
*)
11534 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11535 * sizeof (struct omp_for_data_loop
));
11536 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11538 region
->sched_kind
= fd
.sched_kind
;
11539 region
->sched_modifiers
= fd
.sched_modifiers
;
11541 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11542 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11543 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11546 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11547 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11548 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11551 /* If there isn't a continue then this is a degerate case where
11552 the introduction of abnormal edges during lowering will prevent
11553 original loops from being detected. Fix that up. */
11554 loops_state_set (LOOPS_NEED_FIXUP
);
11556 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11557 expand_omp_simd (region
, &fd
);
11558 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11559 expand_cilk_for (region
, &fd
);
11560 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11562 gcc_assert (!inner_stmt
);
11563 expand_oacc_for (region
, &fd
);
11565 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11567 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11568 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11570 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11572 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11573 && !fd
.have_ordered
)
11575 if (fd
.chunk_size
== NULL
)
11576 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11578 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11582 int fn_index
, start_ix
, next_ix
;
11584 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11585 == GF_OMP_FOR_KIND_FOR
);
11586 if (fd
.chunk_size
== NULL
11587 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11588 fd
.chunk_size
= integer_zero_node
;
11589 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11590 switch (fd
.sched_kind
)
11592 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11595 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11596 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11597 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11599 && !fd
.have_ordered
)
11601 fn_index
= 3 + fd
.sched_kind
;
11606 fn_index
= fd
.sched_kind
;
11610 fn_index
+= fd
.have_ordered
* 6;
11612 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11614 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11615 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11616 if (fd
.iter_type
== long_long_unsigned_type_node
)
11618 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11619 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11620 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11621 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11623 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11624 (enum built_in_function
) next_ix
, inner_stmt
);
11627 if (gimple_in_ssa_p (cfun
))
11628 update_ssa (TODO_update_ssa_only_virtuals
);
11632 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11634 v = GOMP_sections_start (n);
11651 v = GOMP_sections_next ();
11656 If this is a combined parallel sections, replace the call to
11657 GOMP_sections_start with call to GOMP_sections_next. */
11660 expand_omp_sections (struct omp_region
*region
)
11662 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11664 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11665 gimple_stmt_iterator si
, switch_si
;
11666 gomp_sections
*sections_stmt
;
11668 gomp_continue
*cont
;
11671 struct omp_region
*inner
;
11673 bool exit_reachable
= region
->cont
!= NULL
;
11675 gcc_assert (region
->exit
!= NULL
);
11676 entry_bb
= region
->entry
;
11677 l0_bb
= single_succ (entry_bb
);
11678 l1_bb
= region
->cont
;
11679 l2_bb
= region
->exit
;
11680 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11681 l2
= gimple_block_label (l2_bb
);
11684 /* This can happen if there are reductions. */
11685 len
= EDGE_COUNT (l0_bb
->succs
);
11686 gcc_assert (len
> 0);
11687 e
= EDGE_SUCC (l0_bb
, len
- 1);
11688 si
= gsi_last_bb (e
->dest
);
11691 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11692 l2
= gimple_block_label (e
->dest
);
11694 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11696 si
= gsi_last_bb (e
->dest
);
11698 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11700 l2
= gimple_block_label (e
->dest
);
11705 if (exit_reachable
)
11706 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11708 default_bb
= create_empty_bb (l0_bb
);
11710 /* We will build a switch() with enough cases for all the
11711 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11712 and a default case to abort if something goes wrong. */
11713 len
= EDGE_COUNT (l0_bb
->succs
);
11715 /* Use vec::quick_push on label_vec throughout, since we know the size
11717 auto_vec
<tree
> label_vec (len
);
11719 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11720 GIMPLE_OMP_SECTIONS statement. */
11721 si
= gsi_last_bb (entry_bb
);
11722 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11723 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11724 vin
= gimple_omp_sections_control (sections_stmt
);
11725 if (!is_combined_parallel (region
))
11727 /* If we are not inside a combined parallel+sections region,
11728 call GOMP_sections_start. */
11729 t
= build_int_cst (unsigned_type_node
, len
- 1);
11730 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11731 stmt
= gimple_build_call (u
, 1, t
);
11735 /* Otherwise, call GOMP_sections_next. */
11736 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11737 stmt
= gimple_build_call (u
, 0);
11739 gimple_call_set_lhs (stmt
, vin
);
11740 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11741 gsi_remove (&si
, true);
11743 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11745 switch_si
= gsi_last_bb (l0_bb
);
11746 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11747 if (exit_reachable
)
11749 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11750 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11751 vmain
= gimple_omp_continue_control_use (cont
);
11752 vnext
= gimple_omp_continue_control_def (cont
);
11760 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11761 label_vec
.quick_push (t
);
11764 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11765 for (inner
= region
->inner
, casei
= 1;
11767 inner
= inner
->next
, i
++, casei
++)
11769 basic_block s_entry_bb
, s_exit_bb
;
11771 /* Skip optional reduction region. */
11772 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11779 s_entry_bb
= inner
->entry
;
11780 s_exit_bb
= inner
->exit
;
11782 t
= gimple_block_label (s_entry_bb
);
11783 u
= build_int_cst (unsigned_type_node
, casei
);
11784 u
= build_case_label (u
, NULL
, t
);
11785 label_vec
.quick_push (u
);
11787 si
= gsi_last_bb (s_entry_bb
);
11788 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11789 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11790 gsi_remove (&si
, true);
11791 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11793 if (s_exit_bb
== NULL
)
11796 si
= gsi_last_bb (s_exit_bb
);
11797 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11798 gsi_remove (&si
, true);
11800 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11803 /* Error handling code goes in DEFAULT_BB. */
11804 t
= gimple_block_label (default_bb
);
11805 u
= build_case_label (NULL
, NULL
, t
);
11806 make_edge (l0_bb
, default_bb
, 0);
11807 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11809 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11810 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11811 gsi_remove (&switch_si
, true);
11813 si
= gsi_start_bb (default_bb
);
11814 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11815 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11817 if (exit_reachable
)
11821 /* Code to get the next section goes in L1_BB. */
11822 si
= gsi_last_bb (l1_bb
);
11823 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11825 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11826 stmt
= gimple_build_call (bfn_decl
, 0);
11827 gimple_call_set_lhs (stmt
, vnext
);
11828 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11829 gsi_remove (&si
, true);
11831 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11834 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11835 si
= gsi_last_bb (l2_bb
);
11836 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11837 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11838 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11839 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11841 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11842 stmt
= gimple_build_call (t
, 0);
11843 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11844 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11845 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11846 gsi_remove (&si
, true);
11848 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11852 /* Expand code for an OpenMP single directive. We've already expanded
11853 much of the code, here we simply place the GOMP_barrier call. */
11856 expand_omp_single (struct omp_region
*region
)
11858 basic_block entry_bb
, exit_bb
;
11859 gimple_stmt_iterator si
;
11861 entry_bb
= region
->entry
;
11862 exit_bb
= region
->exit
;
11864 si
= gsi_last_bb (entry_bb
);
11865 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11866 gsi_remove (&si
, true);
11867 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11869 si
= gsi_last_bb (exit_bb
);
11870 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11872 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11873 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11875 gsi_remove (&si
, true);
11876 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11880 /* Generic expansion for OpenMP synchronization directives: master,
11881 ordered and critical. All we need to do here is remove the entry
11882 and exit markers for REGION. */
11885 expand_omp_synch (struct omp_region
*region
)
11887 basic_block entry_bb
, exit_bb
;
11888 gimple_stmt_iterator si
;
11890 entry_bb
= region
->entry
;
11891 exit_bb
= region
->exit
;
11893 si
= gsi_last_bb (entry_bb
);
11894 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11895 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11896 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11897 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11898 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11899 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11900 gsi_remove (&si
, true);
11901 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11905 si
= gsi_last_bb (exit_bb
);
11906 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11907 gsi_remove (&si
, true);
11908 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11912 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11913 operation as a normal volatile load. */
11916 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11917 tree loaded_val
, int index
)
11919 enum built_in_function tmpbase
;
11920 gimple_stmt_iterator gsi
;
11921 basic_block store_bb
;
11924 tree decl
, call
, type
, itype
;
11926 gsi
= gsi_last_bb (load_bb
);
11927 stmt
= gsi_stmt (gsi
);
11928 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11929 loc
= gimple_location (stmt
);
11931 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11932 is smaller than word size, then expand_atomic_load assumes that the load
11933 is atomic. We could avoid the builtin entirely in this case. */
11935 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11936 decl
= builtin_decl_explicit (tmpbase
);
11937 if (decl
== NULL_TREE
)
11940 type
= TREE_TYPE (loaded_val
);
11941 itype
= TREE_TYPE (TREE_TYPE (decl
));
11943 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11944 build_int_cst (NULL
,
11945 gimple_omp_atomic_seq_cst_p (stmt
)
11947 : MEMMODEL_RELAXED
));
11948 if (!useless_type_conversion_p (type
, itype
))
11949 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11950 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11952 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11953 gsi_remove (&gsi
, true);
11955 store_bb
= single_succ (load_bb
);
11956 gsi
= gsi_last_bb (store_bb
);
11957 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11958 gsi_remove (&gsi
, true);
11960 if (gimple_in_ssa_p (cfun
))
11961 update_ssa (TODO_update_ssa_no_phi
);
11966 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11967 operation as a normal volatile store. */
11970 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11971 tree loaded_val
, tree stored_val
, int index
)
11973 enum built_in_function tmpbase
;
11974 gimple_stmt_iterator gsi
;
11975 basic_block store_bb
= single_succ (load_bb
);
11978 tree decl
, call
, type
, itype
;
11979 machine_mode imode
;
11982 gsi
= gsi_last_bb (load_bb
);
11983 stmt
= gsi_stmt (gsi
);
11984 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11986 /* If the load value is needed, then this isn't a store but an exchange. */
11987 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11989 gsi
= gsi_last_bb (store_bb
);
11990 stmt
= gsi_stmt (gsi
);
11991 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11992 loc
= gimple_location (stmt
);
11994 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11995 is smaller than word size, then expand_atomic_store assumes that the store
11996 is atomic. We could avoid the builtin entirely in this case. */
11998 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11999 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
12000 decl
= builtin_decl_explicit (tmpbase
);
12001 if (decl
== NULL_TREE
)
12004 type
= TREE_TYPE (stored_val
);
12006 /* Dig out the type of the function's second argument. */
12007 itype
= TREE_TYPE (decl
);
12008 itype
= TYPE_ARG_TYPES (itype
);
12009 itype
= TREE_CHAIN (itype
);
12010 itype
= TREE_VALUE (itype
);
12011 imode
= TYPE_MODE (itype
);
12013 if (exchange
&& !can_atomic_exchange_p (imode
, true))
12016 if (!useless_type_conversion_p (itype
, type
))
12017 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
12018 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
12019 build_int_cst (NULL
,
12020 gimple_omp_atomic_seq_cst_p (stmt
)
12022 : MEMMODEL_RELAXED
));
12025 if (!useless_type_conversion_p (type
, itype
))
12026 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
12027 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
12030 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12031 gsi_remove (&gsi
, true);
12033 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
12034 gsi
= gsi_last_bb (load_bb
);
12035 gsi_remove (&gsi
, true);
12037 if (gimple_in_ssa_p (cfun
))
12038 update_ssa (TODO_update_ssa_no_phi
);
12043 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
12044 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
12045 size of the data type, and thus usable to find the index of the builtin
12046 decl. Returns false if the expression is not of the proper form. */
12049 expand_omp_atomic_fetch_op (basic_block load_bb
,
12050 tree addr
, tree loaded_val
,
12051 tree stored_val
, int index
)
12053 enum built_in_function oldbase
, newbase
, tmpbase
;
12054 tree decl
, itype
, call
;
12056 basic_block store_bb
= single_succ (load_bb
);
12057 gimple_stmt_iterator gsi
;
12060 enum tree_code code
;
12061 bool need_old
, need_new
;
12062 machine_mode imode
;
12065 /* We expect to find the following sequences:
12068 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12071 val = tmp OP something; (or: something OP tmp)
12072 GIMPLE_OMP_STORE (val)
12074 ???FIXME: Allow a more flexible sequence.
12075 Perhaps use data flow to pick the statements.
12079 gsi
= gsi_after_labels (store_bb
);
12080 stmt
= gsi_stmt (gsi
);
12081 loc
= gimple_location (stmt
);
12082 if (!is_gimple_assign (stmt
))
12085 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
12087 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
12088 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
12089 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
12090 gcc_checking_assert (!need_old
|| !need_new
);
12092 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
12095 /* Check for one of the supported fetch-op operations. */
12096 code
= gimple_assign_rhs_code (stmt
);
12100 case POINTER_PLUS_EXPR
:
12101 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
12102 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
12105 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
12106 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
12109 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
12110 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
12113 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
12114 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
12117 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
12118 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
12124 /* Make sure the expression is of the proper form. */
12125 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
12126 rhs
= gimple_assign_rhs2 (stmt
);
12127 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
12128 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
12129 rhs
= gimple_assign_rhs1 (stmt
);
12133 tmpbase
= ((enum built_in_function
)
12134 ((need_new
? newbase
: oldbase
) + index
+ 1));
12135 decl
= builtin_decl_explicit (tmpbase
);
12136 if (decl
== NULL_TREE
)
12138 itype
= TREE_TYPE (TREE_TYPE (decl
));
12139 imode
= TYPE_MODE (itype
);
12141 /* We could test all of the various optabs involved, but the fact of the
12142 matter is that (with the exception of i486 vs i586 and xadd) all targets
12143 that support any atomic operaton optab also implements compare-and-swap.
12144 Let optabs.c take care of expanding any compare-and-swap loop. */
12145 if (!can_compare_and_swap_p (imode
, true))
12148 gsi
= gsi_last_bb (load_bb
);
12149 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12151 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12152 It only requires that the operation happen atomically. Thus we can
12153 use the RELAXED memory model. */
12154 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
12155 fold_convert_loc (loc
, itype
, rhs
),
12156 build_int_cst (NULL
,
12157 seq_cst
? MEMMODEL_SEQ_CST
12158 : MEMMODEL_RELAXED
));
12160 if (need_old
|| need_new
)
12162 lhs
= need_old
? loaded_val
: stored_val
;
12163 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
12164 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
12167 call
= fold_convert_loc (loc
, void_type_node
, call
);
12168 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12169 gsi_remove (&gsi
, true);
12171 gsi
= gsi_last_bb (store_bb
);
12172 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
12173 gsi_remove (&gsi
, true);
12174 gsi
= gsi_last_bb (store_bb
);
12175 stmt
= gsi_stmt (gsi
);
12176 gsi_remove (&gsi
, true);
12178 if (gimple_in_ssa_p (cfun
))
12180 release_defs (stmt
);
12181 update_ssa (TODO_update_ssa_no_phi
);
12187 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12191 newval = rhs; // with oldval replacing *addr in rhs
12192 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12193 if (oldval != newval)
12196 INDEX is log2 of the size of the data type, and thus usable to find the
12197 index of the builtin decl. */
12200 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
12201 tree addr
, tree loaded_val
, tree stored_val
,
12204 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
12205 tree type
, itype
, cmpxchg
, iaddr
;
12206 gimple_stmt_iterator si
;
12207 basic_block loop_header
= single_succ (load_bb
);
12208 gimple
*phi
, *stmt
;
12210 enum built_in_function fncode
;
12212 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12213 order to use the RELAXED memory model effectively. */
12214 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12216 cmpxchg
= builtin_decl_explicit (fncode
);
12217 if (cmpxchg
== NULL_TREE
)
12219 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12220 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
12222 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
12225 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12226 si
= gsi_last_bb (load_bb
);
12227 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12229 /* For floating-point values, we'll need to view-convert them to integers
12230 so that we can perform the atomic compare and swap. Simplify the
12231 following code by always setting up the "i"ntegral variables. */
12232 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
12236 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
12239 = force_gimple_operand_gsi (&si
,
12240 fold_convert (TREE_TYPE (iaddr
), addr
),
12241 false, NULL_TREE
, true, GSI_SAME_STMT
);
12242 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
12243 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12244 loadedi
= create_tmp_var (itype
);
12245 if (gimple_in_ssa_p (cfun
))
12246 loadedi
= make_ssa_name (loadedi
);
12251 loadedi
= loaded_val
;
12254 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
12255 tree loaddecl
= builtin_decl_explicit (fncode
);
12258 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12259 build_call_expr (loaddecl
, 2, iaddr
,
12260 build_int_cst (NULL_TREE
,
12261 MEMMODEL_RELAXED
)));
12263 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12264 build_int_cst (TREE_TYPE (iaddr
), 0));
12267 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12270 /* Move the value to the LOADEDI temporary. */
12271 if (gimple_in_ssa_p (cfun
))
12273 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12274 phi
= create_phi_node (loadedi
, loop_header
);
12275 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12279 gsi_insert_before (&si
,
12280 gimple_build_assign (loadedi
, initial
),
12282 if (loadedi
!= loaded_val
)
12284 gimple_stmt_iterator gsi2
;
12287 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12288 gsi2
= gsi_start_bb (loop_header
);
12289 if (gimple_in_ssa_p (cfun
))
12292 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12293 true, GSI_SAME_STMT
);
12294 stmt
= gimple_build_assign (loaded_val
, x
);
12295 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12299 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12300 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12301 true, GSI_SAME_STMT
);
12304 gsi_remove (&si
, true);
12306 si
= gsi_last_bb (store_bb
);
12307 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12310 storedi
= stored_val
;
12313 force_gimple_operand_gsi (&si
,
12314 build1 (VIEW_CONVERT_EXPR
, itype
,
12315 stored_val
), true, NULL_TREE
, true,
12318 /* Build the compare&swap statement. */
12319 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12320 new_storedi
= force_gimple_operand_gsi (&si
,
12321 fold_convert (TREE_TYPE (loadedi
),
12324 true, GSI_SAME_STMT
);
12326 if (gimple_in_ssa_p (cfun
))
12327 old_vali
= loadedi
;
12330 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12331 stmt
= gimple_build_assign (old_vali
, loadedi
);
12332 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12334 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12335 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12338 /* Note that we always perform the comparison as an integer, even for
12339 floating point. This allows the atomic operation to properly
12340 succeed even with NaNs and -0.0. */
12341 stmt
= gimple_build_cond_empty
12342 (build2 (NE_EXPR
, boolean_type_node
,
12343 new_storedi
, old_vali
));
12344 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12347 e
= single_succ_edge (store_bb
);
12348 e
->flags
&= ~EDGE_FALLTHRU
;
12349 e
->flags
|= EDGE_FALSE_VALUE
;
12351 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12353 /* Copy the new value to loadedi (we already did that before the condition
12354 if we are not in SSA). */
12355 if (gimple_in_ssa_p (cfun
))
12357 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12358 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12361 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12362 gsi_remove (&si
, true);
12364 struct loop
*loop
= alloc_loop ();
12365 loop
->header
= loop_header
;
12366 loop
->latch
= store_bb
;
12367 add_loop (loop
, loop_header
->loop_father
);
12369 if (gimple_in_ssa_p (cfun
))
12370 update_ssa (TODO_update_ssa_no_phi
);
12375 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12377 GOMP_atomic_start ();
12379 GOMP_atomic_end ();
12381 The result is not globally atomic, but works so long as all parallel
12382 references are within #pragma omp atomic directives. According to
12383 responses received from omp@openmp.org, appears to be within spec.
12384 Which makes sense, since that's how several other compilers handle
12385 this situation as well.
12386 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12387 expanding. STORED_VAL is the operand of the matching
12388 GIMPLE_OMP_ATOMIC_STORE.
12391 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12392 loaded_val = *addr;
12395 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12396 *addr = stored_val;
12400 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12401 tree addr
, tree loaded_val
, tree stored_val
)
12403 gimple_stmt_iterator si
;
12407 si
= gsi_last_bb (load_bb
);
12408 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12410 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12411 t
= build_call_expr (t
, 0);
12412 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12414 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12415 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12416 gsi_remove (&si
, true);
12418 si
= gsi_last_bb (store_bb
);
12419 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12421 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12423 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12425 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12426 t
= build_call_expr (t
, 0);
12427 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12428 gsi_remove (&si
, true);
12430 if (gimple_in_ssa_p (cfun
))
12431 update_ssa (TODO_update_ssa_no_phi
);
12435 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12436 using expand_omp_atomic_fetch_op. If it failed, we try to
12437 call expand_omp_atomic_pipeline, and if it fails too, the
12438 ultimate fallback is wrapping the operation in a mutex
12439 (expand_omp_atomic_mutex). REGION is the atomic region built
12440 by build_omp_regions_1(). */
12443 expand_omp_atomic (struct omp_region
*region
)
12445 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12446 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12447 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12448 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12449 tree addr
= gimple_omp_atomic_load_rhs (load
);
12450 tree stored_val
= gimple_omp_atomic_store_val (store
);
12451 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12452 HOST_WIDE_INT index
;
12454 /* Make sure the type is one of the supported sizes. */
12455 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12456 index
= exact_log2 (index
);
12457 if (index
>= 0 && index
<= 4)
12459 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12461 /* __sync builtins require strict data alignment. */
12462 if (exact_log2 (align
) >= index
)
12465 if (loaded_val
== stored_val
12466 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12467 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12468 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12469 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12472 /* Atomic store. */
12473 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12474 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12475 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12476 && store_bb
== single_succ (load_bb
)
12477 && first_stmt (store_bb
) == store
12478 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12479 stored_val
, index
))
12482 /* When possible, use specialized atomic update functions. */
12483 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12484 && store_bb
== single_succ (load_bb
)
12485 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12486 loaded_val
, stored_val
, index
))
12489 /* If we don't have specialized __sync builtins, try and implement
12490 as a compare and swap loop. */
12491 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12492 loaded_val
, stored_val
, index
))
12497 /* The ultimate fallback is wrapping the operation in a mutex. */
12498 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12502 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12503 macro on gomp-constants.h. We do not check for overflow. */
12506 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12510 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12513 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12514 device
, build_int_cst (unsigned_type_node
,
12515 GOMP_LAUNCH_DEVICE_SHIFT
));
12516 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12521 /* Look for compute grid dimension clauses and convert to an attribute
12522 attached to FN. This permits the target-side code to (a) massage
12523 the dimensions, (b) emit that data and (c) optimize. Non-constant
12524 dimensions are pushed onto ARGS.
12526 The attribute value is a TREE_LIST. A set of dimensions is
12527 represented as a list of INTEGER_CST. Those that are runtime
12528 exprs are represented as an INTEGER_CST of zero.
12530 TOOO. Normally the attribute will just contain a single such list. If
12531 however it contains a list of lists, this will represent the use of
12532 device_type. Each member of the outer list is an assoc list of
12533 dimensions, keyed by the device type. The first entry will be the
12534 default. Well, that's the plan. */
12536 #define OACC_FN_ATTRIB "oacc function"
12538 /* Replace any existing oacc fn attribute with updated dimensions. */
12541 replace_oacc_fn_attrib (tree fn
, tree dims
)
12543 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12544 tree attribs
= DECL_ATTRIBUTES (fn
);
12546 /* If we happen to be present as the first attrib, drop it. */
12547 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12548 attribs
= TREE_CHAIN (attribs
);
12549 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12552 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12553 function attribute. Push any that are non-constant onto the ARGS
12554 list, along with an appropriate GOMP_LAUNCH_DIM tag. IS_KERNEL is
12555 true, if these are for a kernels region offload function. */
12558 set_oacc_fn_attrib (tree fn
, tree clauses
, bool is_kernel
, vec
<tree
> *args
)
12560 /* Must match GOMP_DIM ordering. */
12561 static const omp_clause_code ids
[]
12562 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12563 OMP_CLAUSE_VECTOR_LENGTH
};
12565 tree dims
[GOMP_DIM_MAX
];
12566 tree attr
= NULL_TREE
;
12567 unsigned non_const
= 0;
12569 for (ix
= GOMP_DIM_MAX
; ix
--;)
12571 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12572 tree dim
= NULL_TREE
;
12575 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12577 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12579 dim
= integer_zero_node
;
12580 non_const
|= GOMP_DIM_MASK (ix
);
12582 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12583 /* Note kernelness with TREE_PUBLIC. */
12585 TREE_PUBLIC (attr
) = 1;
12588 replace_oacc_fn_attrib (fn
, attr
);
12592 /* Push a dynamic argument set. */
12593 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12594 NULL_TREE
, non_const
));
12595 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12596 if (non_const
& GOMP_DIM_MASK (ix
))
12597 args
->safe_push (dims
[ix
]);
12601 /* Process the routine's dimension clauess to generate an attribute
12602 value. Issue diagnostics as appropriate. We default to SEQ
12603 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12604 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12605 can have a loop partitioned on it. non-zero indicates
12606 yes, zero indicates no. By construction once a non-zero has been
12607 reached, further inner dimensions must also be non-zero. We set
12608 TREE_VALUE to zero for the dimensions that may be partitioned and
12609 1 for the other ones -- if a loop is (erroneously) spawned at
12610 an outer level, we don't want to try and partition it. */
12613 build_oacc_routine_dims (tree clauses
)
12615 /* Must match GOMP_DIM ordering. */
12616 static const omp_clause_code ids
[] =
12617 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12621 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12622 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12623 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12626 error_at (OMP_CLAUSE_LOCATION (clauses
),
12627 "multiple loop axes specified for routine");
12632 /* Default to SEQ. */
12634 level
= GOMP_DIM_MAX
;
12636 tree dims
= NULL_TREE
;
12638 for (ix
= GOMP_DIM_MAX
; ix
--;)
12639 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12640 build_int_cst (integer_type_node
, ix
< level
), dims
);
12645 /* Retrieve the oacc function attrib and return it. Non-oacc
12646 functions will return NULL. */
12649 get_oacc_fn_attrib (tree fn
)
12651 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12654 /* Return true if this oacc fn attrib is for a kernels offload
12655 region. We use the TREE_PUBLIC flag of each dimension -- only
12656 need to check the first one. */
12659 oacc_fn_attrib_kernels_p (tree attr
)
12661 return TREE_PUBLIC (TREE_VALUE (attr
));
12664 /* Return level at which oacc routine may spawn a partitioned loop, or
12665 -1 if it is not a routine (i.e. is an offload fn). */
12668 oacc_fn_attrib_level (tree attr
)
12670 tree pos
= TREE_VALUE (attr
);
12672 if (!TREE_PURPOSE (pos
))
12676 for (ix
= 0; ix
!= GOMP_DIM_MAX
;
12677 ix
++, pos
= TREE_CHAIN (pos
))
12678 if (!integer_zerop (TREE_PURPOSE (pos
)))
12684 /* Extract an oacc execution dimension from FN. FN must be an
12685 offloaded function or routine that has already had its execution
12686 dimensions lowered to the target-specific values. */
12689 get_oacc_fn_dim_size (tree fn
, int axis
)
12691 tree attrs
= get_oacc_fn_attrib (fn
);
12693 gcc_assert (axis
< GOMP_DIM_MAX
);
12695 tree dims
= TREE_VALUE (attrs
);
12697 dims
= TREE_CHAIN (dims
);
12699 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12704 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12705 IFN_GOACC_DIM_SIZE call. */
12708 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12710 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12711 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12712 tree arg
= gimple_call_arg (stmt
, 0);
12713 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12715 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12719 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12723 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
12724 basic_block region_exit
)
12726 struct loop
*outer
= region_entry
->loop_father
;
12727 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
12729 /* Don't parallelize the kernels region if it contains more than one outer
12731 unsigned int nr_outer_loops
= 0;
12732 struct loop
*single_outer
= NULL
;
12733 for (struct loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
12735 gcc_assert (loop_outer (loop
) == outer
);
12737 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
12740 if (region_exit
!= NULL
12741 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
12745 single_outer
= loop
;
12747 if (nr_outer_loops
!= 1)
12750 for (struct loop
*loop
= single_outer
->inner
; loop
!= NULL
; loop
= loop
->inner
)
12754 /* Mark the loops in the region. */
12755 for (struct loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
12756 loop
->in_oacc_kernels_region
= true;
12759 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12761 struct GTY(()) grid_launch_attributes_trees
12763 tree kernel_dim_array_type
;
12764 tree kernel_lattrs_dimnum_decl
;
12765 tree kernel_lattrs_grid_decl
;
12766 tree kernel_lattrs_group_decl
;
12767 tree kernel_launch_attributes_type
;
12770 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
12772 /* Create types used to pass kernel launch attributes to target. */
12775 grid_create_kernel_launch_attr_types (void)
12777 if (grid_attr_trees
)
12779 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
12781 tree dim_arr_index_type
12782 = build_index_type (build_int_cst (integer_type_node
, 2));
12783 grid_attr_trees
->kernel_dim_array_type
12784 = build_array_type (uint32_type_node
, dim_arr_index_type
);
12786 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
12787 grid_attr_trees
->kernel_lattrs_dimnum_decl
12788 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
12790 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
12792 grid_attr_trees
->kernel_lattrs_grid_decl
12793 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
12794 grid_attr_trees
->kernel_dim_array_type
);
12795 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
12796 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
12797 grid_attr_trees
->kernel_lattrs_group_decl
12798 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
12799 grid_attr_trees
->kernel_dim_array_type
);
12800 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
12801 = grid_attr_trees
->kernel_lattrs_grid_decl
;
12802 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
12803 "__gomp_kernel_launch_attributes",
12804 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
12807 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12808 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12809 of type uint32_type_node. */
12812 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
12813 tree fld_decl
, int index
, tree value
)
12815 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
12816 build3 (COMPONENT_REF
,
12817 grid_attr_trees
->kernel_dim_array_type
,
12818 range_var
, fld_decl
, NULL_TREE
),
12819 build_int_cst (integer_type_node
, index
),
12820 NULL_TREE
, NULL_TREE
);
12821 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
12824 /* Return a tree representation of a pointer to a structure with grid and
12825 work-group size information. Statements filling that information will be
12826 inserted before GSI, TGT_STMT is the target statement which has the
12827 necessary information in it. */
12830 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
12831 gomp_target
*tgt_stmt
)
12833 grid_create_kernel_launch_attr_types ();
12834 tree u32_one
= build_one_cst (uint32_type_node
);
12835 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
12836 "__kernel_launch_attrs");
12838 unsigned max_dim
= 0;
12839 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
12841 clause
= OMP_CLAUSE_CHAIN (clause
))
12843 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
12846 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
12847 max_dim
= MAX (dim
, max_dim
);
12849 grid_insert_store_range_dim (gsi
, lattrs
,
12850 grid_attr_trees
->kernel_lattrs_grid_decl
,
12851 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
12852 grid_insert_store_range_dim (gsi
, lattrs
,
12853 grid_attr_trees
->kernel_lattrs_group_decl
,
12854 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
12857 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
12858 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
12859 /* At this moment we cannot gridify a loop with a collapse clause. */
12860 /* TODO: Adjust when we support bigger collapse. */
12861 gcc_assert (max_dim
== 0);
12862 gsi_insert_before (gsi
, gimple_build_assign (dimref
, u32_one
), GSI_SAME_STMT
);
12863 TREE_ADDRESSABLE (lattrs
) = 1;
12864 return build_fold_addr_expr (lattrs
);
12867 /* Build target argument identifier from the DEVICE identifier, value
12868 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12871 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
12873 tree t
= build_int_cst (integer_type_node
, device
);
12874 if (subseqent_param
)
12875 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12876 build_int_cst (integer_type_node
,
12877 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
12878 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12879 build_int_cst (integer_type_node
, id
));
12883 /* Like above but return it in type that can be directly stored as an element
12884 of the argument array. */
12887 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
12889 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
12890 return fold_convert (ptr_type_node
, t
);
12893 /* Return a target argument consisting of DEVICE identifier, value identifier
12894 ID, and the actual VALUE. */
12897 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
12900 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
12901 fold_convert (integer_type_node
, value
),
12902 build_int_cst (unsigned_type_node
,
12903 GOMP_TARGET_ARG_VALUE_SHIFT
));
12904 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12905 get_target_argument_identifier_1 (device
, false, id
));
12906 t
= fold_convert (ptr_type_node
, t
);
12907 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
12910 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12911 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12912 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12916 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
12917 int id
, tree value
, vec
<tree
> *args
)
12919 if (tree_fits_shwi_p (value
)
12920 && tree_to_shwi (value
) > -(1 << 15)
12921 && tree_to_shwi (value
) < (1 << 15))
12922 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
12925 args
->quick_push (get_target_argument_identifier (device
, true, id
));
12926 value
= fold_convert (ptr_type_node
, value
);
12927 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
12929 args
->quick_push (value
);
12933 /* Create an array of arguments that is then passed to GOMP_target. */
12936 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
12938 auto_vec
<tree
, 6> args
;
12939 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
12940 tree t
, c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12942 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
12944 t
= integer_minus_one_node
;
12945 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12946 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
12948 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12950 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
12952 t
= integer_minus_one_node
;
12953 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12954 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
12957 /* Add HSA-specific grid sizes, if available. */
12958 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
12959 OMP_CLAUSE__GRIDDIM_
))
12961 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true,
12962 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
);
12963 args
.quick_push (t
);
12964 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
12967 /* Produce more, perhaps device specific, arguments here. */
12969 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
12970 args
.length () + 1),
12971 ".omp_target_args");
12972 for (unsigned i
= 0; i
< args
.length (); i
++)
12974 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12975 build_int_cst (integer_type_node
, i
),
12976 NULL_TREE
, NULL_TREE
);
12977 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
12980 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12981 build_int_cst (integer_type_node
, args
.length ()),
12982 NULL_TREE
, NULL_TREE
);
12983 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
12985 TREE_ADDRESSABLE (argarray
) = 1;
12986 return build_fold_addr_expr (argarray
);
12989 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12992 expand_omp_target (struct omp_region
*region
)
12994 basic_block entry_bb
, exit_bb
, new_bb
;
12995 struct function
*child_cfun
;
12996 tree child_fn
, block
, t
;
12997 gimple_stmt_iterator gsi
;
12998 gomp_target
*entry_stmt
;
13001 bool offloaded
, data_region
;
13003 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
13004 new_bb
= region
->entry
;
13006 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
13007 switch (gimple_omp_target_kind (entry_stmt
))
13009 case GF_OMP_TARGET_KIND_REGION
:
13010 case GF_OMP_TARGET_KIND_UPDATE
:
13011 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13012 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13013 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13014 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13015 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13016 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13017 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13018 data_region
= false;
13020 case GF_OMP_TARGET_KIND_DATA
:
13021 case GF_OMP_TARGET_KIND_OACC_DATA
:
13022 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13023 data_region
= true;
13026 gcc_unreachable ();
13029 child_fn
= NULL_TREE
;
13033 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
13034 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
13037 /* Supported by expand_omp_taskreg, but not here. */
13038 if (child_cfun
!= NULL
)
13039 gcc_checking_assert (!child_cfun
->cfg
);
13040 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
13042 entry_bb
= region
->entry
;
13043 exit_bb
= region
->exit
;
13045 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
13046 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
13050 unsigned srcidx
, dstidx
, num
;
13052 /* If the offloading region needs data sent from the parent
13053 function, then the very first statement (except possible
13054 tree profile counter updates) of the offloading body
13055 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
13056 &.OMP_DATA_O is passed as an argument to the child function,
13057 we need to replace it with the argument as seen by the child
13060 In most cases, this will end up being the identity assignment
13061 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
13062 a function call that has been inlined, the original PARM_DECL
13063 .OMP_DATA_I may have been converted into a different local
13064 variable. In which case, we need to keep the assignment. */
13065 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
13068 basic_block entry_succ_bb
= single_succ (entry_bb
);
13069 gimple_stmt_iterator gsi
;
13071 gimple
*tgtcopy_stmt
= NULL
;
13072 tree sender
= TREE_VEC_ELT (data_arg
, 0);
13074 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
13076 gcc_assert (!gsi_end_p (gsi
));
13077 stmt
= gsi_stmt (gsi
);
13078 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
13081 if (gimple_num_ops (stmt
) == 2)
13083 tree arg
= gimple_assign_rhs1 (stmt
);
13085 /* We're ignoring the subcode because we're
13086 effectively doing a STRIP_NOPS. */
13088 if (TREE_CODE (arg
) == ADDR_EXPR
13089 && TREE_OPERAND (arg
, 0) == sender
)
13091 tgtcopy_stmt
= stmt
;
13097 gcc_assert (tgtcopy_stmt
!= NULL
);
13098 arg
= DECL_ARGUMENTS (child_fn
);
13100 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
13101 gsi_remove (&gsi
, true);
13104 /* Declare local variables needed in CHILD_CFUN. */
13105 block
= DECL_INITIAL (child_fn
);
13106 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
13107 /* The gimplifier could record temporaries in the offloading block
13108 rather than in containing function's local_decls chain,
13109 which would mean cgraph missed finalizing them. Do it now. */
13110 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
13111 if (TREE_CODE (t
) == VAR_DECL
13113 && !DECL_EXTERNAL (t
))
13114 varpool_node::finalize_decl (t
);
13115 DECL_SAVED_TREE (child_fn
) = NULL
;
13116 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13117 gimple_set_body (child_fn
, NULL
);
13118 TREE_USED (block
) = 1;
13120 /* Reset DECL_CONTEXT on function arguments. */
13121 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13122 DECL_CONTEXT (t
) = child_fn
;
13124 /* Split ENTRY_BB at GIMPLE_*,
13125 so that it can be moved to the child function. */
13126 gsi
= gsi_last_bb (entry_bb
);
13127 stmt
= gsi_stmt (gsi
);
13129 && gimple_code (stmt
) == gimple_code (entry_stmt
));
13130 e
= split_block (entry_bb
, stmt
);
13131 gsi_remove (&gsi
, true);
13132 entry_bb
= e
->dest
;
13133 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
13135 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13138 gsi
= gsi_last_bb (exit_bb
);
13139 gcc_assert (!gsi_end_p (gsi
)
13140 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13141 stmt
= gimple_build_return (NULL
);
13142 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
13143 gsi_remove (&gsi
, true);
13146 /* Move the offloading region into CHILD_CFUN. */
13148 block
= gimple_block (entry_stmt
);
13150 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
13152 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
13153 /* When the OMP expansion process cannot guarantee an up-to-date
13154 loop tree arrange for the child function to fixup loops. */
13155 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13156 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
13158 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13159 num
= vec_safe_length (child_cfun
->local_decls
);
13160 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
13162 t
= (*child_cfun
->local_decls
)[srcidx
];
13163 if (DECL_CONTEXT (t
) == cfun
->decl
)
13165 if (srcidx
!= dstidx
)
13166 (*child_cfun
->local_decls
)[dstidx
] = t
;
13170 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
13172 /* Inform the callgraph about the new function. */
13173 child_cfun
->curr_properties
= cfun
->curr_properties
;
13174 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
13175 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
13176 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
13177 node
->parallelized_function
= 1;
13178 cgraph_node::add_new_function (child_fn
, true);
13180 /* Add the new function to the offload table. */
13181 if (ENABLE_OFFLOADING
)
13182 vec_safe_push (offload_funcs
, child_fn
);
13184 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
13185 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
13187 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13188 fixed in a following pass. */
13189 push_cfun (child_cfun
);
13191 assign_assembler_name_if_neeeded (child_fn
);
13192 cgraph_edge::rebuild_edges ();
13194 /* Some EH regions might become dead, see PR34608. If
13195 pass_cleanup_cfg isn't the first pass to happen with the
13196 new child, these dead EH edges might cause problems.
13197 Clean them up now. */
13198 if (flag_exceptions
)
13201 bool changed
= false;
13203 FOR_EACH_BB_FN (bb
, cfun
)
13204 changed
|= gimple_purge_dead_eh_edges (bb
);
13206 cleanup_tree_cfg ();
13208 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13209 verify_loop_structure ();
13212 if (dump_file
&& !gimple_in_ssa_p (cfun
))
13214 omp_any_child_fn_dumped
= true;
13215 dump_function_header (dump_file
, child_fn
, dump_flags
);
13216 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
13220 /* Emit a library call to launch the offloading region, or do data
13222 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
13223 enum built_in_function start_ix
;
13224 location_t clause_loc
;
13225 unsigned int flags_i
= 0;
13226 bool oacc_kernels_p
= false;
13228 switch (gimple_omp_target_kind (entry_stmt
))
13230 case GF_OMP_TARGET_KIND_REGION
:
13231 start_ix
= BUILT_IN_GOMP_TARGET
;
13233 case GF_OMP_TARGET_KIND_DATA
:
13234 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
13236 case GF_OMP_TARGET_KIND_UPDATE
:
13237 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
13239 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13240 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13242 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13243 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13244 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
13246 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13247 oacc_kernels_p
= true;
13249 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13250 start_ix
= BUILT_IN_GOACC_PARALLEL
;
13252 case GF_OMP_TARGET_KIND_OACC_DATA
:
13253 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13254 start_ix
= BUILT_IN_GOACC_DATA_START
;
13256 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13257 start_ix
= BUILT_IN_GOACC_UPDATE
;
13259 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13260 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
13262 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13263 start_ix
= BUILT_IN_GOACC_DECLARE
;
13266 gcc_unreachable ();
13269 clauses
= gimple_omp_target_clauses (entry_stmt
);
13271 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13272 library choose) and there is no conditional. */
13274 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
13276 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
13278 cond
= OMP_CLAUSE_IF_EXPR (c
);
13280 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
13283 /* Even if we pass it to all library function calls, it is currently only
13284 defined/used for the OpenMP target ones. */
13285 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
13286 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
13287 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
13288 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
13290 device
= OMP_CLAUSE_DEVICE_ID (c
);
13291 clause_loc
= OMP_CLAUSE_LOCATION (c
);
13294 clause_loc
= gimple_location (entry_stmt
);
13296 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
13298 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
13300 /* Ensure 'device' is of the correct type. */
13301 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
13303 /* If we found the clause 'if (cond)', build
13304 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13307 cond
= gimple_boolify (cond
);
13309 basic_block cond_bb
, then_bb
, else_bb
;
13313 tmp_var
= create_tmp_var (TREE_TYPE (device
));
13315 e
= split_block_after_labels (new_bb
);
13318 gsi
= gsi_last_bb (new_bb
);
13320 e
= split_block (new_bb
, gsi_stmt (gsi
));
13326 then_bb
= create_empty_bb (cond_bb
);
13327 else_bb
= create_empty_bb (then_bb
);
13328 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
13329 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
13331 stmt
= gimple_build_cond_empty (cond
);
13332 gsi
= gsi_last_bb (cond_bb
);
13333 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13335 gsi
= gsi_start_bb (then_bb
);
13336 stmt
= gimple_build_assign (tmp_var
, device
);
13337 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13339 gsi
= gsi_start_bb (else_bb
);
13340 stmt
= gimple_build_assign (tmp_var
,
13341 build_int_cst (integer_type_node
,
13342 GOMP_DEVICE_HOST_FALLBACK
));
13343 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13345 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
13346 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
13347 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
13348 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
13349 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
13350 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
13355 gsi
= gsi_last_bb (new_bb
);
13356 t
= gimple_omp_target_data_arg (entry_stmt
);
13359 t1
= size_zero_node
;
13360 t2
= build_zero_cst (ptr_type_node
);
13366 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
13367 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
13368 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
13369 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
13370 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
13374 bool tagging
= false;
13375 /* The maximum number used by any start_ix, without varargs. */
13376 auto_vec
<tree
, 11> args
;
13377 args
.quick_push (device
);
13379 args
.quick_push (build_fold_addr_expr (child_fn
));
13380 args
.quick_push (t1
);
13381 args
.quick_push (t2
);
13382 args
.quick_push (t3
);
13383 args
.quick_push (t4
);
13386 case BUILT_IN_GOACC_DATA_START
:
13387 case BUILT_IN_GOACC_DECLARE
:
13388 case BUILT_IN_GOMP_TARGET_DATA
:
13390 case BUILT_IN_GOMP_TARGET
:
13391 case BUILT_IN_GOMP_TARGET_UPDATE
:
13392 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
13393 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
13394 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
13396 depend
= OMP_CLAUSE_DECL (c
);
13398 depend
= build_int_cst (ptr_type_node
, 0);
13399 args
.quick_push (depend
);
13400 if (start_ix
== BUILT_IN_GOMP_TARGET
)
13401 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
13403 case BUILT_IN_GOACC_PARALLEL
:
13405 set_oacc_fn_attrib (child_fn
, clauses
, oacc_kernels_p
, &args
);
13409 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
13410 case BUILT_IN_GOACC_UPDATE
:
13412 tree t_async
= NULL_TREE
;
13414 /* If present, use the value specified by the respective
13415 clause, making sure that is of the correct type. */
13416 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
13418 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13420 OMP_CLAUSE_ASYNC_EXPR (c
));
13422 /* Default values for t_async. */
13423 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
13425 build_int_cst (integer_type_node
,
13427 if (tagging
&& t_async
)
13429 unsigned HOST_WIDE_INT i_async
= GOMP_LAUNCH_OP_MAX
;
13431 if (TREE_CODE (t_async
) == INTEGER_CST
)
13433 /* See if we can pack the async arg in to the tag's
13435 i_async
= TREE_INT_CST_LOW (t_async
);
13436 if (i_async
< GOMP_LAUNCH_OP_MAX
)
13437 t_async
= NULL_TREE
;
13439 i_async
= GOMP_LAUNCH_OP_MAX
;
13441 args
.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC
, NULL_TREE
,
13445 args
.safe_push (t_async
);
13447 /* Save the argument index, and ... */
13448 unsigned t_wait_idx
= args
.length ();
13449 unsigned num_waits
= 0;
13450 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
13452 /* ... push a placeholder. */
13453 args
.safe_push (integer_zero_node
);
13455 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
13456 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
13458 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13460 OMP_CLAUSE_WAIT_EXPR (c
)));
13464 if (!tagging
|| num_waits
)
13468 /* Now that we know the number, update the placeholder. */
13470 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
13472 len
= build_int_cst (integer_type_node
, num_waits
);
13473 len
= fold_convert_loc (gimple_location (entry_stmt
),
13474 unsigned_type_node
, len
);
13475 args
[t_wait_idx
] = len
;
13480 gcc_unreachable ();
13483 /* Push terminal marker - zero. */
13484 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
13486 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
13487 gimple_set_location (g
, gimple_location (entry_stmt
));
13488 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13491 g
= gsi_stmt (gsi
);
13492 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
13493 gsi_remove (&gsi
, true);
13495 if (data_region
&& region
->exit
)
13497 gsi
= gsi_last_bb (region
->exit
);
13498 g
= gsi_stmt (gsi
);
13499 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
13500 gsi_remove (&gsi
, true);
13504 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13505 variable derived from the thread number. */
13508 grid_expand_omp_for_loop (struct omp_region
*kfor
)
13512 gimple_stmt_iterator gsi
;
13514 struct omp_for_data fd
;
13516 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
13517 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
13518 == GF_OMP_FOR_KIND_GRID_LOOP
);
13519 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
13521 gcc_assert (gimple_omp_for_collapse (for_stmt
) == 1);
13522 gcc_assert (kfor
->cont
);
13523 extract_omp_for_data (for_stmt
, &fd
, NULL
);
13525 itype
= type
= TREE_TYPE (fd
.loop
.v
);
13526 if (POINTER_TYPE_P (type
))
13527 itype
= signed_type_for (type
);
13529 gsi
= gsi_start_bb (body_bb
);
13532 step
= fd
.loop
.step
;
13533 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
13534 true, NULL_TREE
, true, GSI_SAME_STMT
);
13535 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
13536 true, NULL_TREE
, true, GSI_SAME_STMT
);
13537 threadid
= build_call_expr (builtin_decl_explicit
13538 (BUILT_IN_OMP_GET_THREAD_NUM
), 0);
13539 threadid
= fold_convert (itype
, threadid
);
13540 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
13541 true, GSI_SAME_STMT
);
13543 tree startvar
= fd
.loop
.v
;
13544 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
13545 if (POINTER_TYPE_P (type
))
13546 t
= fold_build_pointer_plus (n1
, t
);
13548 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
13549 t
= fold_convert (type
, t
);
13550 t
= force_gimple_operand_gsi (&gsi
, t
,
13552 && TREE_ADDRESSABLE (startvar
),
13553 NULL_TREE
, true, GSI_SAME_STMT
);
13554 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
13555 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
13557 /* Remove the omp for statement */
13558 gsi
= gsi_last_bb (kfor
->entry
);
13559 gsi_remove (&gsi
, true);
13561 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13562 gsi
= gsi_last_bb (kfor
->cont
);
13563 gcc_assert (!gsi_end_p (gsi
)
13564 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
13565 gsi_remove (&gsi
, true);
13567 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13568 gsi
= gsi_last_bb (kfor
->exit
);
13569 gcc_assert (!gsi_end_p (gsi
)
13570 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13571 gsi_remove (&gsi
, true);
13573 /* Fixup the much simpler CFG. */
13574 remove_edge (find_edge (kfor
->cont
, body_bb
));
13576 if (kfor
->cont
!= body_bb
)
13577 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
13578 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
13581 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13584 struct grid_arg_decl_map
13590 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13591 pertaining to kernel function. */
13594 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
13596 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13597 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
13600 if (t
== adm
->old_arg
)
13601 *tp
= adm
->new_arg
;
13602 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
13606 static void expand_omp (struct omp_region
*region
);
13608 /* If TARGET region contains a kernel body for loop, remove its region from the
13609 TARGET and expand it in GPGPU kernel fashion. */
13612 grid_expand_target_grid_body (struct omp_region
*target
)
13614 if (!hsa_gen_requested_p ())
13617 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
13618 struct omp_region
**pp
;
13620 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
13621 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
13624 struct omp_region
*gpukernel
= *pp
;
13626 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
13629 /* HSA cannot handle OACC stuff. */
13630 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
13632 gcc_checking_assert (orig_child_fndecl
);
13633 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13634 OMP_CLAUSE__GRIDDIM_
));
13635 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
13637 hsa_register_kernel (n
);
13641 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13642 OMP_CLAUSE__GRIDDIM_
));
13643 tree inside_block
= gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
13644 *pp
= gpukernel
->next
;
13645 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
13646 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
13649 struct omp_region
*kfor
= *pp
;
13651 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor
)->entry
))
13652 == GF_OMP_FOR_KIND_GRID_LOOP
);
13655 expand_omp (kfor
->inner
);
13656 if (gpukernel
->inner
)
13657 expand_omp (gpukernel
->inner
);
13659 tree kern_fndecl
= copy_node (orig_child_fndecl
);
13660 DECL_NAME (kern_fndecl
) = clone_function_name (kern_fndecl
, "kernel");
13661 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
13662 tree tgtblock
= gimple_block (tgt_stmt
);
13663 tree fniniblock
= make_node (BLOCK
);
13664 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = tgtblock
;
13665 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
13666 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
13667 DECL_INITIAL (kern_fndecl
) = fniniblock
;
13668 push_struct_function (kern_fndecl
);
13669 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
13670 init_tree_ssa (cfun
);
13673 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
13674 gcc_assert (!DECL_CHAIN (old_parm_decl
));
13675 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
13676 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
13677 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
13678 gcc_assert (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (kern_fndecl
))));
13679 DECL_RESULT (kern_fndecl
) = copy_node (DECL_RESULT (kern_fndecl
));
13680 DECL_CONTEXT (DECL_RESULT (kern_fndecl
)) = kern_fndecl
;
13681 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
13682 kern_cfun
->curr_properties
= cfun
->curr_properties
;
13684 remove_edge (BRANCH_EDGE (kfor
->entry
));
13685 grid_expand_omp_for_loop (kfor
);
13687 /* Remove the omp for statement */
13688 gimple_stmt_iterator gsi
= gsi_last_bb (gpukernel
->entry
);
13689 gsi_remove (&gsi
, true);
13690 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13692 gsi
= gsi_last_bb (gpukernel
->exit
);
13693 gcc_assert (!gsi_end_p (gsi
)
13694 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13695 gimple
*ret_stmt
= gimple_build_return (NULL
);
13696 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
13697 gsi_remove (&gsi
, true);
13699 /* Statements in the first BB in the target construct have been produced by
13700 target lowering and must be copied inside the GPUKERNEL, with the two
13701 exceptions of the first OMP statement and the OMP_DATA assignment
13703 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
13704 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
13705 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
13706 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
13707 !gsi_end_p (tsi
); gsi_next (&tsi
))
13709 gimple
*stmt
= gsi_stmt (tsi
);
13710 if (is_gimple_omp (stmt
))
13713 && is_gimple_assign (stmt
)
13714 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
13715 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
13717 gimple
*copy
= gimple_copy (stmt
);
13718 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
13719 gimple_set_block (copy
, fniniblock
);
13722 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
13723 gpukernel
->exit
, inside_block
);
13725 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
13726 kcn
->mark_force_output ();
13727 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
13729 hsa_register_kernel (kcn
, orig_child
);
13731 cgraph_node::add_new_function (kern_fndecl
, true);
13732 push_cfun (kern_cfun
);
13733 cgraph_edge::rebuild_edges ();
13735 /* Re-map any mention of the PARM_DECL of the original function to the
13736 PARM_DECL of the new one.
13738 TODO: It would be great if lowering produced references into the GPU
13739 kernel decl straight away and we did not have to do this. */
13740 struct grid_arg_decl_map adm
;
13741 adm
.old_arg
= old_parm_decl
;
13742 adm
.new_arg
= new_parm_decl
;
13744 FOR_EACH_BB_FN (bb
, kern_cfun
)
13746 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13748 gimple
*stmt
= gsi_stmt (gsi
);
13749 struct walk_stmt_info wi
;
13750 memset (&wi
, 0, sizeof (wi
));
13752 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
13760 /* Expand the parallel region tree rooted at REGION. Expansion
13761 proceeds in depth-first order. Innermost regions are expanded
13762 first. This way, parallel regions that require a new function to
13763 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13764 internal dependencies in their body. */
13767 expand_omp (struct omp_region
*region
)
13769 omp_any_child_fn_dumped
= false;
13772 location_t saved_location
;
13773 gimple
*inner_stmt
= NULL
;
13775 /* First, determine whether this is a combined parallel+workshare
13777 if (region
->type
== GIMPLE_OMP_PARALLEL
)
13778 determine_parallel_type (region
);
13779 else if (region
->type
== GIMPLE_OMP_TARGET
)
13780 grid_expand_target_grid_body (region
);
13782 if (region
->type
== GIMPLE_OMP_FOR
13783 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
13784 inner_stmt
= last_stmt (region
->inner
->entry
);
13787 expand_omp (region
->inner
);
13789 saved_location
= input_location
;
13790 if (gimple_has_location (last_stmt (region
->entry
)))
13791 input_location
= gimple_location (last_stmt (region
->entry
));
13793 switch (region
->type
)
13795 case GIMPLE_OMP_PARALLEL
:
13796 case GIMPLE_OMP_TASK
:
13797 expand_omp_taskreg (region
);
13800 case GIMPLE_OMP_FOR
:
13801 expand_omp_for (region
, inner_stmt
);
13804 case GIMPLE_OMP_SECTIONS
:
13805 expand_omp_sections (region
);
13808 case GIMPLE_OMP_SECTION
:
13809 /* Individual omp sections are handled together with their
13810 parent GIMPLE_OMP_SECTIONS region. */
13813 case GIMPLE_OMP_SINGLE
:
13814 expand_omp_single (region
);
13817 case GIMPLE_OMP_ORDERED
:
13819 gomp_ordered
*ord_stmt
13820 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13821 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13822 OMP_CLAUSE_DEPEND
))
13824 /* We'll expand these when expanding corresponding
13825 worksharing region with ordered(n) clause. */
13826 gcc_assert (region
->outer
13827 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13828 region
->ord_stmt
= ord_stmt
;
13833 case GIMPLE_OMP_MASTER
:
13834 case GIMPLE_OMP_TASKGROUP
:
13835 case GIMPLE_OMP_CRITICAL
:
13836 case GIMPLE_OMP_TEAMS
:
13837 expand_omp_synch (region
);
13840 case GIMPLE_OMP_ATOMIC_LOAD
:
13841 expand_omp_atomic (region
);
13844 case GIMPLE_OMP_TARGET
:
13845 expand_omp_target (region
);
13849 gcc_unreachable ();
13852 input_location
= saved_location
;
13853 region
= region
->next
;
13855 if (omp_any_child_fn_dumped
)
13858 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13859 omp_any_child_fn_dumped
= false;
13864 /* Helper for build_omp_regions. Scan the dominator tree starting at
13865 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13866 true, the function ends once a single tree is built (otherwise, whole
13867 forest of OMP constructs may be built). */
13870 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13873 gimple_stmt_iterator gsi
;
13877 gsi
= gsi_last_bb (bb
);
13878 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13880 struct omp_region
*region
;
13881 enum gimple_code code
;
13883 stmt
= gsi_stmt (gsi
);
13884 code
= gimple_code (stmt
);
13885 if (code
== GIMPLE_OMP_RETURN
)
13887 /* STMT is the return point out of region PARENT. Mark it
13888 as the exit point and make PARENT the immediately
13889 enclosing region. */
13890 gcc_assert (parent
);
13893 parent
= parent
->outer
;
13895 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13897 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13898 GIMPLE_OMP_RETURN, but matches with
13899 GIMPLE_OMP_ATOMIC_LOAD. */
13900 gcc_assert (parent
);
13901 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13904 parent
= parent
->outer
;
13906 else if (code
== GIMPLE_OMP_CONTINUE
)
13908 gcc_assert (parent
);
13911 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13913 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13914 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13918 region
= new_omp_region (bb
, code
, parent
);
13920 if (code
== GIMPLE_OMP_TARGET
)
13922 switch (gimple_omp_target_kind (stmt
))
13924 case GF_OMP_TARGET_KIND_REGION
:
13925 case GF_OMP_TARGET_KIND_DATA
:
13926 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13927 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13928 case GF_OMP_TARGET_KIND_OACC_DATA
:
13929 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13931 case GF_OMP_TARGET_KIND_UPDATE
:
13932 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13933 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13934 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13935 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13936 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13937 /* ..., other than for those stand-alone directives... */
13941 gcc_unreachable ();
13944 else if (code
== GIMPLE_OMP_ORDERED
13945 && find_omp_clause (gimple_omp_ordered_clauses
13946 (as_a
<gomp_ordered
*> (stmt
)),
13947 OMP_CLAUSE_DEPEND
))
13948 /* #pragma omp ordered depend is also just a stand-alone
13951 /* ..., this directive becomes the parent for a new region. */
13957 if (single_tree
&& !parent
)
13960 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13962 son
= next_dom_son (CDI_DOMINATORS
, son
))
13963 build_omp_regions_1 (son
, parent
, single_tree
);
13966 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13967 root_omp_region. */
13970 build_omp_regions_root (basic_block root
)
13972 gcc_assert (root_omp_region
== NULL
);
13973 build_omp_regions_1 (root
, NULL
, true);
13974 gcc_assert (root_omp_region
!= NULL
);
13977 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13980 omp_expand_local (basic_block head
)
13982 build_omp_regions_root (head
);
13983 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13985 fprintf (dump_file
, "\nOMP region tree\n\n");
13986 dump_omp_region (dump_file
, root_omp_region
, 0);
13987 fprintf (dump_file
, "\n");
13990 remove_exit_barriers (root_omp_region
);
13991 expand_omp (root_omp_region
);
13993 free_omp_regions ();
13996 /* Scan the CFG and build a tree of OMP regions. Return the root of
13997 the OMP region tree. */
14000 build_omp_regions (void)
14002 gcc_assert (root_omp_region
== NULL
);
14003 calculate_dominance_info (CDI_DOMINATORS
);
14004 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
14007 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
14009 static unsigned int
14010 execute_expand_omp (void)
14012 build_omp_regions ();
14014 if (!root_omp_region
)
14019 fprintf (dump_file
, "\nOMP region tree\n\n");
14020 dump_omp_region (dump_file
, root_omp_region
, 0);
14021 fprintf (dump_file
, "\n");
14024 remove_exit_barriers (root_omp_region
);
14026 expand_omp (root_omp_region
);
14028 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
14029 verify_loop_structure ();
14030 cleanup_tree_cfg ();
14032 free_omp_regions ();
14037 /* OMP expansion -- the default pass, run before creation of SSA form. */
14041 const pass_data pass_data_expand_omp
=
14043 GIMPLE_PASS
, /* type */
14044 "ompexp", /* name */
14045 OPTGROUP_NONE
, /* optinfo_flags */
14046 TV_NONE
, /* tv_id */
14047 PROP_gimple_any
, /* properties_required */
14048 PROP_gimple_eomp
, /* properties_provided */
14049 0, /* properties_destroyed */
14050 0, /* todo_flags_start */
14051 0, /* todo_flags_finish */
14054 class pass_expand_omp
: public gimple_opt_pass
14057 pass_expand_omp (gcc::context
*ctxt
)
14058 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
14061 /* opt_pass methods: */
14062 virtual unsigned int execute (function
*)
14064 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
14065 || flag_openmp_simd
!= 0)
14066 && !seen_error ());
14068 /* This pass always runs, to provide PROP_gimple_eomp.
14069 But often, there is nothing to do. */
14073 return execute_expand_omp ();
14076 }; // class pass_expand_omp
14078 } // anon namespace
14081 make_pass_expand_omp (gcc::context
*ctxt
)
14083 return new pass_expand_omp (ctxt
);
14088 const pass_data pass_data_expand_omp_ssa
=
14090 GIMPLE_PASS
, /* type */
14091 "ompexpssa", /* name */
14092 OPTGROUP_NONE
, /* optinfo_flags */
14093 TV_NONE
, /* tv_id */
14094 PROP_cfg
| PROP_ssa
, /* properties_required */
14095 PROP_gimple_eomp
, /* properties_provided */
14096 0, /* properties_destroyed */
14097 0, /* todo_flags_start */
14098 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
14101 class pass_expand_omp_ssa
: public gimple_opt_pass
14104 pass_expand_omp_ssa (gcc::context
*ctxt
)
14105 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
14108 /* opt_pass methods: */
14109 virtual bool gate (function
*fun
)
14111 return !(fun
->curr_properties
& PROP_gimple_eomp
);
14113 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
14114 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
14116 }; // class pass_expand_omp_ssa
14118 } // anon namespace
14121 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
14123 return new pass_expand_omp_ssa (ctxt
);
14126 /* Routines to lower OMP directives into OMP-GIMPLE. */
14128 /* If ctx is a worksharing context inside of a cancellable parallel
14129 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14130 and conditional branch to parallel's cancel_label to handle
14131 cancellation in the implicit barrier. */
14134 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
14136 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
14137 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
14138 if (gimple_omp_return_nowait_p (omp_return
))
14141 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
14142 && ctx
->outer
->cancellable
)
14144 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
14145 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
14146 tree lhs
= create_tmp_var (c_bool_type
);
14147 gimple_omp_return_set_lhs (omp_return
, lhs
);
14148 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
14149 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
14150 fold_convert (c_bool_type
,
14151 boolean_false_node
),
14152 ctx
->outer
->cancel_label
, fallthru_label
);
14153 gimple_seq_add_stmt (body
, g
);
14154 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
14158 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14159 CTX is the enclosing OMP context for the current statement. */
14162 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14164 tree block
, control
;
14165 gimple_stmt_iterator tgsi
;
14166 gomp_sections
*stmt
;
14168 gbind
*new_stmt
, *bind
;
14169 gimple_seq ilist
, dlist
, olist
, new_body
;
14171 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
14173 push_gimplify_context ();
14177 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
14178 &ilist
, &dlist
, ctx
, NULL
);
14180 new_body
= gimple_omp_body (stmt
);
14181 gimple_omp_set_body (stmt
, NULL
);
14182 tgsi
= gsi_start (new_body
);
14183 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
14188 sec_start
= gsi_stmt (tgsi
);
14189 sctx
= maybe_lookup_ctx (sec_start
);
14192 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
14193 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
14194 GSI_CONTINUE_LINKING
);
14195 gimple_omp_set_body (sec_start
, NULL
);
14197 if (gsi_one_before_end_p (tgsi
))
14199 gimple_seq l
= NULL
;
14200 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
14202 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
14203 gimple_omp_section_set_last (sec_start
);
14206 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
14207 GSI_CONTINUE_LINKING
);
14210 block
= make_node (BLOCK
);
14211 bind
= gimple_build_bind (NULL
, new_body
, block
);
14214 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
14216 block
= make_node (BLOCK
);
14217 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14218 gsi_replace (gsi_p
, new_stmt
, true);
14220 pop_gimplify_context (new_stmt
);
14221 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14222 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14223 if (BLOCK_VARS (block
))
14224 TREE_USED (block
) = 1;
14227 gimple_seq_add_seq (&new_body
, ilist
);
14228 gimple_seq_add_stmt (&new_body
, stmt
);
14229 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
14230 gimple_seq_add_stmt (&new_body
, bind
);
14232 control
= create_tmp_var (unsigned_type_node
, ".section");
14233 t
= gimple_build_omp_continue (control
, control
);
14234 gimple_omp_sections_set_control (stmt
, control
);
14235 gimple_seq_add_stmt (&new_body
, t
);
14237 gimple_seq_add_seq (&new_body
, olist
);
14238 if (ctx
->cancellable
)
14239 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14240 gimple_seq_add_seq (&new_body
, dlist
);
14242 new_body
= maybe_catch_exception (new_body
);
14244 t
= gimple_build_omp_return
14245 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
14246 OMP_CLAUSE_NOWAIT
));
14247 gimple_seq_add_stmt (&new_body
, t
);
14248 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
14250 gimple_bind_set_body (new_stmt
, new_body
);
14254 /* A subroutine of lower_omp_single. Expand the simple form of
14255 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14257 if (GOMP_single_start ())
14259 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14261 FIXME. It may be better to delay expanding the logic of this until
14262 pass_expand_omp. The expanded logic may make the job more difficult
14263 to a synchronization analysis pass. */
14266 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
14268 location_t loc
= gimple_location (single_stmt
);
14269 tree tlabel
= create_artificial_label (loc
);
14270 tree flabel
= create_artificial_label (loc
);
14271 gimple
*call
, *cond
;
14274 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
14275 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
14276 call
= gimple_build_call (decl
, 0);
14277 gimple_call_set_lhs (call
, lhs
);
14278 gimple_seq_add_stmt (pre_p
, call
);
14280 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
14281 fold_convert_loc (loc
, TREE_TYPE (lhs
),
14282 boolean_true_node
),
14284 gimple_seq_add_stmt (pre_p
, cond
);
14285 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
14286 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14287 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
14291 /* A subroutine of lower_omp_single. Expand the simple form of
14292 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14294 #pragma omp single copyprivate (a, b, c)
14296 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14299 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14305 GOMP_single_copy_end (©out);
14316 FIXME. It may be better to delay expanding the logic of this until
14317 pass_expand_omp. The expanded logic may make the job more difficult
14318 to a synchronization analysis pass. */
14321 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
14324 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
14325 gimple_seq copyin_seq
;
14326 location_t loc
= gimple_location (single_stmt
);
14328 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
14330 ptr_type
= build_pointer_type (ctx
->record_type
);
14331 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
14333 l0
= create_artificial_label (loc
);
14334 l1
= create_artificial_label (loc
);
14335 l2
= create_artificial_label (loc
);
14337 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
14338 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
14339 t
= fold_convert_loc (loc
, ptr_type
, t
);
14340 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
14342 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
14343 build_int_cst (ptr_type
, 0));
14344 t
= build3 (COND_EXPR
, void_type_node
, t
,
14345 build_and_jump (&l0
), build_and_jump (&l1
));
14346 gimplify_and_add (t
, pre_p
);
14348 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
14350 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14353 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
14356 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14357 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
14358 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
14359 gimplify_and_add (t
, pre_p
);
14361 t
= build_and_jump (&l2
);
14362 gimplify_and_add (t
, pre_p
);
14364 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
14366 gimple_seq_add_seq (pre_p
, copyin_seq
);
14368 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
14372 /* Expand code for an OpenMP single directive. */
14375 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14379 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
14381 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
14383 push_gimplify_context ();
14385 block
= make_node (BLOCK
);
14386 bind
= gimple_build_bind (NULL
, NULL
, block
);
14387 gsi_replace (gsi_p
, bind
, true);
14390 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
14391 &bind_body
, &dlist
, ctx
, NULL
);
14392 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
14394 gimple_seq_add_stmt (&bind_body
, single_stmt
);
14396 if (ctx
->record_type
)
14397 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
14399 lower_omp_single_simple (single_stmt
, &bind_body
);
14401 gimple_omp_set_body (single_stmt
, NULL
);
14403 gimple_seq_add_seq (&bind_body
, dlist
);
14405 bind_body
= maybe_catch_exception (bind_body
);
14407 t
= gimple_build_omp_return
14408 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
14409 OMP_CLAUSE_NOWAIT
));
14410 gimple_seq_add_stmt (&bind_body_tail
, t
);
14411 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
14412 if (ctx
->record_type
)
14414 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
14415 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14416 TREE_THIS_VOLATILE (clobber
) = 1;
14417 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
14418 clobber
), GSI_SAME_STMT
);
14420 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
14421 gimple_bind_set_body (bind
, bind_body
);
14423 pop_gimplify_context (bind
);
14425 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14426 BLOCK_VARS (block
) = ctx
->block_vars
;
14427 if (BLOCK_VARS (block
))
14428 TREE_USED (block
) = 1;
14432 /* Expand code for an OpenMP master directive. */
14435 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14437 tree block
, lab
= NULL
, x
, bfn_decl
;
14438 gimple
*stmt
= gsi_stmt (*gsi_p
);
14440 location_t loc
= gimple_location (stmt
);
14443 push_gimplify_context ();
14445 block
= make_node (BLOCK
);
14446 bind
= gimple_build_bind (NULL
, NULL
, block
);
14447 gsi_replace (gsi_p
, bind
, true);
14448 gimple_bind_add_stmt (bind
, stmt
);
14450 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
14451 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
14452 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
14453 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
14455 gimplify_and_add (x
, &tseq
);
14456 gimple_bind_add_seq (bind
, tseq
);
14458 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14459 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14460 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14461 gimple_omp_set_body (stmt
, NULL
);
14463 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
14465 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14467 pop_gimplify_context (bind
);
14469 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14470 BLOCK_VARS (block
) = ctx
->block_vars
;
14474 /* Expand code for an OpenMP taskgroup directive. */
14477 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14479 gimple
*stmt
= gsi_stmt (*gsi_p
);
14482 tree block
= make_node (BLOCK
);
14484 bind
= gimple_build_bind (NULL
, NULL
, block
);
14485 gsi_replace (gsi_p
, bind
, true);
14486 gimple_bind_add_stmt (bind
, stmt
);
14488 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
14490 gimple_bind_add_stmt (bind
, x
);
14492 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14493 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14494 gimple_omp_set_body (stmt
, NULL
);
14496 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14498 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14499 BLOCK_VARS (block
) = ctx
->block_vars
;
14503 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14506 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
14509 struct omp_for_data fd
;
14510 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
14513 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
14514 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
14515 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
14519 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14520 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
14521 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14522 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
14524 /* Merge depend clauses from multiple adjacent
14525 #pragma omp ordered depend(sink:...) constructs
14526 into one #pragma omp ordered depend(sink:...), so that
14527 we can optimize them together. */
14528 gimple_stmt_iterator gsi
= *gsi_p
;
14530 while (!gsi_end_p (gsi
))
14532 gimple
*stmt
= gsi_stmt (gsi
);
14533 if (is_gimple_debug (stmt
)
14534 || gimple_code (stmt
) == GIMPLE_NOP
)
14539 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
14541 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
14542 c
= gimple_omp_ordered_clauses (ord_stmt2
);
14544 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
14545 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14548 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
14550 gsi_remove (&gsi
, true);
14554 /* Canonicalize sink dependence clauses into one folded clause if
14557 The basic algorithm is to create a sink vector whose first
14558 element is the GCD of all the first elements, and whose remaining
14559 elements are the minimum of the subsequent columns.
14561 We ignore dependence vectors whose first element is zero because
14562 such dependencies are known to be executed by the same thread.
14564 We take into account the direction of the loop, so a minimum
14565 becomes a maximum if the loop is iterating forwards. We also
14566 ignore sink clauses where the loop direction is unknown, or where
14567 the offsets are clearly invalid because they are not a multiple
14568 of the loop increment.
14572 #pragma omp for ordered(2)
14573 for (i=0; i < N; ++i)
14574 for (j=0; j < M; ++j)
14576 #pragma omp ordered \
14577 depend(sink:i-8,j-2) \
14578 depend(sink:i,j-1) \ // Completely ignored because i+0.
14579 depend(sink:i-4,j-3) \
14580 depend(sink:i-6,j-4)
14581 #pragma omp ordered depend(source)
14586 depend(sink:-gcd(8,4,6),-min(2,3,4))
14591 /* FIXME: Computing GCD's where the first element is zero is
14592 non-trivial in the presence of collapsed loops. Do this later. */
14593 if (fd
.collapse
> 1)
14596 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
14597 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
14598 tree folded_dep
= NULL_TREE
;
14599 /* TRUE if the first dimension's offset is negative. */
14600 bool neg_offset_p
= false;
14602 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14604 while ((c
= *list_p
) != NULL
)
14606 bool remove
= false;
14608 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
14609 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14610 goto next_ordered_clause
;
14613 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
14614 vec
&& TREE_CODE (vec
) == TREE_LIST
;
14615 vec
= TREE_CHAIN (vec
), ++i
)
14617 gcc_assert (i
< len
);
14619 /* extract_omp_for_data has canonicalized the condition. */
14620 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
14621 || fd
.loops
[i
].cond_code
== GT_EXPR
);
14622 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
14623 bool maybe_lexically_later
= true;
14625 /* While the committee makes up its mind, bail if we have any
14626 non-constant steps. */
14627 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
14628 goto lower_omp_ordered_ret
;
14630 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
14631 if (POINTER_TYPE_P (itype
))
14633 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
14634 TYPE_PRECISION (itype
),
14635 TYPE_SIGN (itype
));
14637 /* Ignore invalid offsets that are not multiples of the step. */
14638 if (!wi::multiple_of_p
14639 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
14642 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
14643 "ignoring sink clause with offset that is not "
14644 "a multiple of the loop step");
14646 goto next_ordered_clause
;
14649 /* Calculate the first dimension. The first dimension of
14650 the folded dependency vector is the GCD of the first
14651 elements, while ignoring any first elements whose offset
14655 /* Ignore dependence vectors whose first dimension is 0. */
14659 goto next_ordered_clause
;
14663 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
14665 error_at (OMP_CLAUSE_LOCATION (c
),
14666 "first offset must be in opposite direction "
14667 "of loop iterations");
14668 goto lower_omp_ordered_ret
;
14672 neg_offset_p
= forward
;
14673 /* Initialize the first time around. */
14674 if (folded_dep
== NULL_TREE
)
14677 folded_deps
[0] = offset
;
14680 folded_deps
[0] = wi::gcd (folded_deps
[0],
14684 /* Calculate minimum for the remaining dimensions. */
14687 folded_deps
[len
+ i
- 1] = offset
;
14688 if (folded_dep
== c
)
14689 folded_deps
[i
] = offset
;
14690 else if (maybe_lexically_later
14691 && !wi::eq_p (folded_deps
[i
], offset
))
14693 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
14697 for (j
= 1; j
<= i
; j
++)
14698 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
14701 maybe_lexically_later
= false;
14705 gcc_assert (i
== len
);
14709 next_ordered_clause
:
14711 *list_p
= OMP_CLAUSE_CHAIN (c
);
14713 list_p
= &OMP_CLAUSE_CHAIN (c
);
14719 folded_deps
[0] = -folded_deps
[0];
14721 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
14722 if (POINTER_TYPE_P (itype
))
14725 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14726 = wide_int_to_tree (itype
, folded_deps
[0]);
14727 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14728 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14731 lower_omp_ordered_ret
:
14733 /* Ordered without clauses is #pragma omp threads, while we want
14734 a nop instead if we remove all clauses. */
14735 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14736 gsi_replace (gsi_p
, gimple_build_nop (), true);
14740 /* Expand code for an OpenMP ordered directive. */
14743 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14746 gimple
*stmt
= gsi_stmt (*gsi_p
);
14747 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14750 bool simd
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14752 bool threads
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14753 OMP_CLAUSE_THREADS
);
14755 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14756 OMP_CLAUSE_DEPEND
))
14758 /* FIXME: This is needs to be moved to the expansion to verify various
14759 conditions only testable on cfg with dominators computed, and also
14760 all the depend clauses to be merged still might need to be available
14761 for the runtime checks. */
14763 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14767 push_gimplify_context ();
14769 block
= make_node (BLOCK
);
14770 bind
= gimple_build_bind (NULL
, NULL
, block
);
14771 gsi_replace (gsi_p
, bind
, true);
14772 gimple_bind_add_stmt (bind
, stmt
);
14776 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
14777 build_int_cst (NULL_TREE
, threads
));
14778 cfun
->has_simduid_loops
= true;
14781 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14783 gimple_bind_add_stmt (bind
, x
);
14785 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14786 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14787 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14788 gimple_omp_set_body (stmt
, NULL
);
14791 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
14792 build_int_cst (NULL_TREE
, threads
));
14794 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14796 gimple_bind_add_stmt (bind
, x
);
14798 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14800 pop_gimplify_context (bind
);
14802 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14803 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14807 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14808 substitution of a couple of function calls. But in the NAMED case,
14809 requires that languages coordinate a symbol name. It is therefore
14810 best put here in common code. */
14812 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14815 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14818 tree name
, lock
, unlock
;
14819 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14821 location_t loc
= gimple_location (stmt
);
14824 name
= gimple_omp_critical_name (stmt
);
14829 if (!critical_name_mutexes
)
14830 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14832 tree
*n
= critical_name_mutexes
->get (name
);
14837 decl
= create_tmp_var_raw (ptr_type_node
);
14839 new_str
= ACONCAT ((".gomp_critical_user_",
14840 IDENTIFIER_POINTER (name
), NULL
));
14841 DECL_NAME (decl
) = get_identifier (new_str
);
14842 TREE_PUBLIC (decl
) = 1;
14843 TREE_STATIC (decl
) = 1;
14844 DECL_COMMON (decl
) = 1;
14845 DECL_ARTIFICIAL (decl
) = 1;
14846 DECL_IGNORED_P (decl
) = 1;
14848 varpool_node::finalize_decl (decl
);
14850 critical_name_mutexes
->put (name
, decl
);
14855 /* If '#pragma omp critical' is inside offloaded region or
14856 inside function marked as offloadable, the symbol must be
14857 marked as offloadable too. */
14859 if (cgraph_node::get (current_function_decl
)->offloadable
)
14860 varpool_node::get_create (decl
)->offloadable
= 1;
14862 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14863 if (is_gimple_omp_offloaded (octx
->stmt
))
14865 varpool_node::get_create (decl
)->offloadable
= 1;
14869 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14870 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14872 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14873 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14874 build_fold_addr_expr_loc (loc
, decl
));
14878 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14879 lock
= build_call_expr_loc (loc
, lock
, 0);
14881 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14882 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14885 push_gimplify_context ();
14887 block
= make_node (BLOCK
);
14888 bind
= gimple_build_bind (NULL
, NULL
, block
);
14889 gsi_replace (gsi_p
, bind
, true);
14890 gimple_bind_add_stmt (bind
, stmt
);
14892 tbody
= gimple_bind_body (bind
);
14893 gimplify_and_add (lock
, &tbody
);
14894 gimple_bind_set_body (bind
, tbody
);
14896 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14897 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14898 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14899 gimple_omp_set_body (stmt
, NULL
);
14901 tbody
= gimple_bind_body (bind
);
14902 gimplify_and_add (unlock
, &tbody
);
14903 gimple_bind_set_body (bind
, tbody
);
14905 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14907 pop_gimplify_context (bind
);
14908 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14909 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14913 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14914 for a lastprivate clause. Given a loop control predicate of (V
14915 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14916 is appended to *DLIST, iterator initialization is appended to
14920 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14921 gimple_seq
*dlist
, struct omp_context
*ctx
)
14923 tree clauses
, cond
, vinit
;
14924 enum tree_code cond_code
;
14927 cond_code
= fd
->loop
.cond_code
;
14928 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14930 /* When possible, use a strict equality expression. This can let VRP
14931 type optimizations deduce the value and remove a copy. */
14932 if (tree_fits_shwi_p (fd
->loop
.step
))
14934 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14935 if (step
== 1 || step
== -1)
14936 cond_code
= EQ_EXPR
;
14939 tree n2
= fd
->loop
.n2
;
14940 if (fd
->collapse
> 1
14941 && TREE_CODE (n2
) != INTEGER_CST
14942 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14944 struct omp_context
*taskreg_ctx
= NULL
;
14945 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14947 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14948 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14949 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14951 if (gimple_omp_for_combined_into_p (gfor
))
14953 gcc_assert (ctx
->outer
->outer
14954 && is_parallel_ctx (ctx
->outer
->outer
));
14955 taskreg_ctx
= ctx
->outer
->outer
;
14959 struct omp_for_data outer_fd
;
14960 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14961 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14964 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14965 taskreg_ctx
= ctx
->outer
->outer
;
14967 else if (is_taskreg_ctx (ctx
->outer
))
14968 taskreg_ctx
= ctx
->outer
;
14973 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14974 OMP_CLAUSE__LOOPTEMP_
);
14975 gcc_assert (innerc
);
14976 for (i
= 0; i
< fd
->collapse
; i
++)
14978 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14979 OMP_CLAUSE__LOOPTEMP_
);
14980 gcc_assert (innerc
);
14982 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14983 OMP_CLAUSE__LOOPTEMP_
);
14985 n2
= fold_convert (TREE_TYPE (n2
),
14986 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14990 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14992 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14994 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14995 if (!gimple_seq_empty_p (stmts
))
14997 gimple_seq_add_seq (&stmts
, *dlist
);
15000 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
15001 vinit
= fd
->loop
.n1
;
15002 if (cond_code
== EQ_EXPR
15003 && tree_fits_shwi_p (fd
->loop
.n2
)
15004 && ! integer_zerop (fd
->loop
.n2
))
15005 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
15007 vinit
= unshare_expr (vinit
);
15009 /* Initialize the iterator variable, so that threads that don't execute
15010 any iterations don't execute the lastprivate clauses by accident. */
15011 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
15016 /* Lower code for an OMP loop directive. */
15019 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15021 tree
*rhs_p
, block
;
15022 struct omp_for_data fd
, *fdp
= NULL
;
15023 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
15025 gimple_seq omp_for_body
, body
, dlist
;
15026 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
15029 push_gimplify_context ();
15031 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
15033 block
= make_node (BLOCK
);
15034 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
15035 /* Replace at gsi right away, so that 'stmt' is no member
15036 of a sequence anymore as we're going to add to a different
15038 gsi_replace (gsi_p
, new_stmt
, true);
15040 /* Move declaration of temporaries in the loop body before we make
15042 omp_for_body
= gimple_omp_body (stmt
);
15043 if (!gimple_seq_empty_p (omp_for_body
)
15044 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
15047 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
15048 tree vars
= gimple_bind_vars (inner_bind
);
15049 gimple_bind_append_vars (new_stmt
, vars
);
15050 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
15051 keep them on the inner_bind and it's block. */
15052 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
15053 if (gimple_bind_block (inner_bind
))
15054 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
15057 if (gimple_omp_for_combined_into_p (stmt
))
15059 extract_omp_for_data (stmt
, &fd
, NULL
);
15062 /* We need two temporaries with fd.loop.v type (istart/iend)
15063 and then (fd.collapse - 1) temporaries with the same
15064 type for count2 ... countN-1 vars if not constant. */
15066 tree type
= fd
.iter_type
;
15067 if (fd
.collapse
> 1
15068 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
15069 count
+= fd
.collapse
- 1;
15071 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
15072 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
15073 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
15074 tree clauses
= *pc
;
15077 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
15078 OMP_CLAUSE__LOOPTEMP_
);
15079 for (i
= 0; i
< count
; i
++)
15084 gcc_assert (outerc
);
15085 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
15086 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
15087 OMP_CLAUSE__LOOPTEMP_
);
15091 temp
= create_tmp_var (type
);
15092 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
15094 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
15095 OMP_CLAUSE_DECL (*pc
) = temp
;
15096 pc
= &OMP_CLAUSE_CHAIN (*pc
);
15101 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15104 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
15106 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
15108 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
15110 /* Lower the header expressions. At this point, we can assume that
15111 the header is of the form:
15113 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15115 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15116 using the .omp_data_s mapping, if needed. */
15117 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
15119 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
15120 if (!is_gimple_min_invariant (*rhs_p
))
15121 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15123 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
15124 if (!is_gimple_min_invariant (*rhs_p
))
15125 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15127 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
15128 if (!is_gimple_min_invariant (*rhs_p
))
15129 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15132 /* Once lowered, extract the bounds and clauses. */
15133 extract_omp_for_data (stmt
, &fd
, NULL
);
15135 if (is_gimple_omp_oacc (ctx
->stmt
)
15136 && !ctx_in_oacc_kernels_region (ctx
))
15137 lower_oacc_head_tail (gimple_location (stmt
),
15138 gimple_omp_for_clauses (stmt
),
15139 &oacc_head
, &oacc_tail
, ctx
);
15141 /* Add OpenACC partitioning and reduction markers just before the loop */
15143 gimple_seq_add_seq (&body
, oacc_head
);
15145 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
15147 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
15148 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15149 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
15150 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
15152 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
15153 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
15154 OMP_CLAUSE_LINEAR_STEP (c
)
15155 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
15159 if (!gimple_omp_for_grid_phony (stmt
))
15160 gimple_seq_add_stmt (&body
, stmt
);
15161 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
15163 if (!gimple_omp_for_grid_phony (stmt
))
15164 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
15167 /* After the loop, add exit clauses. */
15168 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
15170 if (ctx
->cancellable
)
15171 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
15173 gimple_seq_add_seq (&body
, dlist
);
15175 body
= maybe_catch_exception (body
);
15177 if (!gimple_omp_for_grid_phony (stmt
))
15179 /* Region exit marker goes at the end of the loop body. */
15180 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
15181 maybe_add_implicit_barrier_cancel (ctx
, &body
);
15184 /* Add OpenACC joining and reduction markers just after the loop. */
15186 gimple_seq_add_seq (&body
, oacc_tail
);
15188 pop_gimplify_context (new_stmt
);
15190 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
15191 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
15192 if (BLOCK_VARS (block
))
15193 TREE_USED (block
) = 1;
15195 gimple_bind_set_body (new_stmt
, body
);
15196 gimple_omp_set_body (stmt
, NULL
);
15197 gimple_omp_for_set_pre_body (stmt
, NULL
);
15200 /* Callback for walk_stmts. Check if the current statement only contains
15201 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15204 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
15205 bool *handled_ops_p
,
15206 struct walk_stmt_info
*wi
)
15208 int *info
= (int *) wi
->info
;
15209 gimple
*stmt
= gsi_stmt (*gsi_p
);
15211 *handled_ops_p
= true;
15212 switch (gimple_code (stmt
))
15216 case GIMPLE_OMP_FOR
:
15217 case GIMPLE_OMP_SECTIONS
:
15218 *info
= *info
== 0 ? 1 : -1;
15227 struct omp_taskcopy_context
15229 /* This field must be at the beginning, as we do "inheritance": Some
15230 callback functions for tree-inline.c (e.g., omp_copy_decl)
15231 receive a copy_body_data pointer that is up-casted to an
15232 omp_context pointer. */
15238 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
15240 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
15242 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
15243 return create_tmp_var (TREE_TYPE (var
));
15249 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
15251 tree name
, new_fields
= NULL
, type
, f
;
15253 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
15254 name
= DECL_NAME (TYPE_NAME (orig_type
));
15255 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
15256 TYPE_DECL
, name
, type
);
15257 TYPE_NAME (type
) = name
;
15259 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
15261 tree new_f
= copy_node (f
);
15262 DECL_CONTEXT (new_f
) = type
;
15263 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
15264 TREE_CHAIN (new_f
) = new_fields
;
15265 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15266 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15267 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
15269 new_fields
= new_f
;
15270 tcctx
->cb
.decl_map
->put (f
, new_f
);
15272 TYPE_FIELDS (type
) = nreverse (new_fields
);
15273 layout_type (type
);
15277 /* Create task copyfn. */
15280 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
15282 struct function
*child_cfun
;
15283 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
15284 tree record_type
, srecord_type
, bind
, list
;
15285 bool record_needs_remap
= false, srecord_needs_remap
= false;
15287 struct omp_taskcopy_context tcctx
;
15288 location_t loc
= gimple_location (task_stmt
);
15290 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
15291 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
15292 gcc_assert (child_cfun
->cfg
== NULL
);
15293 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
15295 /* Reset DECL_CONTEXT on function arguments. */
15296 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
15297 DECL_CONTEXT (t
) = child_fn
;
15299 /* Populate the function. */
15300 push_gimplify_context ();
15301 push_cfun (child_cfun
);
15303 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
15304 TREE_SIDE_EFFECTS (bind
) = 1;
15306 DECL_SAVED_TREE (child_fn
) = bind
;
15307 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
15309 /* Remap src and dst argument types if needed. */
15310 record_type
= ctx
->record_type
;
15311 srecord_type
= ctx
->srecord_type
;
15312 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
15313 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15315 record_needs_remap
= true;
15318 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
15319 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15321 srecord_needs_remap
= true;
15325 if (record_needs_remap
|| srecord_needs_remap
)
15327 memset (&tcctx
, '\0', sizeof (tcctx
));
15328 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
15329 tcctx
.cb
.dst_fn
= child_fn
;
15330 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
15331 gcc_checking_assert (tcctx
.cb
.src_node
);
15332 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
15333 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
15334 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
15335 tcctx
.cb
.eh_lp_nr
= 0;
15336 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
15337 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
15340 if (record_needs_remap
)
15341 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
15342 if (srecord_needs_remap
)
15343 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
15346 tcctx
.cb
.decl_map
= NULL
;
15348 arg
= DECL_ARGUMENTS (child_fn
);
15349 TREE_TYPE (arg
) = build_pointer_type (record_type
);
15350 sarg
= DECL_CHAIN (arg
);
15351 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
15353 /* First pass: initialize temporaries used in record_type and srecord_type
15354 sizes and field offsets. */
15355 if (tcctx
.cb
.decl_map
)
15356 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15357 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15361 decl
= OMP_CLAUSE_DECL (c
);
15362 p
= tcctx
.cb
.decl_map
->get (decl
);
15365 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15366 sf
= (tree
) n
->value
;
15367 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15368 src
= build_simple_mem_ref_loc (loc
, sarg
);
15369 src
= omp_build_component_ref (src
, sf
);
15370 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
15371 append_to_statement_list (t
, &list
);
15374 /* Second pass: copy shared var pointers and copy construct non-VLA
15375 firstprivate vars. */
15376 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15377 switch (OMP_CLAUSE_CODE (c
))
15379 splay_tree_key key
;
15380 case OMP_CLAUSE_SHARED
:
15381 decl
= OMP_CLAUSE_DECL (c
);
15382 key
= (splay_tree_key
) decl
;
15383 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
15384 key
= (splay_tree_key
) &DECL_UID (decl
);
15385 n
= splay_tree_lookup (ctx
->field_map
, key
);
15388 f
= (tree
) n
->value
;
15389 if (tcctx
.cb
.decl_map
)
15390 f
= *tcctx
.cb
.decl_map
->get (f
);
15391 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
15392 sf
= (tree
) n
->value
;
15393 if (tcctx
.cb
.decl_map
)
15394 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15395 src
= build_simple_mem_ref_loc (loc
, sarg
);
15396 src
= omp_build_component_ref (src
, sf
);
15397 dst
= build_simple_mem_ref_loc (loc
, arg
);
15398 dst
= omp_build_component_ref (dst
, f
);
15399 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15400 append_to_statement_list (t
, &list
);
15402 case OMP_CLAUSE_FIRSTPRIVATE
:
15403 decl
= OMP_CLAUSE_DECL (c
);
15404 if (is_variable_sized (decl
))
15406 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15409 f
= (tree
) n
->value
;
15410 if (tcctx
.cb
.decl_map
)
15411 f
= *tcctx
.cb
.decl_map
->get (f
);
15412 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15415 sf
= (tree
) n
->value
;
15416 if (tcctx
.cb
.decl_map
)
15417 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15418 src
= build_simple_mem_ref_loc (loc
, sarg
);
15419 src
= omp_build_component_ref (src
, sf
);
15420 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
15421 src
= build_simple_mem_ref_loc (loc
, src
);
15425 dst
= build_simple_mem_ref_loc (loc
, arg
);
15426 dst
= omp_build_component_ref (dst
, f
);
15427 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15428 append_to_statement_list (t
, &list
);
15430 case OMP_CLAUSE_PRIVATE
:
15431 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
15433 decl
= OMP_CLAUSE_DECL (c
);
15434 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15435 f
= (tree
) n
->value
;
15436 if (tcctx
.cb
.decl_map
)
15437 f
= *tcctx
.cb
.decl_map
->get (f
);
15438 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15441 sf
= (tree
) n
->value
;
15442 if (tcctx
.cb
.decl_map
)
15443 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15444 src
= build_simple_mem_ref_loc (loc
, sarg
);
15445 src
= omp_build_component_ref (src
, sf
);
15446 if (use_pointer_for_field (decl
, NULL
))
15447 src
= build_simple_mem_ref_loc (loc
, src
);
15451 dst
= build_simple_mem_ref_loc (loc
, arg
);
15452 dst
= omp_build_component_ref (dst
, f
);
15453 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15454 append_to_statement_list (t
, &list
);
15460 /* Last pass: handle VLA firstprivates. */
15461 if (tcctx
.cb
.decl_map
)
15462 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15463 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15467 decl
= OMP_CLAUSE_DECL (c
);
15468 if (!is_variable_sized (decl
))
15470 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15473 f
= (tree
) n
->value
;
15474 f
= *tcctx
.cb
.decl_map
->get (f
);
15475 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
15476 ind
= DECL_VALUE_EXPR (decl
);
15477 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
15478 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
15479 n
= splay_tree_lookup (ctx
->sfield_map
,
15480 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15481 sf
= (tree
) n
->value
;
15482 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15483 src
= build_simple_mem_ref_loc (loc
, sarg
);
15484 src
= omp_build_component_ref (src
, sf
);
15485 src
= build_simple_mem_ref_loc (loc
, src
);
15486 dst
= build_simple_mem_ref_loc (loc
, arg
);
15487 dst
= omp_build_component_ref (dst
, f
);
15488 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15489 append_to_statement_list (t
, &list
);
15490 n
= splay_tree_lookup (ctx
->field_map
,
15491 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15492 df
= (tree
) n
->value
;
15493 df
= *tcctx
.cb
.decl_map
->get (df
);
15494 ptr
= build_simple_mem_ref_loc (loc
, arg
);
15495 ptr
= omp_build_component_ref (ptr
, df
);
15496 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
15497 build_fold_addr_expr_loc (loc
, dst
));
15498 append_to_statement_list (t
, &list
);
15501 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
15502 append_to_statement_list (t
, &list
);
15504 if (tcctx
.cb
.decl_map
)
15505 delete tcctx
.cb
.decl_map
;
15506 pop_gimplify_context (NULL
);
15507 BIND_EXPR_BODY (bind
) = list
;
15512 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
15516 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
15518 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
15519 gcc_assert (clauses
);
15520 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15521 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
15522 switch (OMP_CLAUSE_DEPEND_KIND (c
))
15524 case OMP_CLAUSE_DEPEND_IN
:
15527 case OMP_CLAUSE_DEPEND_OUT
:
15528 case OMP_CLAUSE_DEPEND_INOUT
:
15531 case OMP_CLAUSE_DEPEND_SOURCE
:
15532 case OMP_CLAUSE_DEPEND_SINK
:
15535 gcc_unreachable ();
15537 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
15538 tree array
= create_tmp_var (type
);
15539 TREE_ADDRESSABLE (array
) = 1;
15540 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
15542 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
15543 gimple_seq_add_stmt (iseq
, g
);
15544 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
15546 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
15547 gimple_seq_add_stmt (iseq
, g
);
15548 for (i
= 0; i
< 2; i
++)
15550 if ((i
? n_in
: n_out
) == 0)
15552 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15553 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
15554 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
15556 tree t
= OMP_CLAUSE_DECL (c
);
15557 t
= fold_convert (ptr_type_node
, t
);
15558 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
15559 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
15560 NULL_TREE
, NULL_TREE
);
15561 g
= gimple_build_assign (r
, t
);
15562 gimple_seq_add_stmt (iseq
, g
);
15565 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
15566 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
15567 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
15569 tree clobber
= build_constructor (type
, NULL
);
15570 TREE_THIS_VOLATILE (clobber
) = 1;
15571 g
= gimple_build_assign (array
, clobber
);
15572 gimple_seq_add_stmt (oseq
, g
);
15575 /* Lower the OpenMP parallel or task directive in the current statement
15576 in GSI_P. CTX holds context information for the directive. */
15579 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15583 gimple
*stmt
= gsi_stmt (*gsi_p
);
15584 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
15585 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
15586 location_t loc
= gimple_location (stmt
);
15588 clauses
= gimple_omp_taskreg_clauses (stmt
);
15590 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
15591 par_body
= gimple_bind_body (par_bind
);
15592 child_fn
= ctx
->cb
.dst_fn
;
15593 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15594 && !gimple_omp_parallel_combined_p (stmt
))
15596 struct walk_stmt_info wi
;
15599 memset (&wi
, 0, sizeof (wi
));
15601 wi
.val_only
= true;
15602 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
15604 gimple_omp_parallel_set_combined_p (stmt
, true);
15606 gimple_seq dep_ilist
= NULL
;
15607 gimple_seq dep_olist
= NULL
;
15608 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
15609 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15611 push_gimplify_context ();
15612 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15613 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
15614 &dep_ilist
, &dep_olist
);
15617 if (ctx
->srecord_type
)
15618 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
15620 push_gimplify_context ();
15625 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15626 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
15627 if (phony_construct
&& ctx
->record_type
)
15629 gcc_checking_assert (!ctx
->receiver_decl
);
15630 ctx
->receiver_decl
= create_tmp_var
15631 (build_reference_type (ctx
->record_type
), ".omp_rec");
15633 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
15634 lower_omp (&par_body
, ctx
);
15635 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
15636 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
15638 /* Declare all the variables created by mapping and the variables
15639 declared in the scope of the parallel body. */
15640 record_vars_into (ctx
->block_vars
, child_fn
);
15641 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
15643 if (ctx
->record_type
)
15646 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
15647 : ctx
->record_type
, ".omp_data_o");
15648 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15649 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15650 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
15655 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
15656 lower_send_shared_vars (&ilist
, &olist
, ctx
);
15658 if (ctx
->record_type
)
15660 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
15661 TREE_THIS_VOLATILE (clobber
) = 1;
15662 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15666 /* Once all the expansions are done, sequence all the different
15667 fragments inside gimple_omp_body. */
15671 if (ctx
->record_type
)
15673 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15674 /* fixup_child_record_type might have changed receiver_decl's type. */
15675 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15676 gimple_seq_add_stmt (&new_body
,
15677 gimple_build_assign (ctx
->receiver_decl
, t
));
15680 gimple_seq_add_seq (&new_body
, par_ilist
);
15681 gimple_seq_add_seq (&new_body
, par_body
);
15682 gimple_seq_add_seq (&new_body
, par_rlist
);
15683 if (ctx
->cancellable
)
15684 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
15685 gimple_seq_add_seq (&new_body
, par_olist
);
15686 new_body
= maybe_catch_exception (new_body
);
15687 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
15688 gimple_seq_add_stmt (&new_body
,
15689 gimple_build_omp_continue (integer_zero_node
,
15690 integer_zero_node
));
15691 if (!phony_construct
)
15693 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15694 gimple_omp_set_body (stmt
, new_body
);
15697 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
15698 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15699 gimple_bind_add_seq (bind
, ilist
);
15700 if (!phony_construct
)
15701 gimple_bind_add_stmt (bind
, stmt
);
15703 gimple_bind_add_seq (bind
, new_body
);
15704 gimple_bind_add_seq (bind
, olist
);
15706 pop_gimplify_context (NULL
);
15710 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15711 gimple_bind_add_stmt (dep_bind
, bind
);
15712 gimple_bind_add_seq (dep_bind
, dep_olist
);
15713 pop_gimplify_context (dep_bind
);
15717 /* Lower the GIMPLE_OMP_TARGET in the current statement
15718 in GSI_P. CTX holds context information for the directive. */
15721 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15724 tree child_fn
, t
, c
;
15725 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
15726 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
15727 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
15728 location_t loc
= gimple_location (stmt
);
15729 bool offloaded
, data_region
;
15730 unsigned int map_cnt
= 0;
15732 offloaded
= is_gimple_omp_offloaded (stmt
);
15733 switch (gimple_omp_target_kind (stmt
))
15735 case GF_OMP_TARGET_KIND_REGION
:
15736 case GF_OMP_TARGET_KIND_UPDATE
:
15737 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15738 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15739 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15740 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15741 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15742 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15743 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
15744 data_region
= false;
15746 case GF_OMP_TARGET_KIND_DATA
:
15747 case GF_OMP_TARGET_KIND_OACC_DATA
:
15748 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
15749 data_region
= true;
15752 gcc_unreachable ();
15755 clauses
= gimple_omp_target_clauses (stmt
);
15757 gimple_seq dep_ilist
= NULL
;
15758 gimple_seq dep_olist
= NULL
;
15759 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15761 push_gimplify_context ();
15762 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15763 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15764 &dep_ilist
, &dep_olist
);
15771 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15772 tgt_body
= gimple_bind_body (tgt_bind
);
15774 else if (data_region
)
15775 tgt_body
= gimple_omp_body (stmt
);
15776 child_fn
= ctx
->cb
.dst_fn
;
15778 push_gimplify_context ();
15781 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15782 switch (OMP_CLAUSE_CODE (c
))
15788 case OMP_CLAUSE_MAP
:
15790 /* First check what we're prepared to handle in the following. */
15791 switch (OMP_CLAUSE_MAP_KIND (c
))
15793 case GOMP_MAP_ALLOC
:
15795 case GOMP_MAP_FROM
:
15796 case GOMP_MAP_TOFROM
:
15797 case GOMP_MAP_POINTER
:
15798 case GOMP_MAP_TO_PSET
:
15799 case GOMP_MAP_DELETE
:
15800 case GOMP_MAP_RELEASE
:
15801 case GOMP_MAP_ALWAYS_TO
:
15802 case GOMP_MAP_ALWAYS_FROM
:
15803 case GOMP_MAP_ALWAYS_TOFROM
:
15804 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15805 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
15806 case GOMP_MAP_STRUCT
:
15807 case GOMP_MAP_ALWAYS_POINTER
:
15809 case GOMP_MAP_FORCE_ALLOC
:
15810 case GOMP_MAP_FORCE_TO
:
15811 case GOMP_MAP_FORCE_FROM
:
15812 case GOMP_MAP_FORCE_TOFROM
:
15813 case GOMP_MAP_FORCE_PRESENT
:
15814 case GOMP_MAP_FORCE_DEVICEPTR
:
15815 case GOMP_MAP_DEVICE_RESIDENT
:
15816 case GOMP_MAP_LINK
:
15817 gcc_assert (is_gimple_omp_oacc (stmt
));
15820 gcc_unreachable ();
15824 case OMP_CLAUSE_TO
:
15825 case OMP_CLAUSE_FROM
:
15827 var
= OMP_CLAUSE_DECL (c
);
15830 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15831 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15832 && (OMP_CLAUSE_MAP_KIND (c
)
15833 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15838 if (DECL_SIZE (var
)
15839 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15841 tree var2
= DECL_VALUE_EXPR (var
);
15842 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15843 var2
= TREE_OPERAND (var2
, 0);
15844 gcc_assert (DECL_P (var2
));
15849 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15850 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15851 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15853 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15855 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15856 && varpool_node::get_create (var
)->offloadable
)
15859 tree type
= build_pointer_type (TREE_TYPE (var
));
15860 tree new_var
= lookup_decl (var
, ctx
);
15861 x
= create_tmp_var_raw (type
, get_name (new_var
));
15862 gimple_add_tmp_var (x
);
15863 x
= build_simple_mem_ref (x
);
15864 SET_DECL_VALUE_EXPR (new_var
, x
);
15865 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15870 if (!maybe_lookup_field (var
, ctx
))
15873 /* Don't remap oacc parallel reduction variables, because the
15874 intermediate result must be local to each gang. */
15875 if (offloaded
&& !(OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15876 && OMP_CLAUSE_MAP_IN_REDUCTION (c
)))
15878 x
= build_receiver_ref (var
, true, ctx
);
15879 tree new_var
= lookup_decl (var
, ctx
);
15881 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15882 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15883 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15884 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15885 x
= build_simple_mem_ref (x
);
15886 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15888 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15889 if (is_reference (new_var
))
15891 /* Create a local object to hold the instance
15893 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15894 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15895 tree inst
= create_tmp_var (type
, id
);
15896 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15897 x
= build_fold_addr_expr (inst
);
15899 gimplify_assign (new_var
, x
, &fplist
);
15901 else if (DECL_P (new_var
))
15903 SET_DECL_VALUE_EXPR (new_var
, x
);
15904 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15907 gcc_unreachable ();
15912 case OMP_CLAUSE_FIRSTPRIVATE
:
15913 if (is_oacc_parallel (ctx
))
15914 goto oacc_firstprivate
;
15916 var
= OMP_CLAUSE_DECL (c
);
15917 if (!is_reference (var
)
15918 && !is_gimple_reg_type (TREE_TYPE (var
)))
15920 tree new_var
= lookup_decl (var
, ctx
);
15921 if (is_variable_sized (var
))
15923 tree pvar
= DECL_VALUE_EXPR (var
);
15924 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15925 pvar
= TREE_OPERAND (pvar
, 0);
15926 gcc_assert (DECL_P (pvar
));
15927 tree new_pvar
= lookup_decl (pvar
, ctx
);
15928 x
= build_fold_indirect_ref (new_pvar
);
15929 TREE_THIS_NOTRAP (x
) = 1;
15932 x
= build_receiver_ref (var
, true, ctx
);
15933 SET_DECL_VALUE_EXPR (new_var
, x
);
15934 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15938 case OMP_CLAUSE_PRIVATE
:
15939 if (is_gimple_omp_oacc (ctx
->stmt
))
15941 var
= OMP_CLAUSE_DECL (c
);
15942 if (is_variable_sized (var
))
15944 tree new_var
= lookup_decl (var
, ctx
);
15945 tree pvar
= DECL_VALUE_EXPR (var
);
15946 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15947 pvar
= TREE_OPERAND (pvar
, 0);
15948 gcc_assert (DECL_P (pvar
));
15949 tree new_pvar
= lookup_decl (pvar
, ctx
);
15950 x
= build_fold_indirect_ref (new_pvar
);
15951 TREE_THIS_NOTRAP (x
) = 1;
15952 SET_DECL_VALUE_EXPR (new_var
, x
);
15953 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15957 case OMP_CLAUSE_USE_DEVICE_PTR
:
15958 case OMP_CLAUSE_IS_DEVICE_PTR
:
15959 var
= OMP_CLAUSE_DECL (c
);
15961 if (is_variable_sized (var
))
15963 tree new_var
= lookup_decl (var
, ctx
);
15964 tree pvar
= DECL_VALUE_EXPR (var
);
15965 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15966 pvar
= TREE_OPERAND (pvar
, 0);
15967 gcc_assert (DECL_P (pvar
));
15968 tree new_pvar
= lookup_decl (pvar
, ctx
);
15969 x
= build_fold_indirect_ref (new_pvar
);
15970 TREE_THIS_NOTRAP (x
) = 1;
15971 SET_DECL_VALUE_EXPR (new_var
, x
);
15972 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15974 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15976 tree new_var
= lookup_decl (var
, ctx
);
15977 tree type
= build_pointer_type (TREE_TYPE (var
));
15978 x
= create_tmp_var_raw (type
, get_name (new_var
));
15979 gimple_add_tmp_var (x
);
15980 x
= build_simple_mem_ref (x
);
15981 SET_DECL_VALUE_EXPR (new_var
, x
);
15982 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15986 tree new_var
= lookup_decl (var
, ctx
);
15987 x
= create_tmp_var_raw (TREE_TYPE (new_var
), get_name (new_var
));
15988 gimple_add_tmp_var (x
);
15989 SET_DECL_VALUE_EXPR (new_var
, x
);
15990 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15997 target_nesting_level
++;
15998 lower_omp (&tgt_body
, ctx
);
15999 target_nesting_level
--;
16001 else if (data_region
)
16002 lower_omp (&tgt_body
, ctx
);
16006 /* Declare all the variables created by mapping and the variables
16007 declared in the scope of the target body. */
16008 record_vars_into (ctx
->block_vars
, child_fn
);
16009 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
16014 if (ctx
->record_type
)
16017 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
16018 DECL_NAMELESS (ctx
->sender_decl
) = 1;
16019 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
16020 t
= make_tree_vec (3);
16021 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
16022 TREE_VEC_ELT (t
, 1)
16023 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
16024 ".omp_data_sizes");
16025 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
16026 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
16027 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
16028 tree tkind_type
= short_unsigned_type_node
;
16029 int talign_shift
= 8;
16030 TREE_VEC_ELT (t
, 2)
16031 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
16032 ".omp_data_kinds");
16033 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
16034 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
16035 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
16036 gimple_omp_target_set_data_arg (stmt
, t
);
16038 vec
<constructor_elt
, va_gc
> *vsize
;
16039 vec
<constructor_elt
, va_gc
> *vkind
;
16040 vec_alloc (vsize
, map_cnt
);
16041 vec_alloc (vkind
, map_cnt
);
16042 unsigned int map_idx
= 0;
16044 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16045 switch (OMP_CLAUSE_CODE (c
))
16047 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
16048 unsigned int talign
;
16053 case OMP_CLAUSE_MAP
:
16054 case OMP_CLAUSE_TO
:
16055 case OMP_CLAUSE_FROM
:
16056 oacc_firstprivate_map
:
16058 ovar
= OMP_CLAUSE_DECL (c
);
16059 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16060 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16061 || (OMP_CLAUSE_MAP_KIND (c
)
16062 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16064 if (!DECL_P (ovar
))
16066 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16067 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
16069 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
16070 == get_base_address (ovar
));
16071 nc
= OMP_CLAUSE_CHAIN (c
);
16072 ovar
= OMP_CLAUSE_DECL (nc
);
16076 tree x
= build_sender_ref (ovar
, ctx
);
16078 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
16079 gimplify_assign (x
, v
, &ilist
);
16085 if (DECL_SIZE (ovar
)
16086 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
16088 tree ovar2
= DECL_VALUE_EXPR (ovar
);
16089 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
16090 ovar2
= TREE_OPERAND (ovar2
, 0);
16091 gcc_assert (DECL_P (ovar2
));
16094 if (!maybe_lookup_field (ovar
, ctx
))
16098 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
16099 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
16100 talign
= DECL_ALIGN_UNIT (ovar
);
16103 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16104 x
= build_sender_ref (ovar
, ctx
);
16106 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16107 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
16108 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
16109 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
16111 gcc_assert (offloaded
);
16113 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
16114 mark_addressable (avar
);
16115 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
16116 talign
= DECL_ALIGN_UNIT (avar
);
16117 avar
= build_fold_addr_expr (avar
);
16118 gimplify_assign (x
, avar
, &ilist
);
16120 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16122 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
16123 if (!is_reference (var
))
16125 if (is_gimple_reg (var
)
16126 && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16127 TREE_NO_WARNING (var
) = 1;
16128 var
= build_fold_addr_expr (var
);
16131 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16132 gimplify_assign (x
, var
, &ilist
);
16134 else if (is_gimple_reg (var
))
16136 gcc_assert (offloaded
);
16137 tree avar
= create_tmp_var (TREE_TYPE (var
));
16138 mark_addressable (avar
);
16139 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
16140 if (GOMP_MAP_COPY_TO_P (map_kind
)
16141 || map_kind
== GOMP_MAP_POINTER
16142 || map_kind
== GOMP_MAP_TO_PSET
16143 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16145 /* If we need to initialize a temporary
16146 with VAR because it is not addressable, and
16147 the variable hasn't been initialized yet, then
16148 we'll get a warning for the store to avar.
16149 Don't warn in that case, the mapping might
16151 TREE_NO_WARNING (var
) = 1;
16152 gimplify_assign (avar
, var
, &ilist
);
16154 avar
= build_fold_addr_expr (avar
);
16155 gimplify_assign (x
, avar
, &ilist
);
16156 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
16157 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16158 && !TYPE_READONLY (TREE_TYPE (var
)))
16160 x
= unshare_expr (x
);
16161 x
= build_simple_mem_ref (x
);
16162 gimplify_assign (var
, x
, &olist
);
16167 var
= build_fold_addr_expr (var
);
16168 gimplify_assign (x
, var
, &ilist
);
16172 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16174 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16175 s
= TREE_TYPE (ovar
);
16176 if (TREE_CODE (s
) == REFERENCE_TYPE
)
16178 s
= TYPE_SIZE_UNIT (s
);
16181 s
= OMP_CLAUSE_SIZE (c
);
16182 if (s
== NULL_TREE
)
16183 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16184 s
= fold_convert (size_type_node
, s
);
16185 purpose
= size_int (map_idx
++);
16186 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16187 if (TREE_CODE (s
) != INTEGER_CST
)
16188 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16190 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
16191 switch (OMP_CLAUSE_CODE (c
))
16193 case OMP_CLAUSE_MAP
:
16194 tkind
= OMP_CLAUSE_MAP_KIND (c
);
16195 tkind_zero
= tkind
;
16196 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
16199 case GOMP_MAP_ALLOC
:
16201 case GOMP_MAP_FROM
:
16202 case GOMP_MAP_TOFROM
:
16203 case GOMP_MAP_ALWAYS_TO
:
16204 case GOMP_MAP_ALWAYS_FROM
:
16205 case GOMP_MAP_ALWAYS_TOFROM
:
16206 case GOMP_MAP_RELEASE
:
16207 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
16209 case GOMP_MAP_DELETE
:
16210 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
16214 if (tkind_zero
!= tkind
)
16216 if (integer_zerop (s
))
16217 tkind
= tkind_zero
;
16218 else if (integer_nonzerop (s
))
16219 tkind_zero
= tkind
;
16222 case OMP_CLAUSE_FIRSTPRIVATE
:
16223 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16224 tkind
= GOMP_MAP_TO
;
16225 tkind_zero
= tkind
;
16227 case OMP_CLAUSE_TO
:
16228 tkind
= GOMP_MAP_TO
;
16229 tkind_zero
= tkind
;
16231 case OMP_CLAUSE_FROM
:
16232 tkind
= GOMP_MAP_FROM
;
16233 tkind_zero
= tkind
;
16236 gcc_unreachable ();
16238 gcc_checking_assert (tkind
16239 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16240 gcc_checking_assert (tkind_zero
16241 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16242 talign
= ceil_log2 (talign
);
16243 tkind
|= talign
<< talign_shift
;
16244 tkind_zero
|= talign
<< talign_shift
;
16245 gcc_checking_assert (tkind
16246 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16247 gcc_checking_assert (tkind_zero
16248 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16249 if (tkind
== tkind_zero
)
16250 x
= build_int_cstu (tkind_type
, tkind
);
16253 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
16254 x
= build3 (COND_EXPR
, tkind_type
,
16255 fold_build2 (EQ_EXPR
, boolean_type_node
,
16256 unshare_expr (s
), size_zero_node
),
16257 build_int_cstu (tkind_type
, tkind_zero
),
16258 build_int_cstu (tkind_type
, tkind
));
16260 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
16265 case OMP_CLAUSE_FIRSTPRIVATE
:
16266 if (is_oacc_parallel (ctx
))
16267 goto oacc_firstprivate_map
;
16268 ovar
= OMP_CLAUSE_DECL (c
);
16269 if (is_reference (ovar
))
16270 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16272 talign
= DECL_ALIGN_UNIT (ovar
);
16273 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16274 x
= build_sender_ref (ovar
, ctx
);
16275 tkind
= GOMP_MAP_FIRSTPRIVATE
;
16276 type
= TREE_TYPE (ovar
);
16277 if (is_reference (ovar
))
16278 type
= TREE_TYPE (type
);
16279 if ((INTEGRAL_TYPE_P (type
)
16280 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16281 || TREE_CODE (type
) == POINTER_TYPE
)
16283 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16285 if (is_reference (var
))
16286 t
= build_simple_mem_ref (var
);
16287 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16288 TREE_NO_WARNING (var
) = 1;
16289 if (TREE_CODE (type
) != POINTER_TYPE
)
16290 t
= fold_convert (pointer_sized_int_node
, t
);
16291 t
= fold_convert (TREE_TYPE (x
), t
);
16292 gimplify_assign (x
, t
, &ilist
);
16294 else if (is_reference (var
))
16295 gimplify_assign (x
, var
, &ilist
);
16296 else if (is_gimple_reg (var
))
16298 tree avar
= create_tmp_var (TREE_TYPE (var
));
16299 mark_addressable (avar
);
16300 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16301 TREE_NO_WARNING (var
) = 1;
16302 gimplify_assign (avar
, var
, &ilist
);
16303 avar
= build_fold_addr_expr (avar
);
16304 gimplify_assign (x
, avar
, &ilist
);
16308 var
= build_fold_addr_expr (var
);
16309 gimplify_assign (x
, var
, &ilist
);
16311 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
16313 else if (is_reference (var
))
16314 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16316 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16317 s
= fold_convert (size_type_node
, s
);
16318 purpose
= size_int (map_idx
++);
16319 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16320 if (TREE_CODE (s
) != INTEGER_CST
)
16321 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16323 gcc_checking_assert (tkind
16324 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16325 talign
= ceil_log2 (talign
);
16326 tkind
|= talign
<< talign_shift
;
16327 gcc_checking_assert (tkind
16328 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16329 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16330 build_int_cstu (tkind_type
, tkind
));
16333 case OMP_CLAUSE_USE_DEVICE_PTR
:
16334 case OMP_CLAUSE_IS_DEVICE_PTR
:
16335 ovar
= OMP_CLAUSE_DECL (c
);
16336 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16337 x
= build_sender_ref (ovar
, ctx
);
16338 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16339 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
16341 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16342 type
= TREE_TYPE (ovar
);
16343 if (TREE_CODE (type
) == ARRAY_TYPE
)
16344 var
= build_fold_addr_expr (var
);
16347 if (is_reference (ovar
))
16349 type
= TREE_TYPE (type
);
16350 if (TREE_CODE (type
) != ARRAY_TYPE
)
16351 var
= build_simple_mem_ref (var
);
16352 var
= fold_convert (TREE_TYPE (x
), var
);
16355 gimplify_assign (x
, var
, &ilist
);
16357 purpose
= size_int (map_idx
++);
16358 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16359 gcc_checking_assert (tkind
16360 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16361 gcc_checking_assert (tkind
16362 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16363 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16364 build_int_cstu (tkind_type
, tkind
));
16368 gcc_assert (map_idx
== map_cnt
);
16370 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
16371 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
16372 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
16373 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
16374 for (int i
= 1; i
<= 2; i
++)
16375 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
16377 gimple_seq initlist
= NULL
;
16378 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
16379 TREE_VEC_ELT (t
, i
)),
16380 &initlist
, true, NULL_TREE
);
16381 gimple_seq_add_seq (&ilist
, initlist
);
16383 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
16385 TREE_THIS_VOLATILE (clobber
) = 1;
16386 gimple_seq_add_stmt (&olist
,
16387 gimple_build_assign (TREE_VEC_ELT (t
, i
),
16391 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
16392 TREE_THIS_VOLATILE (clobber
) = 1;
16393 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
16397 /* Once all the expansions are done, sequence all the different
16398 fragments inside gimple_omp_body. */
16403 && ctx
->record_type
)
16405 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
16406 /* fixup_child_record_type might have changed receiver_decl's type. */
16407 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
16408 gimple_seq_add_stmt (&new_body
,
16409 gimple_build_assign (ctx
->receiver_decl
, t
));
16411 gimple_seq_add_seq (&new_body
, fplist
);
16413 if (offloaded
|| data_region
)
16415 tree prev
= NULL_TREE
;
16416 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16417 switch (OMP_CLAUSE_CODE (c
))
16422 case OMP_CLAUSE_FIRSTPRIVATE
:
16423 if (is_gimple_omp_oacc (ctx
->stmt
))
16425 var
= OMP_CLAUSE_DECL (c
);
16426 if (is_reference (var
)
16427 || is_gimple_reg_type (TREE_TYPE (var
)))
16429 tree new_var
= lookup_decl (var
, ctx
);
16431 type
= TREE_TYPE (var
);
16432 if (is_reference (var
))
16433 type
= TREE_TYPE (type
);
16434 if ((INTEGRAL_TYPE_P (type
)
16435 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16436 || TREE_CODE (type
) == POINTER_TYPE
)
16438 x
= build_receiver_ref (var
, false, ctx
);
16439 if (TREE_CODE (type
) != POINTER_TYPE
)
16440 x
= fold_convert (pointer_sized_int_node
, x
);
16441 x
= fold_convert (type
, x
);
16442 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16444 if (is_reference (var
))
16446 tree v
= create_tmp_var_raw (type
, get_name (var
));
16447 gimple_add_tmp_var (v
);
16448 TREE_ADDRESSABLE (v
) = 1;
16449 gimple_seq_add_stmt (&new_body
,
16450 gimple_build_assign (v
, x
));
16451 x
= build_fold_addr_expr (v
);
16453 gimple_seq_add_stmt (&new_body
,
16454 gimple_build_assign (new_var
, x
));
16458 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
16459 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16461 gimple_seq_add_stmt (&new_body
,
16462 gimple_build_assign (new_var
, x
));
16465 else if (is_variable_sized (var
))
16467 tree pvar
= DECL_VALUE_EXPR (var
);
16468 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16469 pvar
= TREE_OPERAND (pvar
, 0);
16470 gcc_assert (DECL_P (pvar
));
16471 tree new_var
= lookup_decl (pvar
, ctx
);
16472 x
= build_receiver_ref (var
, false, ctx
);
16473 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16474 gimple_seq_add_stmt (&new_body
,
16475 gimple_build_assign (new_var
, x
));
16478 case OMP_CLAUSE_PRIVATE
:
16479 if (is_gimple_omp_oacc (ctx
->stmt
))
16481 var
= OMP_CLAUSE_DECL (c
);
16482 if (is_reference (var
))
16484 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16485 tree new_var
= lookup_decl (var
, ctx
);
16486 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16487 if (TREE_CONSTANT (x
))
16489 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
16491 gimple_add_tmp_var (x
);
16492 TREE_ADDRESSABLE (x
) = 1;
16493 x
= build_fold_addr_expr_loc (clause_loc
, x
);
16498 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16499 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16500 gimple_seq_add_stmt (&new_body
,
16501 gimple_build_assign (new_var
, x
));
16504 case OMP_CLAUSE_USE_DEVICE_PTR
:
16505 case OMP_CLAUSE_IS_DEVICE_PTR
:
16506 var
= OMP_CLAUSE_DECL (c
);
16507 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16508 x
= build_sender_ref (var
, ctx
);
16510 x
= build_receiver_ref (var
, false, ctx
);
16511 if (is_variable_sized (var
))
16513 tree pvar
= DECL_VALUE_EXPR (var
);
16514 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16515 pvar
= TREE_OPERAND (pvar
, 0);
16516 gcc_assert (DECL_P (pvar
));
16517 tree new_var
= lookup_decl (pvar
, ctx
);
16518 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16519 gimple_seq_add_stmt (&new_body
,
16520 gimple_build_assign (new_var
, x
));
16522 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16524 tree new_var
= lookup_decl (var
, ctx
);
16525 new_var
= DECL_VALUE_EXPR (new_var
);
16526 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
16527 new_var
= TREE_OPERAND (new_var
, 0);
16528 gcc_assert (DECL_P (new_var
));
16529 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16530 gimple_seq_add_stmt (&new_body
,
16531 gimple_build_assign (new_var
, x
));
16535 tree type
= TREE_TYPE (var
);
16536 tree new_var
= lookup_decl (var
, ctx
);
16537 if (is_reference (var
))
16539 type
= TREE_TYPE (type
);
16540 if (TREE_CODE (type
) != ARRAY_TYPE
)
16542 tree v
= create_tmp_var_raw (type
, get_name (var
));
16543 gimple_add_tmp_var (v
);
16544 TREE_ADDRESSABLE (v
) = 1;
16545 x
= fold_convert (type
, x
);
16546 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16548 gimple_seq_add_stmt (&new_body
,
16549 gimple_build_assign (v
, x
));
16550 x
= build_fold_addr_expr (v
);
16553 new_var
= DECL_VALUE_EXPR (new_var
);
16554 x
= fold_convert (TREE_TYPE (new_var
), x
);
16555 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16556 gimple_seq_add_stmt (&new_body
,
16557 gimple_build_assign (new_var
, x
));
16561 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16562 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16563 are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
16564 or references to VLAs. */
16565 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16566 switch (OMP_CLAUSE_CODE (c
))
16571 case OMP_CLAUSE_MAP
:
16572 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16573 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16575 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16576 HOST_WIDE_INT offset
= 0;
16578 var
= OMP_CLAUSE_DECL (c
);
16580 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
16581 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
16583 && varpool_node::get_create (var
)->offloadable
)
16585 if (TREE_CODE (var
) == INDIRECT_REF
16586 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
16587 var
= TREE_OPERAND (var
, 0);
16588 if (TREE_CODE (var
) == COMPONENT_REF
)
16590 var
= get_addr_base_and_unit_offset (var
, &offset
);
16591 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
16593 else if (DECL_SIZE (var
)
16594 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
16596 tree var2
= DECL_VALUE_EXPR (var
);
16597 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
16598 var2
= TREE_OPERAND (var2
, 0);
16599 gcc_assert (DECL_P (var2
));
16602 tree new_var
= lookup_decl (var
, ctx
), x
;
16603 tree type
= TREE_TYPE (new_var
);
16605 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
16606 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
16609 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
16611 new_var
= build2 (MEM_REF
, type
,
16612 build_fold_addr_expr (new_var
),
16613 build_int_cst (build_pointer_type (type
),
16616 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
16618 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
16619 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
16620 new_var
= build2 (MEM_REF
, type
,
16621 build_fold_addr_expr (new_var
),
16622 build_int_cst (build_pointer_type (type
),
16626 is_ref
= is_reference (var
);
16627 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16629 bool ref_to_array
= false;
16632 type
= TREE_TYPE (type
);
16633 if (TREE_CODE (type
) == ARRAY_TYPE
)
16635 type
= build_pointer_type (type
);
16636 ref_to_array
= true;
16639 else if (TREE_CODE (type
) == ARRAY_TYPE
)
16641 tree decl2
= DECL_VALUE_EXPR (new_var
);
16642 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
16643 decl2
= TREE_OPERAND (decl2
, 0);
16644 gcc_assert (DECL_P (decl2
));
16646 type
= TREE_TYPE (new_var
);
16648 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
16649 x
= fold_convert_loc (clause_loc
, type
, x
);
16650 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
16652 tree bias
= OMP_CLAUSE_SIZE (c
);
16654 bias
= lookup_decl (bias
, ctx
);
16655 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
16656 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
16658 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
16659 TREE_TYPE (x
), x
, bias
);
16662 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16663 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16664 if (is_ref
&& !ref_to_array
)
16666 tree t
= create_tmp_var_raw (type
, get_name (var
));
16667 gimple_add_tmp_var (t
);
16668 TREE_ADDRESSABLE (t
) = 1;
16669 gimple_seq_add_stmt (&new_body
,
16670 gimple_build_assign (t
, x
));
16671 x
= build_fold_addr_expr_loc (clause_loc
, t
);
16673 gimple_seq_add_stmt (&new_body
,
16674 gimple_build_assign (new_var
, x
));
16677 else if (OMP_CLAUSE_CHAIN (c
)
16678 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
16680 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16681 == GOMP_MAP_FIRSTPRIVATE_POINTER
16682 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16683 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16686 case OMP_CLAUSE_PRIVATE
:
16687 var
= OMP_CLAUSE_DECL (c
);
16688 if (is_variable_sized (var
))
16690 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16691 tree new_var
= lookup_decl (var
, ctx
);
16692 tree pvar
= DECL_VALUE_EXPR (var
);
16693 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16694 pvar
= TREE_OPERAND (pvar
, 0);
16695 gcc_assert (DECL_P (pvar
));
16696 tree new_pvar
= lookup_decl (pvar
, ctx
);
16697 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16698 tree al
= size_int (DECL_ALIGN (var
));
16699 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
16700 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16701 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
16702 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16703 gimple_seq_add_stmt (&new_body
,
16704 gimple_build_assign (new_pvar
, x
));
16706 else if (is_reference (var
) && !is_gimple_omp_oacc (ctx
->stmt
))
16708 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16709 tree new_var
= lookup_decl (var
, ctx
);
16710 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16711 if (TREE_CONSTANT (x
))
16716 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16717 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
16718 tree al
= size_int (TYPE_ALIGN (rtype
));
16719 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16722 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16723 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16724 gimple_seq_add_stmt (&new_body
,
16725 gimple_build_assign (new_var
, x
));
16730 gimple_seq fork_seq
= NULL
;
16731 gimple_seq join_seq
= NULL
;
16733 if (is_oacc_parallel (ctx
))
16735 /* If there are reductions on the offloaded region itself, treat
16736 them as a dummy GANG loop. */
16737 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
16739 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
16740 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
16743 gimple_seq_add_seq (&new_body
, fork_seq
);
16744 gimple_seq_add_seq (&new_body
, tgt_body
);
16745 gimple_seq_add_seq (&new_body
, join_seq
);
16748 new_body
= maybe_catch_exception (new_body
);
16750 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
16751 gimple_omp_set_body (stmt
, new_body
);
16754 bind
= gimple_build_bind (NULL
, NULL
,
16755 tgt_bind
? gimple_bind_block (tgt_bind
)
16757 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
16758 gimple_bind_add_seq (bind
, ilist
);
16759 gimple_bind_add_stmt (bind
, stmt
);
16760 gimple_bind_add_seq (bind
, olist
);
16762 pop_gimplify_context (NULL
);
16766 gimple_bind_add_seq (dep_bind
, dep_ilist
);
16767 gimple_bind_add_stmt (dep_bind
, bind
);
16768 gimple_bind_add_seq (dep_bind
, dep_olist
);
16769 pop_gimplify_context (dep_bind
);
16773 /* Expand code for an OpenMP teams directive. */
16776 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16778 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
16779 push_gimplify_context ();
16781 tree block
= make_node (BLOCK
);
16782 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
16783 gsi_replace (gsi_p
, bind
, true);
16784 gimple_seq bind_body
= NULL
;
16785 gimple_seq dlist
= NULL
;
16786 gimple_seq olist
= NULL
;
16788 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16789 OMP_CLAUSE_NUM_TEAMS
);
16790 if (num_teams
== NULL_TREE
)
16791 num_teams
= build_int_cst (unsigned_type_node
, 0);
16794 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16795 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16796 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16798 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16799 OMP_CLAUSE_THREAD_LIMIT
);
16800 if (thread_limit
== NULL_TREE
)
16801 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16804 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16805 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16806 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16810 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16811 &bind_body
, &dlist
, ctx
, NULL
);
16812 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16813 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16814 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16816 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16817 location_t loc
= gimple_location (teams_stmt
);
16818 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16819 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16820 gimple_set_location (call
, loc
);
16821 gimple_seq_add_stmt (&bind_body
, call
);
16824 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16825 gimple_omp_set_body (teams_stmt
, NULL
);
16826 gimple_seq_add_seq (&bind_body
, olist
);
16827 gimple_seq_add_seq (&bind_body
, dlist
);
16828 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16829 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16830 gimple_bind_set_body (bind
, bind_body
);
16832 pop_gimplify_context (bind
);
16834 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16835 BLOCK_VARS (block
) = ctx
->block_vars
;
16836 if (BLOCK_VARS (block
))
16837 TREE_USED (block
) = 1;
16840 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16843 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16845 gimple
*stmt
= gsi_stmt (*gsi_p
);
16846 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
16847 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
16848 gimple_build_omp_return (false));
16852 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16853 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16854 of OMP context, but with task_shared_vars set. */
16857 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16862 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16863 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16866 if (task_shared_vars
16868 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16871 /* If a global variable has been privatized, TREE_CONSTANT on
16872 ADDR_EXPR might be wrong. */
16873 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16874 recompute_tree_invariant_for_addr_expr (t
);
16876 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16880 /* Data to be communicated between lower_omp_regimplify_operands and
16881 lower_omp_regimplify_operands_p. */
16883 struct lower_omp_regimplify_operands_data
16889 /* Helper function for lower_omp_regimplify_operands. Find
16890 omp_member_access_dummy_var vars and adjust temporarily their
16891 DECL_VALUE_EXPRs if needed. */
16894 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16897 tree t
= omp_member_access_dummy_var (*tp
);
16900 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16901 lower_omp_regimplify_operands_data
*ldata
16902 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16903 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16906 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16907 ldata
->decls
->safe_push (*tp
);
16908 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16909 SET_DECL_VALUE_EXPR (*tp
, v
);
16912 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16916 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16917 of omp_member_access_dummy_var vars during regimplification. */
16920 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16921 gimple_stmt_iterator
*gsi_p
)
16923 auto_vec
<tree
, 10> decls
;
16926 struct walk_stmt_info wi
;
16927 memset (&wi
, '\0', sizeof (wi
));
16928 struct lower_omp_regimplify_operands_data data
;
16930 data
.decls
= &decls
;
16932 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16934 gimple_regimplify_operands (stmt
, gsi_p
);
16935 while (!decls
.is_empty ())
16937 tree t
= decls
.pop ();
16938 tree v
= decls
.pop ();
16939 SET_DECL_VALUE_EXPR (t
, v
);
16944 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16946 gimple
*stmt
= gsi_stmt (*gsi_p
);
16947 struct walk_stmt_info wi
;
16950 if (gimple_has_location (stmt
))
16951 input_location
= gimple_location (stmt
);
16953 if (task_shared_vars
)
16954 memset (&wi
, '\0', sizeof (wi
));
16956 /* If we have issued syntax errors, avoid doing any heavy lifting.
16957 Just replace the OMP directives with a NOP to avoid
16958 confusing RTL expansion. */
16959 if (seen_error () && is_gimple_omp (stmt
))
16961 gsi_replace (gsi_p
, gimple_build_nop (), true);
16965 switch (gimple_code (stmt
))
16969 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16970 if ((ctx
|| task_shared_vars
)
16971 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16972 lower_omp_regimplify_p
,
16973 ctx
? NULL
: &wi
, NULL
)
16974 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16975 lower_omp_regimplify_p
,
16976 ctx
? NULL
: &wi
, NULL
)))
16977 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16981 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16983 case GIMPLE_EH_FILTER
:
16984 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16987 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16988 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16990 case GIMPLE_TRANSACTION
:
16991 lower_omp (gimple_transaction_body_ptr (
16992 as_a
<gtransaction
*> (stmt
)),
16996 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16998 case GIMPLE_OMP_PARALLEL
:
16999 case GIMPLE_OMP_TASK
:
17000 ctx
= maybe_lookup_ctx (stmt
);
17002 if (ctx
->cancellable
)
17003 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17004 lower_omp_taskreg (gsi_p
, ctx
);
17006 case GIMPLE_OMP_FOR
:
17007 ctx
= maybe_lookup_ctx (stmt
);
17009 if (ctx
->cancellable
)
17010 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17011 lower_omp_for (gsi_p
, ctx
);
17013 case GIMPLE_OMP_SECTIONS
:
17014 ctx
= maybe_lookup_ctx (stmt
);
17016 if (ctx
->cancellable
)
17017 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17018 lower_omp_sections (gsi_p
, ctx
);
17020 case GIMPLE_OMP_SINGLE
:
17021 ctx
= maybe_lookup_ctx (stmt
);
17023 lower_omp_single (gsi_p
, ctx
);
17025 case GIMPLE_OMP_MASTER
:
17026 ctx
= maybe_lookup_ctx (stmt
);
17028 lower_omp_master (gsi_p
, ctx
);
17030 case GIMPLE_OMP_TASKGROUP
:
17031 ctx
= maybe_lookup_ctx (stmt
);
17033 lower_omp_taskgroup (gsi_p
, ctx
);
17035 case GIMPLE_OMP_ORDERED
:
17036 ctx
= maybe_lookup_ctx (stmt
);
17038 lower_omp_ordered (gsi_p
, ctx
);
17040 case GIMPLE_OMP_CRITICAL
:
17041 ctx
= maybe_lookup_ctx (stmt
);
17043 lower_omp_critical (gsi_p
, ctx
);
17045 case GIMPLE_OMP_ATOMIC_LOAD
:
17046 if ((ctx
|| task_shared_vars
)
17047 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
17048 as_a
<gomp_atomic_load
*> (stmt
)),
17049 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
17050 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17052 case GIMPLE_OMP_TARGET
:
17053 ctx
= maybe_lookup_ctx (stmt
);
17055 lower_omp_target (gsi_p
, ctx
);
17057 case GIMPLE_OMP_TEAMS
:
17058 ctx
= maybe_lookup_ctx (stmt
);
17060 lower_omp_teams (gsi_p
, ctx
);
17062 case GIMPLE_OMP_GRID_BODY
:
17063 ctx
= maybe_lookup_ctx (stmt
);
17065 lower_omp_grid_body (gsi_p
, ctx
);
17069 call_stmt
= as_a
<gcall
*> (stmt
);
17070 fndecl
= gimple_call_fndecl (call_stmt
);
17072 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
17073 switch (DECL_FUNCTION_CODE (fndecl
))
17075 case BUILT_IN_GOMP_BARRIER
:
17079 case BUILT_IN_GOMP_CANCEL
:
17080 case BUILT_IN_GOMP_CANCELLATION_POINT
:
17083 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
17084 cctx
= cctx
->outer
;
17085 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
17086 if (!cctx
->cancellable
)
17088 if (DECL_FUNCTION_CODE (fndecl
)
17089 == BUILT_IN_GOMP_CANCELLATION_POINT
)
17091 stmt
= gimple_build_nop ();
17092 gsi_replace (gsi_p
, stmt
, false);
17096 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
17098 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
17099 gimple_call_set_fndecl (call_stmt
, fndecl
);
17100 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
17103 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
17104 gimple_call_set_lhs (call_stmt
, lhs
);
17105 tree fallthru_label
;
17106 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
17108 g
= gimple_build_label (fallthru_label
);
17109 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17110 g
= gimple_build_cond (NE_EXPR
, lhs
,
17111 fold_convert (TREE_TYPE (lhs
),
17112 boolean_false_node
),
17113 cctx
->cancel_label
, fallthru_label
);
17114 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17121 if ((ctx
|| task_shared_vars
)
17122 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
17125 /* Just remove clobbers, this should happen only if we have
17126 "privatized" local addressable variables in SIMD regions,
17127 the clobber isn't needed in that case and gimplifying address
17128 of the ARRAY_REF into a pointer and creating MEM_REF based
17129 clobber would create worse code than we get with the clobber
17131 if (gimple_clobber_p (stmt
))
17133 gsi_replace (gsi_p
, gimple_build_nop (), true);
17136 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17143 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
17145 location_t saved_location
= input_location
;
17146 gimple_stmt_iterator gsi
;
17147 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17148 lower_omp_1 (&gsi
, ctx
);
17149 /* During gimplification, we haven't folded statments inside offloading
17150 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17151 if (target_nesting_level
|| taskreg_nesting_level
)
17152 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17154 input_location
= saved_location
;
17157 /* Returen true if STMT is an assignment of a register-type into a local
17161 grid_reg_assignment_to_local_var_p (gimple
*stmt
)
17163 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
17166 tree lhs
= gimple_assign_lhs (assign
);
17167 if (TREE_CODE (lhs
) != VAR_DECL
17168 || !is_gimple_reg_type (TREE_TYPE (lhs
))
17169 || is_global_var (lhs
))
17174 /* Return true if all statements in SEQ are assignments to local register-type
17178 grid_seq_only_contains_local_assignments (gimple_seq seq
)
17183 gimple_stmt_iterator gsi
;
17184 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17185 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi
)))
17190 /* Scan statements in SEQ and call itself recursively on any bind. If during
17191 whole search only assignments to register-type local variables and one
17192 single OMP statement is encountered, return true, otherwise return false.
17193 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17194 are used for dumping a note about a failure. */
17197 grid_find_single_omp_among_assignments_1 (gimple_seq seq
, location_t target_loc
,
17198 const char *name
, gimple
**ret
)
17200 gimple_stmt_iterator gsi
;
17201 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17203 gimple
*stmt
= gsi_stmt (gsi
);
17205 if (grid_reg_assignment_to_local_var_p (stmt
))
17207 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17209 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind
),
17210 target_loc
, name
, ret
))
17213 else if (is_gimple_omp (stmt
))
17217 if (dump_enabled_p ())
17218 dump_printf_loc (MSG_NOTE
, target_loc
,
17219 "Will not turn target construct into a simple "
17220 "GPGPU kernel because %s construct contains "
17221 "multiple OpenMP constructs\n", name
);
17228 if (dump_enabled_p ())
17229 dump_printf_loc (MSG_NOTE
, target_loc
,
17230 "Will not turn target construct into a simple "
17231 "GPGPU kernel because %s construct contains "
17232 "a complex statement\n", name
);
17239 /* Scan statements in SEQ and make sure that it and any binds in it contain
17240 only assignments to local register-type variables and one OMP construct. If
17241 so, return that construct, otherwise return NULL. If dumping is enabled and
17242 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17246 grid_find_single_omp_among_assignments (gimple_seq seq
, location_t target_loc
,
17251 if (dump_enabled_p ())
17252 dump_printf_loc (MSG_NOTE
, target_loc
,
17253 "Will not turn target construct into a simple "
17254 "GPGPU kernel because %s construct has empty "
17260 gimple
*ret
= NULL
;
17261 if (grid_find_single_omp_among_assignments_1 (seq
, target_loc
, name
, &ret
))
17263 if (!ret
&& dump_enabled_p ())
17264 dump_printf_loc (MSG_NOTE
, target_loc
,
17265 "Will not turn target construct into a simple "
17266 "GPGPU kernel because %s construct does not contain"
17267 "any other OpenMP construct\n", name
);
17274 /* Walker function looking for statements there is no point gridifying (and for
17275 noreturn function calls which we cannot do). Return non-NULL if such a
17276 function is found. */
17279 grid_find_ungridifiable_statement (gimple_stmt_iterator
*gsi
,
17280 bool *handled_ops_p
,
17281 struct walk_stmt_info
*wi
)
17283 *handled_ops_p
= false;
17284 gimple
*stmt
= gsi_stmt (*gsi
);
17285 switch (gimple_code (stmt
))
17288 if (gimple_call_noreturn_p (as_a
<gcall
*> (stmt
)))
17290 *handled_ops_p
= true;
17292 return error_mark_node
;
17296 /* We may reduce the following list if we find a way to implement the
17297 clauses, but now there is no point trying further. */
17298 case GIMPLE_OMP_CRITICAL
:
17299 case GIMPLE_OMP_TASKGROUP
:
17300 case GIMPLE_OMP_TASK
:
17301 case GIMPLE_OMP_SECTION
:
17302 case GIMPLE_OMP_SECTIONS
:
17303 case GIMPLE_OMP_SECTIONS_SWITCH
:
17304 case GIMPLE_OMP_TARGET
:
17305 case GIMPLE_OMP_ORDERED
:
17306 *handled_ops_p
= true;
17308 return error_mark_node
;
17310 case GIMPLE_OMP_FOR
:
17311 if ((gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
17312 && gimple_omp_for_combined_into_p (stmt
))
17314 *handled_ops_p
= true;
17316 return error_mark_node
;
17327 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17328 kernel, return true, otherwise return false. In the case of success, also
17329 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17333 grid_target_follows_gridifiable_pattern (gomp_target
*target
, tree
*group_size_p
)
17335 if (gimple_omp_target_kind (target
) != GF_OMP_TARGET_KIND_REGION
)
17338 location_t tloc
= gimple_location (target
);
17340 = grid_find_single_omp_among_assignments (gimple_omp_body (target
),
17344 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17345 tree group_size
= NULL
;
17348 dump_printf_loc (MSG_NOTE
, tloc
,
17349 "Will not turn target construct into a simple "
17350 "GPGPU kernel because it does not have a sole teams "
17351 "construct in it.\n");
17355 tree clauses
= gimple_omp_teams_clauses (teams
);
17358 switch (OMP_CLAUSE_CODE (clauses
))
17360 case OMP_CLAUSE_NUM_TEAMS
:
17361 if (dump_enabled_p ())
17362 dump_printf_loc (MSG_NOTE
, tloc
,
17363 "Will not turn target construct into a "
17364 "gridified GPGPU kernel because we cannot "
17365 "handle num_teams clause of teams "
17369 case OMP_CLAUSE_REDUCTION
:
17370 if (dump_enabled_p ())
17371 dump_printf_loc (MSG_NOTE
, tloc
,
17372 "Will not turn target construct into a "
17373 "gridified GPGPU kernel because a reduction "
17374 "clause is present\n ");
17377 case OMP_CLAUSE_LASTPRIVATE
:
17378 if (dump_enabled_p ())
17379 dump_printf_loc (MSG_NOTE
, tloc
,
17380 "Will not turn target construct into a "
17381 "gridified GPGPU kernel because a lastprivate "
17382 "clause is present\n ");
17385 case OMP_CLAUSE_THREAD_LIMIT
:
17386 group_size
= OMP_CLAUSE_OPERAND (clauses
, 0);
17392 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17395 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (teams
), tloc
,
17399 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17402 dump_printf_loc (MSG_NOTE
, tloc
,
17403 "Will not turn target construct into a simple "
17404 "GPGPU kernel because the teams construct does not have "
17405 "a sole distribute construct in it.\n");
17409 gcc_assert (gimple_omp_for_kind (dist
) == GF_OMP_FOR_KIND_DISTRIBUTE
);
17410 if (!gimple_omp_for_combined_p (dist
))
17412 if (dump_enabled_p ())
17413 dump_printf_loc (MSG_NOTE
, tloc
,
17414 "Will not turn target construct into a gridified GPGPU "
17415 "kernel because we cannot handle a standalone "
17416 "distribute construct\n ");
17419 if (dist
->collapse
> 1)
17421 if (dump_enabled_p ())
17422 dump_printf_loc (MSG_NOTE
, tloc
,
17423 "Will not turn target construct into a gridified GPGPU "
17424 "kernel because the distribute construct contains "
17425 "collapse clause\n");
17428 struct omp_for_data fd
;
17429 extract_omp_for_data (dist
, &fd
, NULL
);
17432 if (group_size
&& !operand_equal_p (group_size
, fd
.chunk_size
, 0))
17434 if (dump_enabled_p ())
17435 dump_printf_loc (MSG_NOTE
, tloc
,
17436 "Will not turn target construct into a "
17437 "gridified GPGPU kernel because the teams "
17438 "thread limit is different from distribute "
17439 "schedule chunk\n");
17442 group_size
= fd
.chunk_size
;
17444 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (dist
), tloc
,
17446 gomp_parallel
*par
;
17447 if (!stmt
|| !(par
= dyn_cast
<gomp_parallel
*> (stmt
)))
17450 clauses
= gimple_omp_parallel_clauses (par
);
17453 switch (OMP_CLAUSE_CODE (clauses
))
17455 case OMP_CLAUSE_NUM_THREADS
:
17456 if (dump_enabled_p ())
17457 dump_printf_loc (MSG_NOTE
, tloc
,
17458 "Will not turn target construct into a gridified"
17459 "GPGPU kernel because there is a num_threads "
17460 "clause of the parallel construct\n");
17463 case OMP_CLAUSE_REDUCTION
:
17464 if (dump_enabled_p ())
17465 dump_printf_loc (MSG_NOTE
, tloc
,
17466 "Will not turn target construct into a "
17467 "gridified GPGPU kernel because a reduction "
17468 "clause is present\n ");
17471 case OMP_CLAUSE_LASTPRIVATE
:
17472 if (dump_enabled_p ())
17473 dump_printf_loc (MSG_NOTE
, tloc
,
17474 "Will not turn target construct into a "
17475 "gridified GPGPU kernel because a lastprivate "
17476 "clause is present\n ");
17482 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17485 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (par
), tloc
,
17488 if (!stmt
|| !(gfor
= dyn_cast
<gomp_for
*> (stmt
)))
17491 if (gimple_omp_for_kind (gfor
) != GF_OMP_FOR_KIND_FOR
)
17493 if (dump_enabled_p ())
17494 dump_printf_loc (MSG_NOTE
, tloc
,
17495 "Will not turn target construct into a gridified GPGPU "
17496 "kernel because the inner loop is not a simple for "
17500 if (gfor
->collapse
> 1)
17502 if (dump_enabled_p ())
17503 dump_printf_loc (MSG_NOTE
, tloc
,
17504 "Will not turn target construct into a gridified GPGPU "
17505 "kernel because the inner loop contains collapse "
17510 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor
)))
17512 if (dump_enabled_p ())
17513 dump_printf_loc (MSG_NOTE
, tloc
,
17514 "Will not turn target construct into a gridified GPGPU "
17515 "kernel because the inner loop pre_body contains"
17516 "a complex instruction\n");
17520 clauses
= gimple_omp_for_clauses (gfor
);
17523 switch (OMP_CLAUSE_CODE (clauses
))
17525 case OMP_CLAUSE_SCHEDULE
:
17526 if (OMP_CLAUSE_SCHEDULE_KIND (clauses
) != OMP_CLAUSE_SCHEDULE_AUTO
)
17528 if (dump_enabled_p ())
17529 dump_printf_loc (MSG_NOTE
, tloc
,
17530 "Will not turn target construct into a "
17531 "gridified GPGPU kernel because the inner "
17532 "loop has a non-automatic scheduling clause\n");
17537 case OMP_CLAUSE_REDUCTION
:
17538 if (dump_enabled_p ())
17539 dump_printf_loc (MSG_NOTE
, tloc
,
17540 "Will not turn target construct into a "
17541 "gridified GPGPU kernel because a reduction "
17542 "clause is present\n ");
17545 case OMP_CLAUSE_LASTPRIVATE
:
17546 if (dump_enabled_p ())
17547 dump_printf_loc (MSG_NOTE
, tloc
,
17548 "Will not turn target construct into a "
17549 "gridified GPGPU kernel because a lastprivate "
17550 "clause is present\n ");
17556 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17559 struct walk_stmt_info wi
;
17560 memset (&wi
, 0, sizeof (wi
));
17561 if (walk_gimple_seq (gimple_omp_body (gfor
),
17562 grid_find_ungridifiable_statement
,
17565 gimple
*bad
= (gimple
*) wi
.info
;
17566 if (dump_enabled_p ())
17568 if (is_gimple_call (bad
))
17569 dump_printf_loc (MSG_NOTE
, tloc
,
17570 "Will not turn target construct into a gridified "
17571 " GPGPU kernel because the inner loop contains "
17572 "call to a noreturn function\n");
17573 if (gimple_code (bad
) == GIMPLE_OMP_FOR
)
17574 dump_printf_loc (MSG_NOTE
, tloc
,
17575 "Will not turn target construct into a gridified "
17576 " GPGPU kernel because the inner loop contains "
17577 "a simd construct\n");
17579 dump_printf_loc (MSG_NOTE
, tloc
,
17580 "Will not turn target construct into a gridified "
17581 "GPGPU kernel because the inner loop contains "
17582 "statement %s which cannot be transformed\n",
17583 gimple_code_name
[(int) gimple_code (bad
)]);
17588 *group_size_p
= group_size
;
17592 /* Operand walker, used to remap pre-body declarations according to a hash map
17593 provided in DATA. */
17596 grid_remap_prebody_decls (tree
*tp
, int *walk_subtrees
, void *data
)
17600 if (DECL_P (t
) || TYPE_P (t
))
17601 *walk_subtrees
= 0;
17603 *walk_subtrees
= 1;
17605 if (TREE_CODE (t
) == VAR_DECL
)
17607 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17608 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17609 tree
*repl
= declmap
->get (t
);
17616 /* Copy leading register-type assignments to local variables in SRC to just
17617 before DST, Creating temporaries, adjusting mapping of operands in WI and
17618 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17619 Return the first statement that does not conform to
17620 grid_reg_assignment_to_local_var_p or NULL. */
17623 grid_copy_leading_local_assignments (gimple_seq src
, gimple_stmt_iterator
*dst
,
17624 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17626 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17627 gimple_stmt_iterator gsi
;
17628 for (gsi
= gsi_start (src
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17630 gimple
*stmt
= gsi_stmt (gsi
);
17631 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17633 gimple
*r
= grid_copy_leading_local_assignments
17634 (gimple_bind_body (bind
), dst
, tgt_bind
, wi
);
17640 if (!grid_reg_assignment_to_local_var_p (stmt
))
17642 tree lhs
= gimple_assign_lhs (as_a
<gassign
*> (stmt
));
17643 tree repl
= copy_var_decl (lhs
, create_tmp_var_name (NULL
),
17645 DECL_CONTEXT (repl
) = current_function_decl
;
17646 gimple_bind_append_vars (tgt_bind
, repl
);
17648 declmap
->put (lhs
, repl
);
17649 gassign
*copy
= as_a
<gassign
*> (gimple_copy (stmt
));
17650 walk_gimple_op (copy
, grid_remap_prebody_decls
, wi
);
17651 gsi_insert_before (dst
, copy
, GSI_SAME_STMT
);
17656 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17657 components, mark them as part of kernel and return the inner loop, and copy
17658 assignment leading to them just before DST, remapping them using WI and
17659 adding new temporaries to TGT_BIND. */
17662 grid_process_kernel_body_copy (gimple_seq seq
, gimple_stmt_iterator
*dst
,
17663 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17665 gimple
*stmt
= grid_copy_leading_local_assignments (seq
, dst
, tgt_bind
, wi
);
17666 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17667 gcc_assert (teams
);
17668 gimple_omp_teams_set_grid_phony (teams
, true);
17669 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (teams
), dst
,
17671 gcc_checking_assert (stmt
);
17672 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17674 gimple_seq prebody
= gimple_omp_for_pre_body (dist
);
17676 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17677 gimple_omp_for_set_grid_phony (dist
, true);
17678 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (dist
), dst
,
17680 gcc_checking_assert (stmt
);
17682 gomp_parallel
*parallel
= as_a
<gomp_parallel
*> (stmt
);
17683 gimple_omp_parallel_set_grid_phony (parallel
, true);
17684 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (parallel
), dst
,
17686 gomp_for
*inner_loop
= as_a
<gomp_for
*> (stmt
);
17687 gimple_omp_for_set_kind (inner_loop
, GF_OMP_FOR_KIND_GRID_LOOP
);
17688 prebody
= gimple_omp_for_pre_body (inner_loop
);
17690 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17695 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17696 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17697 is the bind into which temporaries inserted before TARGET should be
17701 grid_attempt_target_gridification (gomp_target
*target
,
17702 gimple_stmt_iterator
*gsi
,
17706 if (!target
|| !grid_target_follows_gridifiable_pattern (target
, &group_size
))
17709 location_t loc
= gimple_location (target
);
17710 if (dump_enabled_p ())
17711 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
17712 "Target construct will be turned into a gridified GPGPU "
17715 /* Copy target body to a GPUKERNEL construct: */
17716 gimple_seq kernel_seq
= copy_gimple_seq_and_replace_locals
17717 (gimple_omp_body (target
));
17719 hash_map
<tree
, tree
> *declmap
= new hash_map
<tree
, tree
>;
17720 struct walk_stmt_info wi
;
17721 memset (&wi
, 0, sizeof (struct walk_stmt_info
));
17724 /* Copy assignments in between OMP statements before target, mark OMP
17725 statements within copy appropriatly. */
17726 gomp_for
*inner_loop
= grid_process_kernel_body_copy (kernel_seq
, gsi
,
17729 gbind
*old_bind
= as_a
<gbind
*> (gimple_seq_first (gimple_omp_body (target
)));
17730 gbind
*new_bind
= as_a
<gbind
*> (gimple_seq_first (kernel_seq
));
17731 tree new_block
= gimple_bind_block (new_bind
);
17732 tree enc_block
= BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind
));
17733 BLOCK_CHAIN (new_block
) = BLOCK_SUBBLOCKS (enc_block
);
17734 BLOCK_SUBBLOCKS (enc_block
) = new_block
;
17735 BLOCK_SUPERCONTEXT (new_block
) = enc_block
;
17736 gimple
*gpukernel
= gimple_build_omp_grid_body (kernel_seq
);
17737 gimple_seq_add_stmt
17738 (gimple_bind_body_ptr (as_a
<gbind
*> (gimple_omp_body (target
))),
17741 walk_tree (&group_size
, grid_remap_prebody_decls
, &wi
, NULL
);
17742 push_gimplify_context ();
17743 size_t collapse
= gimple_omp_for_collapse (inner_loop
);
17744 for (size_t i
= 0; i
< collapse
; i
++)
17746 tree itype
, type
= TREE_TYPE (gimple_omp_for_index (inner_loop
, i
));
17747 if (POINTER_TYPE_P (type
))
17748 itype
= signed_type_for (type
);
17752 enum tree_code cond_code
= gimple_omp_for_cond (inner_loop
, i
);
17753 tree n1
= unshare_expr (gimple_omp_for_initial (inner_loop
, i
));
17754 walk_tree (&n1
, grid_remap_prebody_decls
, &wi
, NULL
);
17755 tree n2
= unshare_expr (gimple_omp_for_final (inner_loop
, i
));
17756 walk_tree (&n2
, grid_remap_prebody_decls
, &wi
, NULL
);
17757 adjust_for_condition (loc
, &cond_code
, &n2
);
17759 step
= get_omp_for_step_from_incr (loc
,
17760 gimple_omp_for_incr (inner_loop
, i
));
17761 gimple_seq tmpseq
= NULL
;
17762 n1
= fold_convert (itype
, n1
);
17763 n2
= fold_convert (itype
, n2
);
17764 tree t
= build_int_cst (itype
, (cond_code
== LT_EXPR
? -1 : 1));
17765 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
17766 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
17767 t
= fold_build2 (MINUS_EXPR
, itype
, t
, n1
);
17768 if (TYPE_UNSIGNED (itype
) && cond_code
== GT_EXPR
)
17769 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
17770 fold_build1 (NEGATE_EXPR
, itype
, t
),
17771 fold_build1 (NEGATE_EXPR
, itype
, step
));
17773 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
17774 tree gs
= fold_convert (uint32_type_node
, t
);
17775 gimplify_expr (&gs
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17776 if (!gimple_seq_empty_p (tmpseq
))
17777 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17780 if (i
== 0 && group_size
)
17782 ws
= fold_convert (uint32_type_node
, group_size
);
17784 gimplify_expr (&ws
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17785 if (!gimple_seq_empty_p (tmpseq
))
17786 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17789 ws
= build_zero_cst (uint32_type_node
);
17791 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__GRIDDIM_
);
17792 OMP_CLAUSE__GRIDDIM__DIMENSION (c
) = i
;
17793 OMP_CLAUSE__GRIDDIM__SIZE (c
) = gs
;
17794 OMP_CLAUSE__GRIDDIM__GROUP (c
) = ws
;
17795 OMP_CLAUSE_CHAIN (c
) = gimple_omp_target_clauses (target
);
17796 gimple_omp_target_set_clauses (target
, c
);
17798 pop_gimplify_context (tgt_bind
);
17803 /* Walker function doing all the work for create_target_kernels. */
17806 grid_gridify_all_targets_stmt (gimple_stmt_iterator
*gsi
,
17807 bool *handled_ops_p
,
17808 struct walk_stmt_info
*incoming
)
17810 *handled_ops_p
= false;
17812 gimple
*stmt
= gsi_stmt (*gsi
);
17813 gomp_target
*target
= dyn_cast
<gomp_target
*> (stmt
);
17816 gbind
*tgt_bind
= (gbind
*) incoming
->info
;
17817 gcc_checking_assert (tgt_bind
);
17818 grid_attempt_target_gridification (target
, gsi
, tgt_bind
);
17821 gbind
*bind
= dyn_cast
<gbind
*> (stmt
);
17824 *handled_ops_p
= true;
17825 struct walk_stmt_info wi
;
17826 memset (&wi
, 0, sizeof (wi
));
17828 walk_gimple_seq_mod (gimple_bind_body_ptr (bind
),
17829 grid_gridify_all_targets_stmt
, NULL
, &wi
);
17834 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17835 have their bodies duplicated, with the new copy being put into a
17836 gimple_omp_grid_body statement. All kernel-related construct within the
17837 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17838 re-structuring is often needed, such as copying pre-bodies before the target
17839 construct so that kernel grid sizes can be computed. */
17842 grid_gridify_all_targets (gimple_seq
*body_p
)
17844 struct walk_stmt_info wi
;
17845 memset (&wi
, 0, sizeof (wi
));
17846 walk_gimple_seq_mod (body_p
, grid_gridify_all_targets_stmt
, NULL
, &wi
);
17850 /* Main entry point. */
17852 static unsigned int
17853 execute_lower_omp (void)
17859 /* This pass always runs, to provide PROP_gimple_lomp.
17860 But often, there is nothing to do. */
17861 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
17862 && flag_openmp_simd
== 0)
17865 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
17866 delete_omp_context
);
17868 body
= gimple_body (current_function_decl
);
17870 if (hsa_gen_requested_p ())
17871 grid_gridify_all_targets (&body
);
17873 scan_omp (&body
, NULL
);
17874 gcc_assert (taskreg_nesting_level
== 0);
17875 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
17876 finish_taskreg_scan (ctx
);
17877 taskreg_contexts
.release ();
17879 if (all_contexts
->root
)
17881 if (task_shared_vars
)
17882 push_gimplify_context ();
17883 lower_omp (&body
, NULL
);
17884 if (task_shared_vars
)
17885 pop_gimplify_context (NULL
);
17890 splay_tree_delete (all_contexts
);
17891 all_contexts
= NULL
;
17893 BITMAP_FREE (task_shared_vars
);
17899 const pass_data pass_data_lower_omp
=
17901 GIMPLE_PASS
, /* type */
17902 "omplower", /* name */
17903 OPTGROUP_NONE
, /* optinfo_flags */
17904 TV_NONE
, /* tv_id */
17905 PROP_gimple_any
, /* properties_required */
17906 PROP_gimple_lomp
, /* properties_provided */
17907 0, /* properties_destroyed */
17908 0, /* todo_flags_start */
17909 0, /* todo_flags_finish */
17912 class pass_lower_omp
: public gimple_opt_pass
17915 pass_lower_omp (gcc::context
*ctxt
)
17916 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
17919 /* opt_pass methods: */
17920 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
17922 }; // class pass_lower_omp
17924 } // anon namespace
17927 make_pass_lower_omp (gcc::context
*ctxt
)
17929 return new pass_lower_omp (ctxt
);
17932 /* The following is a utility to diagnose structured block violations.
17933 It is not part of the "omplower" pass, as that's invoked too late. It
17934 should be invoked by the respective front ends after gimplification. */
17936 static splay_tree all_labels
;
17938 /* Check for mismatched contexts and generate an error if needed. Return
17939 true if an error is detected. */
17942 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
17943 gimple
*branch_ctx
, gimple
*label_ctx
)
17945 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
17946 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
17948 if (label_ctx
== branch_ctx
)
17951 const char* kind
= NULL
;
17956 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
17957 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
17959 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
17960 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
17961 kind
= "Cilk Plus";
17965 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
17966 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
17968 gcc_checking_assert (kind
== NULL
);
17974 gcc_checking_assert (flag_openmp
);
17979 Previously we kept track of the label's entire context in diagnose_sb_[12]
17980 so we could traverse it and issue a correct "exit" or "enter" error
17981 message upon a structured block violation.
17983 We built the context by building a list with tree_cons'ing, but there is
17984 no easy counterpart in gimple tuples. It seems like far too much work
17985 for issuing exit/enter error messages. If someone really misses the
17986 distinct error message... patches welcome.
17990 /* Try to avoid confusing the user by producing and error message
17991 with correct "exit" or "enter" verbiage. We prefer "exit"
17992 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
17993 if (branch_ctx
== NULL
)
17999 if (TREE_VALUE (label_ctx
) == branch_ctx
)
18004 label_ctx
= TREE_CHAIN (label_ctx
);
18009 error ("invalid exit from %s structured block", kind
);
18011 error ("invalid entry to %s structured block", kind
);
18014 /* If it's obvious we have an invalid entry, be specific about the error. */
18015 if (branch_ctx
== NULL
)
18016 error ("invalid entry to %s structured block", kind
);
18019 /* Otherwise, be vague and lazy, but efficient. */
18020 error ("invalid branch to/from %s structured block", kind
);
18023 gsi_replace (gsi_p
, gimple_build_nop (), false);
18027 /* Pass 1: Create a minimal tree of structured blocks, and record
18028 where each label is found. */
18031 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18032 struct walk_stmt_info
*wi
)
18034 gimple
*context
= (gimple
*) wi
->info
;
18035 gimple
*inner_context
;
18036 gimple
*stmt
= gsi_stmt (*gsi_p
);
18038 *handled_ops_p
= true;
18040 switch (gimple_code (stmt
))
18044 case GIMPLE_OMP_PARALLEL
:
18045 case GIMPLE_OMP_TASK
:
18046 case GIMPLE_OMP_SECTIONS
:
18047 case GIMPLE_OMP_SINGLE
:
18048 case GIMPLE_OMP_SECTION
:
18049 case GIMPLE_OMP_MASTER
:
18050 case GIMPLE_OMP_ORDERED
:
18051 case GIMPLE_OMP_CRITICAL
:
18052 case GIMPLE_OMP_TARGET
:
18053 case GIMPLE_OMP_TEAMS
:
18054 case GIMPLE_OMP_TASKGROUP
:
18055 /* The minimal context here is just the current OMP construct. */
18056 inner_context
= stmt
;
18057 wi
->info
= inner_context
;
18058 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18059 wi
->info
= context
;
18062 case GIMPLE_OMP_FOR
:
18063 inner_context
= stmt
;
18064 wi
->info
= inner_context
;
18065 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18067 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
18068 diagnose_sb_1
, NULL
, wi
);
18069 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18070 wi
->info
= context
;
18074 splay_tree_insert (all_labels
,
18075 (splay_tree_key
) gimple_label_label (
18076 as_a
<glabel
*> (stmt
)),
18077 (splay_tree_value
) context
);
18087 /* Pass 2: Check each branch and see if its context differs from that of
18088 the destination label's context. */
18091 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18092 struct walk_stmt_info
*wi
)
18094 gimple
*context
= (gimple
*) wi
->info
;
18096 gimple
*stmt
= gsi_stmt (*gsi_p
);
18098 *handled_ops_p
= true;
18100 switch (gimple_code (stmt
))
18104 case GIMPLE_OMP_PARALLEL
:
18105 case GIMPLE_OMP_TASK
:
18106 case GIMPLE_OMP_SECTIONS
:
18107 case GIMPLE_OMP_SINGLE
:
18108 case GIMPLE_OMP_SECTION
:
18109 case GIMPLE_OMP_MASTER
:
18110 case GIMPLE_OMP_ORDERED
:
18111 case GIMPLE_OMP_CRITICAL
:
18112 case GIMPLE_OMP_TARGET
:
18113 case GIMPLE_OMP_TEAMS
:
18114 case GIMPLE_OMP_TASKGROUP
:
18116 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18117 wi
->info
= context
;
18120 case GIMPLE_OMP_FOR
:
18122 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18124 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
18125 diagnose_sb_2
, NULL
, wi
);
18126 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18127 wi
->info
= context
;
18132 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
18133 tree lab
= gimple_cond_true_label (cond_stmt
);
18136 n
= splay_tree_lookup (all_labels
,
18137 (splay_tree_key
) lab
);
18138 diagnose_sb_0 (gsi_p
, context
,
18139 n
? (gimple
*) n
->value
: NULL
);
18141 lab
= gimple_cond_false_label (cond_stmt
);
18144 n
= splay_tree_lookup (all_labels
,
18145 (splay_tree_key
) lab
);
18146 diagnose_sb_0 (gsi_p
, context
,
18147 n
? (gimple
*) n
->value
: NULL
);
18154 tree lab
= gimple_goto_dest (stmt
);
18155 if (TREE_CODE (lab
) != LABEL_DECL
)
18158 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18159 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
18163 case GIMPLE_SWITCH
:
18165 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
18167 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
18169 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
18170 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18171 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
18177 case GIMPLE_RETURN
:
18178 diagnose_sb_0 (gsi_p
, context
, NULL
);
18188 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18191 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
18194 gimple
*last
= last_stmt (bb
);
18195 enum gimple_code code
= gimple_code (last
);
18196 struct omp_region
*cur_region
= *region
;
18197 bool fallthru
= false;
18201 case GIMPLE_OMP_PARALLEL
:
18202 case GIMPLE_OMP_TASK
:
18203 case GIMPLE_OMP_FOR
:
18204 case GIMPLE_OMP_SINGLE
:
18205 case GIMPLE_OMP_TEAMS
:
18206 case GIMPLE_OMP_MASTER
:
18207 case GIMPLE_OMP_TASKGROUP
:
18208 case GIMPLE_OMP_CRITICAL
:
18209 case GIMPLE_OMP_SECTION
:
18210 case GIMPLE_OMP_GRID_BODY
:
18211 cur_region
= new_omp_region (bb
, code
, cur_region
);
18215 case GIMPLE_OMP_ORDERED
:
18216 cur_region
= new_omp_region (bb
, code
, cur_region
);
18218 if (find_omp_clause (gimple_omp_ordered_clauses
18219 (as_a
<gomp_ordered
*> (last
)),
18220 OMP_CLAUSE_DEPEND
))
18221 cur_region
= cur_region
->outer
;
18224 case GIMPLE_OMP_TARGET
:
18225 cur_region
= new_omp_region (bb
, code
, cur_region
);
18227 switch (gimple_omp_target_kind (last
))
18229 case GF_OMP_TARGET_KIND_REGION
:
18230 case GF_OMP_TARGET_KIND_DATA
:
18231 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
18232 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
18233 case GF_OMP_TARGET_KIND_OACC_DATA
:
18234 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
18236 case GF_OMP_TARGET_KIND_UPDATE
:
18237 case GF_OMP_TARGET_KIND_ENTER_DATA
:
18238 case GF_OMP_TARGET_KIND_EXIT_DATA
:
18239 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
18240 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
18241 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
18242 cur_region
= cur_region
->outer
;
18245 gcc_unreachable ();
18249 case GIMPLE_OMP_SECTIONS
:
18250 cur_region
= new_omp_region (bb
, code
, cur_region
);
18254 case GIMPLE_OMP_SECTIONS_SWITCH
:
18258 case GIMPLE_OMP_ATOMIC_LOAD
:
18259 case GIMPLE_OMP_ATOMIC_STORE
:
18263 case GIMPLE_OMP_RETURN
:
18264 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18265 somewhere other than the next block. This will be
18267 cur_region
->exit
= bb
;
18268 if (cur_region
->type
== GIMPLE_OMP_TASK
)
18269 /* Add an edge corresponding to not scheduling the task
18271 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
18272 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
18273 cur_region
= cur_region
->outer
;
18276 case GIMPLE_OMP_CONTINUE
:
18277 cur_region
->cont
= bb
;
18278 switch (cur_region
->type
)
18280 case GIMPLE_OMP_FOR
:
18281 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18282 succs edges as abnormal to prevent splitting
18284 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
18285 /* Make the loopback edge. */
18286 make_edge (bb
, single_succ (cur_region
->entry
),
18289 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18290 corresponds to the case that the body of the loop
18291 is not executed at all. */
18292 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
18293 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
18297 case GIMPLE_OMP_SECTIONS
:
18298 /* Wire up the edges into and out of the nested sections. */
18300 basic_block switch_bb
= single_succ (cur_region
->entry
);
18302 struct omp_region
*i
;
18303 for (i
= cur_region
->inner
; i
; i
= i
->next
)
18305 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
18306 make_edge (switch_bb
, i
->entry
, 0);
18307 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
18310 /* Make the loopback edge to the block with
18311 GIMPLE_OMP_SECTIONS_SWITCH. */
18312 make_edge (bb
, switch_bb
, 0);
18314 /* Make the edge from the switch to exit. */
18315 make_edge (switch_bb
, bb
->next_bb
, 0);
18320 case GIMPLE_OMP_TASK
:
18325 gcc_unreachable ();
18330 gcc_unreachable ();
18333 if (*region
!= cur_region
)
18335 *region
= cur_region
;
18337 *region_idx
= cur_region
->entry
->index
;
18345 static unsigned int
18346 diagnose_omp_structured_block_errors (void)
18348 struct walk_stmt_info wi
;
18349 gimple_seq body
= gimple_body (current_function_decl
);
18351 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
18353 memset (&wi
, 0, sizeof (wi
));
18354 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
18356 memset (&wi
, 0, sizeof (wi
));
18357 wi
.want_locations
= true;
18358 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
18360 gimple_set_body (current_function_decl
, body
);
18362 splay_tree_delete (all_labels
);
18370 const pass_data pass_data_diagnose_omp_blocks
=
18372 GIMPLE_PASS
, /* type */
18373 "*diagnose_omp_blocks", /* name */
18374 OPTGROUP_NONE
, /* optinfo_flags */
18375 TV_NONE
, /* tv_id */
18376 PROP_gimple_any
, /* properties_required */
18377 0, /* properties_provided */
18378 0, /* properties_destroyed */
18379 0, /* todo_flags_start */
18380 0, /* todo_flags_finish */
18383 class pass_diagnose_omp_blocks
: public gimple_opt_pass
18386 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18387 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
18390 /* opt_pass methods: */
18391 virtual bool gate (function
*)
18393 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
18395 virtual unsigned int execute (function
*)
18397 return diagnose_omp_structured_block_errors ();
18400 }; // class pass_diagnose_omp_blocks
18402 } // anon namespace
18405 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18407 return new pass_diagnose_omp_blocks (ctxt
);
18410 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18411 adds their addresses and sizes to constructor-vector V_CTOR. */
18413 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18414 vec
<constructor_elt
, va_gc
> *v_ctor
)
18416 unsigned len
= vec_safe_length (v_decls
);
18417 for (unsigned i
= 0; i
< len
; i
++)
18419 tree it
= (*v_decls
)[i
];
18420 bool is_var
= TREE_CODE (it
) == VAR_DECL
;
18423 #ifdef ACCEL_COMPILER
18424 && DECL_HAS_VALUE_EXPR_P (it
)
18426 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
18428 tree size
= NULL_TREE
;
18430 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
18434 addr
= build_fold_addr_expr (it
);
18437 #ifdef ACCEL_COMPILER
18438 /* For "omp declare target link" vars add address of the pointer to
18439 the target table, instead of address of the var. */
18440 tree value_expr
= DECL_VALUE_EXPR (it
);
18441 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
18442 varpool_node::finalize_decl (link_ptr_decl
);
18443 addr
= build_fold_addr_expr (link_ptr_decl
);
18445 addr
= build_fold_addr_expr (it
);
18448 /* Most significant bit of the size marks "omp declare target link"
18449 vars in host and target tables. */
18450 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
18451 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
18452 * BITS_PER_UNIT
- 1);
18453 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
18456 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
18458 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
18462 /* Create new symbols containing (address, size) pairs for global variables,
18463 marked with "omp declare target" attribute, as well as addresses for the
18464 functions, which are outlined offloading regions. */
18466 omp_finish_file (void)
18468 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18469 unsigned num_vars
= vec_safe_length (offload_vars
);
18471 if (num_funcs
== 0 && num_vars
== 0)
18474 if (targetm_common
.have_named_sections
)
18476 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18477 vec_alloc (v_f
, num_funcs
);
18478 vec_alloc (v_v
, num_vars
* 2);
18480 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18481 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18483 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18485 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18487 SET_TYPE_ALIGN (vars_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
18488 SET_TYPE_ALIGN (funcs_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
18489 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18490 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18491 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18492 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18493 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18494 get_identifier (".offload_func_table"),
18496 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18497 get_identifier (".offload_var_table"),
18499 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18500 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18501 otherwise a joint table in a binary will contain padding between
18502 tables from multiple object files. */
18503 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18504 SET_DECL_ALIGN (funcs_decl
, TYPE_ALIGN (funcs_decl_type
));
18505 SET_DECL_ALIGN (vars_decl
, TYPE_ALIGN (vars_decl_type
));
18506 DECL_INITIAL (funcs_decl
) = ctor_f
;
18507 DECL_INITIAL (vars_decl
) = ctor_v
;
18508 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18509 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18511 varpool_node::finalize_decl (vars_decl
);
18512 varpool_node::finalize_decl (funcs_decl
);
18516 for (unsigned i
= 0; i
< num_funcs
; i
++)
18518 tree it
= (*offload_funcs
)[i
];
18519 targetm
.record_offload_symbol (it
);
18521 for (unsigned i
= 0; i
< num_vars
; i
++)
18523 tree it
= (*offload_vars
)[i
];
18524 targetm
.record_offload_symbol (it
);
18529 /* Find the number of threads (POS = false), or thread number (POS =
18530 true) for an OpenACC region partitioned as MASK. Setup code
18531 required for the calculation is added to SEQ. */
18534 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18536 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18539 /* Start at gang level, and examine relevant dimension indices. */
18540 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18541 if (GOMP_DIM_MASK (ix
) & mask
)
18543 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18547 /* We had an outer index, so scale that by the size of
18549 tree n
= create_tmp_var (integer_type_node
);
18551 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18553 gimple_call_set_lhs (call
, n
);
18554 gimple_seq_add_stmt (seq
, call
);
18555 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18559 /* Determine index in this dimension. */
18560 tree id
= create_tmp_var (integer_type_node
);
18561 gimple
*call
= gimple_build_call_internal
18562 (IFN_GOACC_DIM_POS
, 1, arg
);
18564 gimple_call_set_lhs (call
, id
);
18565 gimple_seq_add_stmt (seq
, call
);
18567 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18573 if (res
== NULL_TREE
)
18574 res
= integer_zero_node
;
18579 /* Transform IFN_GOACC_LOOP calls to actual code. See
18580 expand_oacc_for for where these are generated. At the vector
18581 level, we stride loops, such that each member of a warp will
18582 operate on adjacent iterations. At the worker and gang level,
18583 each gang/warp executes a set of contiguous iterations. Chunking
18584 can override this such that each iteration engine executes a
18585 contiguous chunk, and then moves on to stride to the next chunk. */
18588 oacc_xform_loop (gcall
*call
)
18590 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18591 enum ifn_goacc_loop_kind code
18592 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18593 tree dir
= gimple_call_arg (call
, 1);
18594 tree range
= gimple_call_arg (call
, 2);
18595 tree step
= gimple_call_arg (call
, 3);
18596 tree chunk_size
= NULL_TREE
;
18597 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18598 tree lhs
= gimple_call_lhs (call
);
18599 tree type
= TREE_TYPE (lhs
);
18600 tree diff_type
= TREE_TYPE (range
);
18601 tree r
= NULL_TREE
;
18602 gimple_seq seq
= NULL
;
18603 bool chunking
= false, striding
= true;
18604 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18605 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18607 #ifdef ACCEL_COMPILER
18608 chunk_size
= gimple_call_arg (call
, 4);
18609 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18610 || integer_zerop (chunk_size
)) /* Default (also static). */
18612 /* If we're at the gang level, we want each to execute a
18613 contiguous run of iterations. Otherwise we want each element
18615 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18620 /* Chunk of size 1 is striding. */
18621 striding
= integer_onep (chunk_size
);
18622 chunking
= !striding
;
18626 /* striding=true, chunking=true
18628 striding=true, chunking=false
18630 striding=false,chunking=true
18631 -> chunks=ceil (range/(chunksize*threads*step))
18632 striding=false,chunking=false
18633 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18634 push_gimplify_context (true);
18638 default: gcc_unreachable ();
18640 case IFN_GOACC_LOOP_CHUNKS
:
18642 r
= build_int_cst (type
, 1);
18646 = (range - dir) / (chunks * step * num_threads) + dir */
18647 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18648 per
= fold_convert (type
, per
);
18649 chunk_size
= fold_convert (type
, chunk_size
);
18650 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18651 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18652 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18653 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18654 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18658 case IFN_GOACC_LOOP_STEP
:
18660 /* If striding, step by the entire compute volume, otherwise
18661 step by the inner volume. */
18662 unsigned volume
= striding
? mask
: inner_mask
;
18664 r
= oacc_thread_numbers (false, volume
, &seq
);
18665 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18669 case IFN_GOACC_LOOP_OFFSET
:
18672 r
= oacc_thread_numbers (true, mask
, &seq
);
18673 r
= fold_convert (diff_type
, r
);
18677 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18678 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18679 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18680 inner_size
, outer_size
);
18682 volume
= fold_convert (diff_type
, volume
);
18684 chunk_size
= fold_convert (diff_type
, chunk_size
);
18687 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18689 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18690 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18691 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18694 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18695 fold_convert (diff_type
, inner_size
));
18696 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18697 r
= fold_convert (diff_type
, r
);
18698 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18700 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18701 inner
= fold_convert (diff_type
, inner
);
18702 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18706 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18708 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18709 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18711 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18714 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18715 if (type
!= diff_type
)
18716 r
= fold_convert (type
, r
);
18719 case IFN_GOACC_LOOP_BOUND
:
18724 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18725 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18726 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18727 inner_size
, outer_size
);
18729 volume
= fold_convert (diff_type
, volume
);
18731 chunk_size
= fold_convert (diff_type
, chunk_size
);
18734 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18736 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18737 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18738 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18741 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18742 fold_convert (diff_type
, inner_size
));
18744 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18746 tree offset
= gimple_call_arg (call
, 6);
18747 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18748 fold_convert (diff_type
, offset
));
18749 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18750 diff_type
, r
, range
);
18752 if (diff_type
!= type
)
18753 r
= fold_convert (type
, r
);
18757 gimplify_assign (lhs
, r
, &seq
);
18759 pop_gimplify_context (NULL
);
18761 gsi_replace_with_seq (&gsi
, seq
, true);
18764 /* Default partitioned and minimum partitioned dimensions. */
18766 static int oacc_default_dims
[GOMP_DIM_MAX
];
18767 static int oacc_min_dims
[GOMP_DIM_MAX
];
18769 /* Parse the default dimension parameter. This is a set of
18770 :-separated optional compute dimensions. Each specified dimension
18771 is a positive integer. When device type support is added, it is
18772 planned to be a comma separated list of such compute dimensions,
18773 with all but the first prefixed by the colon-terminated device
18777 oacc_parse_default_dims (const char *dims
)
18781 for (ix
= GOMP_DIM_MAX
; ix
--;)
18783 oacc_default_dims
[ix
] = -1;
18784 oacc_min_dims
[ix
] = 1;
18787 #ifndef ACCEL_COMPILER
18788 /* Cannot be overridden on the host. */
18793 const char *pos
= dims
;
18795 for (ix
= 0; *pos
&& ix
!= GOMP_DIM_MAX
; ix
++)
18810 val
= strtol (pos
, CONST_CAST (char **, &eptr
), 10);
18811 if (errno
|| val
<= 0 || (int) val
!= val
)
18814 oacc_default_dims
[ix
] = (int) val
;
18820 error_at (UNKNOWN_LOCATION
,
18821 "-fopenacc-dim operand is malformed at '%s'", pos
);
18825 /* Allow the backend to validate the dimensions. */
18826 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_default_dims
, -1);
18827 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_min_dims
, -2);
18830 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18831 raw attribute. DIMS is an array of dimensions, which is filled in.
18832 LEVEL is the partitioning level of a routine, or -1 for an offload
18833 region itself. USED is the mask of partitioned execution in the
18837 oacc_validate_dims (tree fn
, tree attrs
, int *dims
, int level
, unsigned used
)
18839 tree purpose
[GOMP_DIM_MAX
];
18841 tree pos
= TREE_VALUE (attrs
);
18842 bool is_kernel
= oacc_fn_attrib_kernels_p (attrs
);
18844 /* Make sure the attribute creator attached the dimension
18848 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18850 purpose
[ix
] = TREE_PURPOSE (pos
);
18851 tree val
= TREE_VALUE (pos
);
18852 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18853 pos
= TREE_CHAIN (pos
);
18856 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, level
);
18858 /* Default anything left to 1 or a partitioned default. */
18859 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18862 /* The OpenACC spec says 'If the [num_gangs] clause is not
18863 specified, an implementation-defined default will be used;
18864 the default may depend on the code within the construct.'
18865 (2.5.6). Thus an implementation is free to choose
18866 non-unity default for a parallel region that doesn't have
18867 any gang-partitioned loops. However, it appears that there
18868 is a sufficient body of user code that expects non-gang
18869 partitioned regions to not execute in gang-redundant mode.
18870 So we (a) don't warn about the non-portability and (b) pick
18871 the minimum permissible dimension size when there is no
18872 partitioned execution. Otherwise we pick the global
18873 default for the dimension, which the user can control. The
18874 same wording and logic applies to num_workers and
18875 vector_length, however the worker- or vector- single
18876 execution doesn't have the same impact as gang-redundant
18877 execution. (If the minimum gang-level partioning is not 1,
18878 the target is probably too confusing.) */
18879 dims
[ix
] = (used
& GOMP_DIM_MASK (ix
)
18880 ? oacc_default_dims
[ix
] : oacc_min_dims
[ix
]);
18886 /* Replace the attribute with new values. */
18888 for (ix
= GOMP_DIM_MAX
; ix
--;)
18890 pos
= tree_cons (purpose
[ix
],
18891 build_int_cst (integer_type_node
, dims
[ix
]),
18894 TREE_PUBLIC (pos
) = 1;
18896 replace_oacc_fn_attrib (fn
, pos
);
18900 /* Create an empty OpenACC loop structure at LOC. */
18903 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18905 oacc_loop
*loop
= XCNEW (oacc_loop
);
18907 loop
->parent
= parent
;
18908 loop
->child
= loop
->sibling
= NULL
;
18912 loop
->sibling
= parent
->child
;
18913 parent
->child
= loop
;
18917 loop
->marker
= NULL
;
18918 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18919 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18920 loop
->routine
= NULL_TREE
;
18922 loop
->mask
= loop
->flags
= loop
->inner
= 0;
18924 loop
->chunk_size
= 0;
18925 loop
->head_end
= NULL
;
18930 /* Create an outermost, dummy OpenACC loop for offloaded function
18934 new_oacc_loop_outer (tree decl
)
18936 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18939 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18940 Link into PARENT loop. Return the new loop. */
18943 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18945 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18947 loop
->marker
= marker
;
18949 /* TODO: This is where device_type flattening would occur for the loop
18952 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18954 tree chunk_size
= integer_zero_node
;
18955 if (loop
->flags
& OLF_GANG_STATIC
)
18956 chunk_size
= gimple_call_arg (marker
, 4);
18957 loop
->chunk_size
= chunk_size
;
18962 /* Create a dummy loop encompassing a call to a openACC routine.
18963 Extract the routine's partitioning requirements. */
18966 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18968 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18969 int level
= oacc_fn_attrib_level (attrs
);
18971 gcc_assert (level
>= 0);
18973 loop
->marker
= call
;
18974 loop
->routine
= decl
;
18975 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18976 ^ (GOMP_DIM_MASK (level
) - 1));
18979 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18980 Return the parent loop. */
18983 finish_oacc_loop (oacc_loop
*loop
)
18985 /* If the loop has been collapsed, don't partition it. */
18987 loop
->mask
= loop
->flags
= 0;
18988 return loop
->parent
;
18991 /* Free all OpenACC loop structures within LOOP (inclusive). */
18994 free_oacc_loop (oacc_loop
*loop
)
18997 free_oacc_loop (loop
->sibling
);
18999 free_oacc_loop (loop
->child
);
19004 /* Dump out the OpenACC loop head or tail beginning at FROM. */
19007 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
19008 const char *title
, int level
)
19010 enum ifn_unique_kind kind
19011 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19013 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
19014 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19016 gimple
*stmt
= gsi_stmt (gsi
);
19018 if (is_gimple_call (stmt
)
19019 && gimple_call_internal_p (stmt
)
19020 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19022 enum ifn_unique_kind k
19023 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
19024 (gimple_call_arg (stmt
, 0)));
19026 if (k
== kind
&& stmt
!= from
)
19029 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
19032 while (gsi_end_p (gsi
))
19033 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19037 /* Dump OpenACC loops LOOP, its siblings and its children. */
19040 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
19044 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
19045 loop
->flags
, loop
->mask
,
19046 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
19049 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
19052 fprintf (file
, "%*sRoutine %s:%u:%s\n",
19053 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
19054 DECL_SOURCE_LINE (loop
->routine
),
19055 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
19057 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
19058 if (loop
->heads
[ix
])
19059 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
19060 for (ix
= GOMP_DIM_MAX
; ix
--;)
19061 if (loop
->tails
[ix
])
19062 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
19065 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
19067 dump_oacc_loop (file
, loop
->sibling
, depth
);
19070 void debug_oacc_loop (oacc_loop
*);
19072 /* Dump loops to stderr. */
19074 DEBUG_FUNCTION
void
19075 debug_oacc_loop (oacc_loop
*loop
)
19077 dump_oacc_loop (stderr
, loop
, 0);
19080 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
19081 structures as we go. By construction these loops are properly
19085 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
19090 if (bb
->flags
& BB_VISITED
)
19094 bb
->flags
|= BB_VISITED
;
19096 /* Scan for loop markers. */
19097 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
19100 gimple
*stmt
= gsi_stmt (gsi
);
19102 if (!is_gimple_call (stmt
))
19105 gcall
*call
= as_a
<gcall
*> (stmt
);
19107 /* If this is a routine, make a dummy loop for it. */
19108 if (tree decl
= gimple_call_fndecl (call
))
19109 if (tree attrs
= get_oacc_fn_attrib (decl
))
19111 gcc_assert (!marker
);
19112 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
19115 if (!gimple_call_internal_p (call
))
19118 switch (gimple_call_internal_fn (call
))
19123 case IFN_GOACC_LOOP
:
19124 /* Count the goacc loop abstraction fns, to determine if the
19125 loop was collapsed already. */
19130 enum ifn_unique_kind kind
19131 = (enum ifn_unique_kind
) (TREE_INT_CST_LOW
19132 (gimple_call_arg (call
, 0)));
19133 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
19134 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19136 if (gimple_call_num_args (call
) == 2)
19138 gcc_assert (marker
&& !remaining
);
19140 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19141 loop
= finish_oacc_loop (loop
);
19143 loop
->head_end
= call
;
19147 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
19151 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19152 loop
= new_oacc_loop (loop
, call
);
19155 gcc_assert (count
== remaining
);
19159 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19160 loop
->heads
[marker
] = call
;
19162 loop
->tails
[remaining
] = call
;
19169 if (remaining
|| marker
)
19171 bb
= single_succ (bb
);
19172 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
19176 /* Walk successor blocks. */
19180 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
19181 oacc_loop_discover_walk (loop
, e
->dest
);
19184 /* LOOP is the first sibling. Reverse the order in place and return
19185 the new first sibling. Recurse to child loops. */
19188 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
19190 oacc_loop
*last
= NULL
;
19194 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
19196 oacc_loop
*next
= loop
->sibling
;
19197 loop
->sibling
= last
;
19206 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19207 the current function. */
19210 oacc_loop_discovery ()
19214 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19215 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19217 /* The siblings were constructed in reverse order, reverse them so
19218 that diagnostics come out in an unsurprising order. */
19219 top
= oacc_loop_sibling_nreverse (top
);
19221 /* Reset the visited flags. */
19222 FOR_ALL_BB_FN (bb
, cfun
)
19223 bb
->flags
&= ~BB_VISITED
;
19228 /* Transform the abstract internal function markers starting at FROM
19229 to be for partitioning level LEVEL. Stop when we meet another HEAD
19233 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19235 enum ifn_unique_kind kind
19236 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19237 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19239 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19241 gimple
*stmt
= gsi_stmt (gsi
);
19243 if (is_gimple_call (stmt
)
19244 && gimple_call_internal_p (stmt
)
19245 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19247 enum ifn_unique_kind k
19248 = ((enum ifn_unique_kind
)
19249 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19251 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19252 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19253 else if (k
== kind
&& stmt
!= from
)
19256 else if (is_gimple_call (stmt
)
19257 && gimple_call_internal_p (stmt
)
19258 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
19259 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
19262 while (gsi_end_p (gsi
))
19263 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19267 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19268 determined partitioning mask and chunking argument. END_MARKER
19269 points at the end IFN_HEAD_TAIL call intgroducing the loop. IFNS
19270 is the number of IFN_GOACC_LOOP calls for the loop. MASK_ARG is
19271 the replacement partitioning mask and CHUNK_ARG is the replacement
19275 oacc_loop_xform_loop (gcall
*end_marker
, unsigned ifns
,
19276 tree mask_arg
, tree chunk_arg
)
19278 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19280 gcc_checking_assert (ifns
);
19283 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19285 gimple
*stmt
= gsi_stmt (gsi
);
19287 if (!is_gimple_call (stmt
))
19290 gcall
*call
= as_a
<gcall
*> (stmt
);
19292 if (!gimple_call_internal_p (call
))
19295 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19298 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19299 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19305 /* The LOOP_BOUND ifn could be in the single successor
19307 basic_block bb
= single_succ (gsi_bb (gsi
));
19308 gsi
= gsi_start_bb (bb
);
19312 /* Process the discovered OpenACC loops, setting the correct
19313 partitioning level etc. */
19316 oacc_loop_process (oacc_loop
*loop
)
19319 oacc_loop_process (loop
->child
);
19321 if (loop
->mask
&& !loop
->routine
)
19324 unsigned mask
= loop
->mask
;
19325 unsigned dim
= GOMP_DIM_GANG
;
19326 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19327 tree chunk_arg
= loop
->chunk_size
;
19329 oacc_loop_xform_loop (loop
->head_end
, loop
->ifns
, mask_arg
, chunk_arg
);
19331 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& mask
; ix
++)
19333 while (!(GOMP_DIM_MASK (dim
) & mask
))
19336 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19337 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19339 mask
^= GOMP_DIM_MASK (dim
);
19344 oacc_loop_process (loop
->sibling
);
19347 /* Walk the OpenACC loop heirarchy checking and assigning the
19348 programmer-specified partitionings. OUTER_MASK is the partitioning
19349 this loop is contained within. Return mask of partitioning
19350 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
19354 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19356 unsigned this_mask
= loop
->mask
;
19357 unsigned mask_all
= 0;
19360 #ifdef ACCEL_COMPILER
19361 /* When device_type is supported, we want the device compiler to be
19362 noisy, if the loop parameters are device_type-specific. */
19366 if (!loop
->routine
)
19368 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19369 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19371 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19372 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19374 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19377 error_at (loop
->loc
,
19379 ? "%<seq%> overrides other OpenACC loop specifiers"
19380 : "%<auto%> conflicts with other OpenACC loop specifiers");
19382 loop
->flags
&= ~OLF_AUTO
;
19386 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19390 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19391 mask_all
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19394 if (this_mask
& outer_mask
)
19396 const oacc_loop
*outer
;
19397 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19398 if (outer
->mask
& this_mask
)
19405 error_at (loop
->loc
,
19406 "%s uses same OpenACC parallelism as containing loop",
19407 loop
->routine
? "routine call" : "inner loop");
19408 inform (outer
->loc
, "containing loop here");
19411 error_at (loop
->loc
,
19412 "%s uses OpenACC parallelism disallowed by containing routine",
19413 loop
->routine
? "routine call" : "loop");
19416 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19417 "routine %qD declared here", loop
->routine
);
19419 this_mask
&= ~outer_mask
;
19423 unsigned outermost
= this_mask
& -this_mask
;
19425 if (outermost
&& outermost
<= outer_mask
)
19429 error_at (loop
->loc
,
19430 "incorrectly nested OpenACC loop parallelism");
19432 const oacc_loop
*outer
;
19433 for (outer
= loop
->parent
;
19434 outer
->flags
&& outer
->flags
< outermost
;
19435 outer
= outer
->parent
)
19437 inform (outer
->loc
, "containing loop here");
19440 this_mask
&= ~outermost
;
19444 loop
->mask
= this_mask
;
19445 mask_all
|= this_mask
;
19449 loop
->inner
= oacc_loop_fixed_partitions (loop
->child
,
19450 outer_mask
| this_mask
);
19451 mask_all
|= loop
->inner
;
19455 mask_all
|= oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
);
19460 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
19461 OUTER_MASK is the partitioning this loop is contained within.
19462 Return the cumulative partitioning used by this loop, siblings and
19466 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19468 bool assign
= (loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
);
19471 #ifdef ACCEL_COMPILER
19472 /* When device_type is supported, we want the device compiler to be
19473 noisy, if the loop parameters are device_type-specific. */
19477 if (assign
&& outer_mask
< GOMP_DIM_MASK (GOMP_DIM_MAX
- 1))
19479 /* Allocate the outermost loop at the outermost available
19481 unsigned this_mask
= outer_mask
+ 1;
19483 if (!(this_mask
& loop
->inner
))
19484 loop
->mask
= this_mask
;
19489 unsigned child_mask
= outer_mask
| loop
->mask
;
19491 if (loop
->mask
|| assign
)
19492 child_mask
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19494 loop
->inner
= oacc_loop_auto_partitions (loop
->child
, child_mask
);
19497 if (assign
&& !loop
->mask
)
19499 /* Allocate the loop at the innermost available level. */
19500 unsigned this_mask
= 0;
19502 /* Determine the outermost partitioning used within this loop. */
19503 this_mask
= loop
->inner
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
19504 this_mask
= (this_mask
& -this_mask
);
19506 /* Pick the partitioning just inside that one. */
19509 /* And avoid picking one use by an outer loop. */
19510 this_mask
&= ~outer_mask
;
19512 if (!this_mask
&& noisy
)
19513 warning_at (loop
->loc
, 0,
19514 "insufficient partitioning available to parallelize loop");
19516 loop
->mask
= this_mask
;
19519 if (assign
&& dump_file
)
19520 fprintf (dump_file
, "Auto loop %s:%d assigned %d\n",
19521 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
19524 unsigned inner_mask
= 0;
19527 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
19529 inner_mask
|= loop
->inner
| loop
->mask
;
19534 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19535 axes. Return mask of partitioning. */
19538 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
19540 unsigned mask_all
= oacc_loop_fixed_partitions (loop
, outer_mask
);
19542 if (mask_all
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
19544 mask_all
^= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19545 mask_all
|= oacc_loop_auto_partitions (loop
, outer_mask
);
19550 /* Default fork/join early expander. Delete the function calls if
19551 there is no RTL expander. */
19554 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19555 const int *ARG_UNUSED (dims
), bool is_fork
)
19558 return targetm
.have_oacc_fork ();
19560 return targetm
.have_oacc_join ();
19563 /* Default goacc.reduction early expander.
19565 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19566 If RES_PTR is not integer-zerop:
19567 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19568 TEARDOWN - emit '*RES_PTR = VAR'
19570 emit 'LHS = VAR' */
19573 default_goacc_reduction (gcall
*call
)
19575 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19576 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19577 tree lhs
= gimple_call_lhs (call
);
19578 tree var
= gimple_call_arg (call
, 2);
19579 gimple_seq seq
= NULL
;
19581 if (code
== IFN_GOACC_REDUCTION_SETUP
19582 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19584 /* Setup and Teardown need to copy from/to the receiver object,
19585 if there is one. */
19586 tree ref_to_res
= gimple_call_arg (call
, 1);
19588 if (!integer_zerop (ref_to_res
))
19590 tree dst
= build_simple_mem_ref (ref_to_res
);
19593 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19599 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19603 /* Copy VAR to LHS, if there is an LHS. */
19605 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19607 gsi_replace_with_seq (&gsi
, seq
, true);
19610 /* Main entry point for oacc transformations which run on the device
19611 compiler after LTO, so we know what the target device is at this
19612 point (including the host fallback). */
19614 static unsigned int
19615 execute_oacc_device_lower ()
19617 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19620 /* Not an offloaded function. */
19623 /* Parse the default dim argument exactly once. */
19624 if ((const void *)flag_openacc_dims
!= &flag_openacc_dims
)
19626 oacc_parse_default_dims (flag_openacc_dims
);
19627 flag_openacc_dims
= (char *)&flag_openacc_dims
;
19630 /* Discover, partition and process the loops. */
19631 oacc_loop
*loops
= oacc_loop_discovery ();
19632 int fn_level
= oacc_fn_attrib_level (attrs
);
19635 fprintf (dump_file
, oacc_fn_attrib_kernels_p (attrs
)
19636 ? "Function is kernels offload\n"
19637 : fn_level
< 0 ? "Function is parallel offload\n"
19638 : "Function is routine level %d\n", fn_level
);
19640 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
19641 unsigned used_mask
= oacc_loop_partition (loops
, outer_mask
);
19642 int dims
[GOMP_DIM_MAX
];
19644 oacc_validate_dims (current_function_decl
, attrs
, dims
, fn_level
, used_mask
);
19648 const char *comma
= "Compute dimensions [";
19649 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, comma
= ", ")
19650 fprintf (dump_file
, "%s%d", comma
, dims
[ix
]);
19651 fprintf (dump_file
, "]\n");
19654 oacc_loop_process (loops
);
19657 fprintf (dump_file
, "OpenACC loops\n");
19658 dump_oacc_loop (dump_file
, loops
, 0);
19659 fprintf (dump_file
, "\n");
19662 /* Offloaded targets may introduce new basic blocks, which require
19663 dominance information to update SSA. */
19664 calculate_dominance_info (CDI_DOMINATORS
);
19666 /* Now lower internal loop functions to target-specific code
19669 FOR_ALL_BB_FN (bb
, cfun
)
19670 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19672 gimple
*stmt
= gsi_stmt (gsi
);
19673 if (!is_gimple_call (stmt
))
19679 gcall
*call
= as_a
<gcall
*> (stmt
);
19680 if (!gimple_call_internal_p (call
))
19686 /* Rewind to allow rescan. */
19688 bool rescan
= false, remove
= false;
19689 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19695 case IFN_GOACC_LOOP
:
19696 oacc_xform_loop (call
);
19700 case IFN_GOACC_REDUCTION
:
19701 /* Mark the function for SSA renaming. */
19702 mark_virtual_operands_for_renaming (cfun
);
19704 /* If the level is -1, this ended up being an unused
19705 axis. Handle as a default. */
19706 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19707 default_goacc_reduction (call
);
19709 targetm
.goacc
.reduction (call
);
19715 enum ifn_unique_kind kind
19716 = ((enum ifn_unique_kind
)
19717 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19722 gcc_unreachable ();
19724 case IFN_UNIQUE_OACC_FORK
:
19725 case IFN_UNIQUE_OACC_JOIN
:
19726 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19728 else if (!targetm
.goacc
.fork_join
19729 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19733 case IFN_UNIQUE_OACC_HEAD_MARK
:
19734 case IFN_UNIQUE_OACC_TAIL_MARK
:
19742 if (gsi_end_p (gsi
))
19743 /* We rewound past the beginning of the BB. */
19744 gsi
= gsi_start_bb (bb
);
19746 /* Undo the rewind. */
19751 if (gimple_vdef (call
))
19752 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19753 if (gimple_call_lhs (call
))
19755 /* Propagate the data dependency var. */
19756 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19757 gimple_call_arg (call
, 1));
19758 gsi_replace (&gsi
, ass
, false);
19761 gsi_remove (&gsi
, true);
19764 /* If not rescanning, advance over the call. */
19768 free_oacc_loop (loops
);
19773 /* Default launch dimension validator. Force everything to 1. A
19774 backend that wants to provide larger dimensions must override this
19778 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19779 int ARG_UNUSED (fn_level
))
19781 bool changed
= false;
19783 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19795 /* Default dimension bound is unknown on accelerator and 1 on host. */
19798 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19800 #ifdef ACCEL_COMPILER
19809 const pass_data pass_data_oacc_device_lower
=
19811 GIMPLE_PASS
, /* type */
19812 "oaccdevlow", /* name */
19813 OPTGROUP_NONE
, /* optinfo_flags */
19814 TV_NONE
, /* tv_id */
19815 PROP_cfg
, /* properties_required */
19816 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19817 0, /* properties_destroyed */
19818 0, /* todo_flags_start */
19819 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19822 class pass_oacc_device_lower
: public gimple_opt_pass
19825 pass_oacc_device_lower (gcc::context
*ctxt
)
19826 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19829 /* opt_pass methods: */
19830 virtual unsigned int execute (function
*)
19832 bool gate
= flag_openacc
!= 0;
19837 return execute_oacc_device_lower ();
19840 }; // class pass_oacc_device_lower
19842 } // anon namespace
19845 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19847 return new pass_oacc_device_lower (ctxt
);
19850 /* "omp declare target link" handling pass. */
19854 const pass_data pass_data_omp_target_link
=
19856 GIMPLE_PASS
, /* type */
19857 "omptargetlink", /* name */
19858 OPTGROUP_NONE
, /* optinfo_flags */
19859 TV_NONE
, /* tv_id */
19860 PROP_ssa
, /* properties_required */
19861 0, /* properties_provided */
19862 0, /* properties_destroyed */
19863 0, /* todo_flags_start */
19864 TODO_update_ssa
, /* todo_flags_finish */
19867 class pass_omp_target_link
: public gimple_opt_pass
19870 pass_omp_target_link (gcc::context
*ctxt
)
19871 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
19874 /* opt_pass methods: */
19875 virtual bool gate (function
*fun
)
19877 #ifdef ACCEL_COMPILER
19878 tree attrs
= DECL_ATTRIBUTES (fun
->decl
);
19879 return lookup_attribute ("omp declare target", attrs
)
19880 || lookup_attribute ("omp target entrypoint", attrs
);
19887 virtual unsigned execute (function
*);
19890 /* Callback for walk_gimple_stmt used to scan for link var operands. */
19893 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
19897 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
)
19898 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
19900 *walk_subtrees
= 0;
19908 pass_omp_target_link::execute (function
*fun
)
19911 FOR_EACH_BB_FN (bb
, fun
)
19913 gimple_stmt_iterator gsi
;
19914 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
19915 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
19916 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
19922 } // anon namespace
19925 make_pass_omp_target_link (gcc::context
*ctxt
)
19927 return new pass_omp_target_link (ctxt
);
19930 #include "gt-omp-low.h"