1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region
*outer
;
107 /* First child region. */
108 struct omp_region
*inner
;
110 /* Next peer region. */
111 struct omp_region
*next
;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec
<tree
, va_gc
> *ws_args
;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type
;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind
;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered
*ord_stmt
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind
;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v
, n1
, n2
, step
;
204 enum tree_code cond_code
;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop
;
217 bool have_nowait
, have_ordered
, simd_schedule
;
218 unsigned char sched_modifiers
;
219 enum omp_clause_schedule_kind sched_kind
;
220 struct omp_for_data_loop
*loops
;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop
*parent
; /* Containing loop. */
230 oacc_loop
*child
; /* First inner loop. */
232 oacc_loop
*sibling
; /* Next loop within same parent. */
234 location_t loc
; /* Location of the loop start. */
236 gcall
*marker
; /* Initial head marker. */
238 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
239 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
241 tree routine
; /* Pseudo-loop enclosing a routine. */
243 unsigned mask
; /* Partitioning mask. */
244 unsigned inner
; /* Partitioning of inner loops. */
245 unsigned flags
; /* Partitioning flags. */
246 unsigned ifns
; /* Contained loop abstraction functions. */
247 tree chunk_size
; /* Chunk size. */
248 gcall
*head_end
; /* Final marker of head sequence. */
251 /* Flags for an OpenACC loop. */
253 enum oacc_loop_flags
{
254 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
255 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
256 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
257 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
259 /* Explicitly specified loop axes. */
261 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
262 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
263 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
265 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
269 static splay_tree all_contexts
;
270 static int taskreg_nesting_level
;
271 static int target_nesting_level
;
272 static struct omp_region
*root_omp_region
;
273 static bitmap task_shared_vars
;
274 static vec
<omp_context
*> taskreg_contexts
;
275 static bool omp_any_child_fn_dumped
;
277 static void scan_omp (gimple_seq
*, omp_context
*);
278 static tree
scan_omp_1_op (tree
*, int *, void *);
279 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
281 #define WALK_SUBSTMTS \
285 case GIMPLE_EH_FILTER: \
286 case GIMPLE_TRANSACTION: \
287 /* The sub-statements for these should be walked. */ \
288 *handled_ops_p = false; \
291 /* Return true if CTX corresponds to an oacc parallel region. */
294 is_oacc_parallel (omp_context
*ctx
)
296 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
297 return ((outer_type
== GIMPLE_OMP_TARGET
)
298 && (gimple_omp_target_kind (ctx
->stmt
)
299 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
302 /* Return true if CTX corresponds to an oacc kernels region. */
305 is_oacc_kernels (omp_context
*ctx
)
307 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
308 return ((outer_type
== GIMPLE_OMP_TARGET
)
309 && (gimple_omp_target_kind (ctx
->stmt
)
310 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
313 /* If DECL is the artificial dummy VAR_DECL created for non-static
314 data member privatization, return the underlying "this" parameter,
315 otherwise return NULL. */
318 omp_member_access_dummy_var (tree decl
)
321 || !DECL_ARTIFICIAL (decl
)
322 || !DECL_IGNORED_P (decl
)
323 || !DECL_HAS_VALUE_EXPR_P (decl
)
324 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
327 tree v
= DECL_VALUE_EXPR (decl
);
328 if (TREE_CODE (v
) != COMPONENT_REF
)
332 switch (TREE_CODE (v
))
338 case POINTER_PLUS_EXPR
:
339 v
= TREE_OPERAND (v
, 0);
342 if (DECL_CONTEXT (v
) == current_function_decl
343 && DECL_ARTIFICIAL (v
)
344 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
352 /* Helper for unshare_and_remap, called through walk_tree. */
355 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
357 tree
*pair
= (tree
*) data
;
360 *tp
= unshare_expr (pair
[1]);
363 else if (IS_TYPE_OR_DECL_P (*tp
))
368 /* Return unshare_expr (X) with all occurrences of FROM
372 unshare_and_remap (tree x
, tree from
, tree to
)
374 tree pair
[2] = { from
, to
};
375 x
= unshare_expr (x
);
376 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
380 /* Holds offload tables with decls. */
381 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
383 /* Convenience function for calling scan_omp_1_op on tree operands. */
386 scan_omp_op (tree
*tp
, omp_context
*ctx
)
388 struct walk_stmt_info wi
;
390 memset (&wi
, 0, sizeof (wi
));
392 wi
.want_locations
= true;
394 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
397 static void lower_omp (gimple_seq
*, omp_context
*);
398 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
399 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
401 /* Find an OMP clause of type KIND within CLAUSES. */
404 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
406 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
407 if (OMP_CLAUSE_CODE (clauses
) == kind
)
413 /* Return true if CTX is for an omp parallel. */
416 is_parallel_ctx (omp_context
*ctx
)
418 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
422 /* Return true if CTX is for an omp task. */
425 is_task_ctx (omp_context
*ctx
)
427 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
431 /* Return true if CTX is for an omp taskloop. */
434 is_taskloop_ctx (omp_context
*ctx
)
436 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
437 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
441 /* Return true if CTX is for an omp parallel or omp task. */
444 is_taskreg_ctx (omp_context
*ctx
)
446 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
450 /* Return true if REGION is a combined parallel+workshare region. */
453 is_combined_parallel (struct omp_region
*region
)
455 return region
->is_combined_parallel
;
458 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
462 adjust_for_condition (location_t loc
, enum tree_code
*cond_code
, tree
*n2
)
471 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
472 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, 1);
474 *n2
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (*n2
), *n2
,
475 build_int_cst (TREE_TYPE (*n2
), 1));
476 *cond_code
= LT_EXPR
;
479 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
480 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, -1);
482 *n2
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (*n2
), *n2
,
483 build_int_cst (TREE_TYPE (*n2
), 1));
484 *cond_code
= GT_EXPR
;
491 /* Return the looping step from INCR, extracted from the step of a gimple omp
495 get_omp_for_step_from_incr (location_t loc
, tree incr
)
498 switch (TREE_CODE (incr
))
501 step
= TREE_OPERAND (incr
, 1);
503 case POINTER_PLUS_EXPR
:
504 step
= fold_convert (ssizetype
, TREE_OPERAND (incr
, 1));
507 step
= TREE_OPERAND (incr
, 1);
508 step
= fold_build1_loc (loc
, NEGATE_EXPR
, TREE_TYPE (step
), step
);
516 /* Extract the header elements of parallel loop FOR_STMT and store
520 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
521 struct omp_for_data_loop
*loops
)
523 tree t
, var
, *collapse_iter
, *collapse_count
;
524 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
525 struct omp_for_data_loop
*loop
;
527 struct omp_for_data_loop dummy_loop
;
528 location_t loc
= gimple_location (for_stmt
);
529 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
530 bool distribute
= gimple_omp_for_kind (for_stmt
)
531 == GF_OMP_FOR_KIND_DISTRIBUTE
;
532 bool taskloop
= gimple_omp_for_kind (for_stmt
)
533 == GF_OMP_FOR_KIND_TASKLOOP
;
536 fd
->for_stmt
= for_stmt
;
538 if (gimple_omp_for_collapse (for_stmt
) > 1)
541 fd
->loops
= &fd
->loop
;
543 fd
->have_nowait
= distribute
|| simd
;
544 fd
->have_ordered
= false;
547 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
548 fd
->sched_modifiers
= 0;
549 fd
->chunk_size
= NULL_TREE
;
550 fd
->simd_schedule
= false;
551 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
552 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
553 collapse_iter
= NULL
;
554 collapse_count
= NULL
;
556 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
557 switch (OMP_CLAUSE_CODE (t
))
559 case OMP_CLAUSE_NOWAIT
:
560 fd
->have_nowait
= true;
562 case OMP_CLAUSE_ORDERED
:
563 fd
->have_ordered
= true;
564 if (OMP_CLAUSE_ORDERED_EXPR (t
))
565 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
567 case OMP_CLAUSE_SCHEDULE
:
568 gcc_assert (!distribute
&& !taskloop
);
570 = (enum omp_clause_schedule_kind
)
571 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
572 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
573 & ~OMP_CLAUSE_SCHEDULE_MASK
);
574 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
575 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
577 case OMP_CLAUSE_DIST_SCHEDULE
:
578 gcc_assert (distribute
);
579 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
581 case OMP_CLAUSE_COLLAPSE
:
582 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
583 if (fd
->collapse
> 1)
585 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
586 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
592 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
597 collapse_iter
= &iterv
;
598 collapse_count
= &countv
;
601 /* FIXME: for now map schedule(auto) to schedule(static).
602 There should be analysis to determine whether all iterations
603 are approximately the same amount of work (then schedule(static)
604 is best) or if it varies (then schedule(dynamic,N) is better). */
605 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
607 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
608 gcc_assert (fd
->chunk_size
== NULL
);
610 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
612 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
613 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
614 gcc_assert (fd
->chunk_size
== NULL
);
615 else if (fd
->chunk_size
== NULL
)
617 /* We only need to compute a default chunk size for ordered
618 static loops and dynamic loops. */
619 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
621 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
622 ? integer_zero_node
: integer_one_node
;
625 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
626 for (i
= 0; i
< cnt
; i
++)
628 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
630 else if (loops
!= NULL
)
635 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
636 gcc_assert (SSA_VAR_P (loop
->v
));
637 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
638 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
639 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
640 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
642 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
643 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
644 gcc_assert (loop
->cond_code
!= NE_EXPR
645 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKSIMD
646 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
);
647 adjust_for_condition (loc
, &loop
->cond_code
, &loop
->n2
);
649 t
= gimple_omp_for_incr (for_stmt
, i
);
650 gcc_assert (TREE_OPERAND (t
, 0) == var
);
651 loop
->step
= get_omp_for_step_from_incr (loc
, t
);
654 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
655 && !fd
->have_ordered
))
657 if (fd
->collapse
== 1)
658 iter_type
= TREE_TYPE (loop
->v
);
660 || TYPE_PRECISION (iter_type
)
661 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
663 = build_nonstandard_integer_type
664 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
666 else if (iter_type
!= long_long_unsigned_type_node
)
668 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
669 iter_type
= long_long_unsigned_type_node
;
670 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
671 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
672 >= TYPE_PRECISION (iter_type
))
676 if (loop
->cond_code
== LT_EXPR
)
677 n
= fold_build2_loc (loc
,
678 PLUS_EXPR
, TREE_TYPE (loop
->v
),
679 loop
->n2
, loop
->step
);
682 if (TREE_CODE (n
) != INTEGER_CST
683 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
684 iter_type
= long_long_unsigned_type_node
;
686 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
687 > TYPE_PRECISION (iter_type
))
691 if (loop
->cond_code
== LT_EXPR
)
694 n2
= fold_build2_loc (loc
,
695 PLUS_EXPR
, TREE_TYPE (loop
->v
),
696 loop
->n2
, loop
->step
);
700 n1
= fold_build2_loc (loc
,
701 MINUS_EXPR
, TREE_TYPE (loop
->v
),
702 loop
->n2
, loop
->step
);
705 if (TREE_CODE (n1
) != INTEGER_CST
706 || TREE_CODE (n2
) != INTEGER_CST
707 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
708 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
709 iter_type
= long_long_unsigned_type_node
;
713 if (i
>= fd
->collapse
)
716 if (collapse_count
&& *collapse_count
== NULL
)
718 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
719 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
720 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
721 if (t
&& integer_zerop (t
))
722 count
= build_zero_cst (long_long_unsigned_type_node
);
723 else if ((i
== 0 || count
!= NULL_TREE
)
724 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
725 && TREE_CONSTANT (loop
->n1
)
726 && TREE_CONSTANT (loop
->n2
)
727 && TREE_CODE (loop
->step
) == INTEGER_CST
)
729 tree itype
= TREE_TYPE (loop
->v
);
731 if (POINTER_TYPE_P (itype
))
732 itype
= signed_type_for (itype
);
733 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
734 t
= fold_build2_loc (loc
,
736 fold_convert_loc (loc
, itype
, loop
->step
), t
);
737 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
738 fold_convert_loc (loc
, itype
, loop
->n2
));
739 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
740 fold_convert_loc (loc
, itype
, loop
->n1
));
741 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
742 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
743 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
744 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
745 fold_convert_loc (loc
, itype
,
748 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
749 fold_convert_loc (loc
, itype
, loop
->step
));
750 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
751 if (count
!= NULL_TREE
)
752 count
= fold_build2_loc (loc
,
753 MULT_EXPR
, long_long_unsigned_type_node
,
757 if (TREE_CODE (count
) != INTEGER_CST
)
760 else if (count
&& !integer_zerop (count
))
767 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
768 || fd
->have_ordered
))
770 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
771 iter_type
= long_long_unsigned_type_node
;
773 iter_type
= long_integer_type_node
;
775 else if (collapse_iter
&& *collapse_iter
!= NULL
)
776 iter_type
= TREE_TYPE (*collapse_iter
);
777 fd
->iter_type
= iter_type
;
778 if (collapse_iter
&& *collapse_iter
== NULL
)
779 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
780 if (collapse_count
&& *collapse_count
== NULL
)
783 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
785 *collapse_count
= create_tmp_var (iter_type
, ".count");
788 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
790 fd
->loop
.v
= *collapse_iter
;
791 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
792 fd
->loop
.n2
= *collapse_count
;
793 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
794 fd
->loop
.cond_code
= LT_EXPR
;
801 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
802 is the immediate dominator of PAR_ENTRY_BB, return true if there
803 are no data dependencies that would prevent expanding the parallel
804 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
806 When expanding a combined parallel+workshare region, the call to
807 the child function may need additional arguments in the case of
808 GIMPLE_OMP_FOR regions. In some cases, these arguments are
809 computed out of variables passed in from the parent to the child
810 via 'struct .omp_data_s'. For instance:
812 #pragma omp parallel for schedule (guided, i * 4)
817 # BLOCK 2 (PAR_ENTRY_BB)
819 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
821 # BLOCK 3 (WS_ENTRY_BB)
822 .omp_data_i = &.omp_data_o;
823 D.1667 = .omp_data_i->i;
825 #pragma omp for schedule (guided, D.1598)
827 When we outline the parallel region, the call to the child function
828 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
829 that value is computed *after* the call site. So, in principle we
830 cannot do the transformation.
832 To see whether the code in WS_ENTRY_BB blocks the combined
833 parallel+workshare call, we collect all the variables used in the
834 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
835 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
838 FIXME. If we had the SSA form built at this point, we could merely
839 hoist the code in block 3 into block 2 and be done with it. But at
840 this point we don't have dataflow information and though we could
841 hack something up here, it is really not worth the aggravation. */
844 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
846 struct omp_for_data fd
;
847 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
849 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
852 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
854 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
856 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
858 if (fd
.iter_type
!= long_integer_type_node
)
861 /* FIXME. We give up too easily here. If any of these arguments
862 are not constants, they will likely involve variables that have
863 been mapped into fields of .omp_data_s for sharing with the child
864 function. With appropriate data flow, it would be possible to
866 if (!is_gimple_min_invariant (fd
.loop
.n1
)
867 || !is_gimple_min_invariant (fd
.loop
.n2
)
868 || !is_gimple_min_invariant (fd
.loop
.step
)
869 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
876 static int omp_max_vf (void);
878 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
879 presence (SIMD_SCHEDULE). */
882 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
887 int vf
= omp_max_vf ();
891 tree type
= TREE_TYPE (chunk_size
);
892 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
893 build_int_cst (type
, vf
- 1));
894 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
895 build_int_cst (type
, -vf
));
899 /* Collect additional arguments needed to emit a combined
900 parallel+workshare call. WS_STMT is the workshare directive being
903 static vec
<tree
, va_gc
> *
904 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
907 location_t loc
= gimple_location (ws_stmt
);
908 vec
<tree
, va_gc
> *ws_args
;
910 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
912 struct omp_for_data fd
;
915 extract_omp_for_data (for_stmt
, &fd
, NULL
);
919 if (gimple_omp_for_combined_into_p (for_stmt
))
922 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
923 OMP_CLAUSE__LOOPTEMP_
);
925 n1
= OMP_CLAUSE_DECL (innerc
);
926 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
927 OMP_CLAUSE__LOOPTEMP_
);
929 n2
= OMP_CLAUSE_DECL (innerc
);
932 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
934 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
935 ws_args
->quick_push (t
);
937 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
938 ws_args
->quick_push (t
);
940 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
941 ws_args
->quick_push (t
);
945 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
946 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
947 ws_args
->quick_push (t
);
952 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
954 /* Number of sections is equal to the number of edges from the
955 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
956 the exit of the sections region. */
957 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
958 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
959 vec_alloc (ws_args
, 1);
960 ws_args
->quick_push (t
);
968 /* Discover whether REGION is a combined parallel+workshare region. */
971 determine_parallel_type (struct omp_region
*region
)
973 basic_block par_entry_bb
, par_exit_bb
;
974 basic_block ws_entry_bb
, ws_exit_bb
;
976 if (region
== NULL
|| region
->inner
== NULL
977 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
978 || region
->inner
->cont
== NULL
)
981 /* We only support parallel+for and parallel+sections. */
982 if (region
->type
!= GIMPLE_OMP_PARALLEL
983 || (region
->inner
->type
!= GIMPLE_OMP_FOR
984 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
987 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
988 WS_EXIT_BB -> PAR_EXIT_BB. */
989 par_entry_bb
= region
->entry
;
990 par_exit_bb
= region
->exit
;
991 ws_entry_bb
= region
->inner
->entry
;
992 ws_exit_bb
= region
->inner
->exit
;
994 if (single_succ (par_entry_bb
) == ws_entry_bb
995 && single_succ (ws_exit_bb
) == par_exit_bb
996 && workshare_safe_to_combine_p (ws_entry_bb
)
997 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
998 || (last_and_only_stmt (ws_entry_bb
)
999 && last_and_only_stmt (par_exit_bb
))))
1001 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1002 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1004 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1006 /* If this is a combined parallel loop, we need to determine
1007 whether or not to use the combined library calls. There
1008 are two cases where we do not apply the transformation:
1009 static loops and any kind of ordered loop. In the first
1010 case, we already open code the loop so there is no need
1011 to do anything else. In the latter case, the combined
1012 parallel loop call would still need extra synchronization
1013 to implement ordered semantics, so there would not be any
1014 gain in using the combined call. */
1015 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1016 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1018 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1019 == OMP_CLAUSE_SCHEDULE_STATIC
)
1020 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1022 region
->is_combined_parallel
= false;
1023 region
->inner
->is_combined_parallel
= false;
1028 region
->is_combined_parallel
= true;
1029 region
->inner
->is_combined_parallel
= true;
1030 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1035 /* Return true if EXPR is variable sized. */
1038 is_variable_sized (const_tree expr
)
1040 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1043 /* Return true if DECL is a reference type. */
1046 is_reference (tree decl
)
1048 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1051 /* Return the type of a decl. If the decl is reference type,
1052 return its base type. */
1054 get_base_type (tree decl
)
1056 tree type
= TREE_TYPE (decl
);
1057 if (is_reference (decl
))
1058 type
= TREE_TYPE (type
);
1062 /* Lookup variables. The "maybe" form
1063 allows for the variable form to not have been entered, otherwise we
1064 assert that the variable must have been entered. */
1067 lookup_decl (tree var
, omp_context
*ctx
)
1069 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1074 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1076 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1077 return n
? *n
: NULL_TREE
;
1081 lookup_field (tree var
, omp_context
*ctx
)
1084 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1085 return (tree
) n
->value
;
1089 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1092 n
= splay_tree_lookup (ctx
->sfield_map
1093 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1094 return (tree
) n
->value
;
1098 lookup_sfield (tree var
, omp_context
*ctx
)
1100 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1104 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1107 n
= splay_tree_lookup (ctx
->field_map
, key
);
1108 return n
? (tree
) n
->value
: NULL_TREE
;
1112 maybe_lookup_field (tree var
, omp_context
*ctx
)
1114 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1117 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1118 the parallel context if DECL is to be shared. */
1121 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1123 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1126 /* We can only use copy-in/copy-out semantics for shared variables
1127 when we know the value is not accessible from an outer scope. */
1130 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1132 /* ??? Trivially accessible from anywhere. But why would we even
1133 be passing an address in this case? Should we simply assert
1134 this to be false, or should we have a cleanup pass that removes
1135 these from the list of mappings? */
1136 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1139 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1140 without analyzing the expression whether or not its location
1141 is accessible to anyone else. In the case of nested parallel
1142 regions it certainly may be. */
1143 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1146 /* Do not use copy-in/copy-out for variables that have their
1148 if (TREE_ADDRESSABLE (decl
))
1151 /* lower_send_shared_vars only uses copy-in, but not copy-out
1153 if (TREE_READONLY (decl
)
1154 || ((TREE_CODE (decl
) == RESULT_DECL
1155 || TREE_CODE (decl
) == PARM_DECL
)
1156 && DECL_BY_REFERENCE (decl
)))
1159 /* Disallow copy-in/out in nested parallel if
1160 decl is shared in outer parallel, otherwise
1161 each thread could store the shared variable
1162 in its own copy-in location, making the
1163 variable no longer really shared. */
1164 if (shared_ctx
->is_nested
)
1168 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1169 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1176 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1177 c
; c
= OMP_CLAUSE_CHAIN (c
))
1178 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1179 && OMP_CLAUSE_DECL (c
) == decl
)
1183 goto maybe_mark_addressable_and_ret
;
1187 /* For tasks avoid using copy-in/out. As tasks can be
1188 deferred or executed in different thread, when GOMP_task
1189 returns, the task hasn't necessarily terminated. */
1190 if (is_task_ctx (shared_ctx
))
1193 maybe_mark_addressable_and_ret
:
1194 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1195 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1197 /* Taking address of OUTER in lower_send_shared_vars
1198 might need regimplification of everything that uses the
1200 if (!task_shared_vars
)
1201 task_shared_vars
= BITMAP_ALLOC (NULL
);
1202 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1203 TREE_ADDRESSABLE (outer
) = 1;
1212 /* Construct a new automatic decl similar to VAR. */
1215 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1217 tree copy
= copy_var_decl (var
, name
, type
);
1219 DECL_CONTEXT (copy
) = current_function_decl
;
1220 DECL_CHAIN (copy
) = ctx
->block_vars
;
1221 /* If VAR is listed in task_shared_vars, it means it wasn't
1222 originally addressable and is just because task needs to take
1223 it's address. But we don't need to take address of privatizations
1225 if (TREE_ADDRESSABLE (var
)
1227 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1228 TREE_ADDRESSABLE (copy
) = 0;
1229 ctx
->block_vars
= copy
;
1235 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1237 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1240 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1243 omp_build_component_ref (tree obj
, tree field
)
1245 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1246 if (TREE_THIS_VOLATILE (field
))
1247 TREE_THIS_VOLATILE (ret
) |= 1;
1248 if (TREE_READONLY (field
))
1249 TREE_READONLY (ret
) |= 1;
1253 /* Build tree nodes to access the field for VAR on the receiver side. */
1256 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1258 tree x
, field
= lookup_field (var
, ctx
);
1260 /* If the receiver record type was remapped in the child function,
1261 remap the field into the new record type. */
1262 x
= maybe_lookup_field (field
, ctx
);
1266 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1267 TREE_THIS_NOTRAP (x
) = 1;
1268 x
= omp_build_component_ref (x
, field
);
1271 x
= build_simple_mem_ref (x
);
1272 TREE_THIS_NOTRAP (x
) = 1;
1278 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1279 of a parallel, this is a component reference; for workshare constructs
1280 this is some variable. */
1283 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1287 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1289 else if (is_variable_sized (var
))
1291 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1292 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1293 x
= build_simple_mem_ref (x
);
1295 else if (is_taskreg_ctx (ctx
))
1297 bool by_ref
= use_pointer_for_field (var
, NULL
);
1298 x
= build_receiver_ref (var
, by_ref
, ctx
);
1300 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1301 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1303 /* #pragma omp simd isn't a worksharing construct, and can reference even
1304 private vars in its linear etc. clauses. */
1306 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1307 x
= lookup_decl (var
, ctx
->outer
);
1308 else if (ctx
->outer
)
1309 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1313 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1315 gcc_assert (ctx
->outer
);
1317 = splay_tree_lookup (ctx
->outer
->field_map
,
1318 (splay_tree_key
) &DECL_UID (var
));
1321 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1324 x
= lookup_decl (var
, ctx
->outer
);
1328 tree field
= (tree
) n
->value
;
1329 /* If the receiver record type was remapped in the child function,
1330 remap the field into the new record type. */
1331 x
= maybe_lookup_field (field
, ctx
->outer
);
1335 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1336 x
= omp_build_component_ref (x
, field
);
1337 if (use_pointer_for_field (var
, ctx
->outer
))
1338 x
= build_simple_mem_ref (x
);
1341 else if (ctx
->outer
)
1343 omp_context
*outer
= ctx
->outer
;
1344 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
1346 outer
= outer
->outer
;
1348 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
1350 x
= lookup_decl (var
, outer
);
1352 else if (is_reference (var
))
1353 /* This can happen with orphaned constructs. If var is reference, it is
1354 possible it is shared and as such valid. */
1356 else if (omp_member_access_dummy_var (var
))
1363 tree t
= omp_member_access_dummy_var (var
);
1366 x
= DECL_VALUE_EXPR (var
);
1367 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1369 x
= unshare_and_remap (x
, t
, o
);
1371 x
= unshare_expr (x
);
1375 if (is_reference (var
))
1376 x
= build_simple_mem_ref (x
);
1381 /* Build tree nodes to access the field for VAR on the sender side. */
1384 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1386 tree field
= lookup_sfield (key
, ctx
);
1387 return omp_build_component_ref (ctx
->sender_decl
, field
);
1391 build_sender_ref (tree var
, omp_context
*ctx
)
1393 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1396 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1397 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1400 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
,
1401 bool base_pointers_restrict
= false)
1403 tree field
, type
, sfield
= NULL_TREE
;
1404 splay_tree_key key
= (splay_tree_key
) var
;
1406 if ((mask
& 8) != 0)
1408 key
= (splay_tree_key
) &DECL_UID (var
);
1409 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1411 gcc_assert ((mask
& 1) == 0
1412 || !splay_tree_lookup (ctx
->field_map
, key
));
1413 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1414 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1415 gcc_assert ((mask
& 3) == 3
1416 || !is_gimple_omp_oacc (ctx
->stmt
));
1418 type
= TREE_TYPE (var
);
1419 /* Prevent redeclaring the var in the split-off function with a restrict
1420 pointer type. Note that we only clear type itself, restrict qualifiers in
1421 the pointed-to type will be ignored by points-to analysis. */
1422 if (POINTER_TYPE_P (type
)
1423 && TYPE_RESTRICT (type
))
1424 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
1428 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1429 type
= build_pointer_type (build_pointer_type (type
));
1433 type
= build_pointer_type (type
);
1434 if (base_pointers_restrict
)
1435 type
= build_qualified_type (type
, TYPE_QUAL_RESTRICT
);
1437 else if ((mask
& 3) == 1 && is_reference (var
))
1438 type
= TREE_TYPE (type
);
1440 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1441 FIELD_DECL
, DECL_NAME (var
), type
);
1443 /* Remember what variable this field was created for. This does have a
1444 side effect of making dwarf2out ignore this member, so for helpful
1445 debugging we clear it later in delete_omp_context. */
1446 DECL_ABSTRACT_ORIGIN (field
) = var
;
1447 if (type
== TREE_TYPE (var
))
1449 SET_DECL_ALIGN (field
, DECL_ALIGN (var
));
1450 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1451 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1454 SET_DECL_ALIGN (field
, TYPE_ALIGN (type
));
1456 if ((mask
& 3) == 3)
1458 insert_field_into_struct (ctx
->record_type
, field
);
1459 if (ctx
->srecord_type
)
1461 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1462 FIELD_DECL
, DECL_NAME (var
), type
);
1463 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1464 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
1465 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1466 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1467 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1472 if (ctx
->srecord_type
== NULL_TREE
)
1476 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1477 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1478 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1480 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1481 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1482 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1483 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1484 splay_tree_insert (ctx
->sfield_map
,
1485 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1486 (splay_tree_value
) sfield
);
1490 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1491 : ctx
->srecord_type
, field
);
1495 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1496 if ((mask
& 2) && ctx
->sfield_map
)
1497 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1501 install_var_local (tree var
, omp_context
*ctx
)
1503 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1504 insert_decl_map (&ctx
->cb
, var
, new_var
);
1508 /* Adjust the replacement for DECL in CTX for the new context. This means
1509 copying the DECL_VALUE_EXPR, and fixing up the type. */
1512 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1514 tree new_decl
, size
;
1516 new_decl
= lookup_decl (decl
, ctx
);
1518 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1520 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1521 && DECL_HAS_VALUE_EXPR_P (decl
))
1523 tree ve
= DECL_VALUE_EXPR (decl
);
1524 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1525 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1526 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1529 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1531 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1532 if (size
== error_mark_node
)
1533 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1534 DECL_SIZE (new_decl
) = size
;
1536 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1537 if (size
== error_mark_node
)
1538 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1539 DECL_SIZE_UNIT (new_decl
) = size
;
1543 /* The callback for remap_decl. Search all containing contexts for a
1544 mapping of the variable; this avoids having to duplicate the splay
1545 tree ahead of time. We know a mapping doesn't already exist in the
1546 given context. Create new mappings to implement default semantics. */
1549 omp_copy_decl (tree var
, copy_body_data
*cb
)
1551 omp_context
*ctx
= (omp_context
*) cb
;
1554 if (TREE_CODE (var
) == LABEL_DECL
)
1556 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1557 DECL_CONTEXT (new_var
) = current_function_decl
;
1558 insert_decl_map (&ctx
->cb
, var
, new_var
);
1562 while (!is_taskreg_ctx (ctx
))
1567 new_var
= maybe_lookup_decl (var
, ctx
);
1572 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1575 return error_mark_node
;
1579 /* Debugging dumps for parallel regions. */
1580 void dump_omp_region (FILE *, struct omp_region
*, int);
1581 void debug_omp_region (struct omp_region
*);
1582 void debug_all_omp_regions (void);
1584 /* Dump the parallel region tree rooted at REGION. */
1587 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1589 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1590 gimple_code_name
[region
->type
]);
1593 dump_omp_region (file
, region
->inner
, indent
+ 4);
1597 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1598 region
->cont
->index
);
1602 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1603 region
->exit
->index
);
1605 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1608 dump_omp_region (file
, region
->next
, indent
);
1612 debug_omp_region (struct omp_region
*region
)
1614 dump_omp_region (stderr
, region
, 0);
1618 debug_all_omp_regions (void)
1620 dump_omp_region (stderr
, root_omp_region
, 0);
1624 /* Create a new parallel region starting at STMT inside region PARENT. */
1626 static struct omp_region
*
1627 new_omp_region (basic_block bb
, enum gimple_code type
,
1628 struct omp_region
*parent
)
1630 struct omp_region
*region
= XCNEW (struct omp_region
);
1632 region
->outer
= parent
;
1634 region
->type
= type
;
1638 /* This is a nested region. Add it to the list of inner
1639 regions in PARENT. */
1640 region
->next
= parent
->inner
;
1641 parent
->inner
= region
;
1645 /* This is a toplevel region. Add it to the list of toplevel
1646 regions in ROOT_OMP_REGION. */
1647 region
->next
= root_omp_region
;
1648 root_omp_region
= region
;
1654 /* Release the memory associated with the region tree rooted at REGION. */
1657 free_omp_region_1 (struct omp_region
*region
)
1659 struct omp_region
*i
, *n
;
1661 for (i
= region
->inner
; i
; i
= n
)
1664 free_omp_region_1 (i
);
1670 /* Release the memory for the entire omp region tree. */
1673 free_omp_regions (void)
1675 struct omp_region
*r
, *n
;
1676 for (r
= root_omp_region
; r
; r
= n
)
1679 free_omp_region_1 (r
);
1681 root_omp_region
= NULL
;
1685 /* Create a new context, with OUTER_CTX being the surrounding context. */
1687 static omp_context
*
1688 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1690 omp_context
*ctx
= XCNEW (omp_context
);
1692 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1693 (splay_tree_value
) ctx
);
1698 ctx
->outer
= outer_ctx
;
1699 ctx
->cb
= outer_ctx
->cb
;
1700 ctx
->cb
.block
= NULL
;
1701 ctx
->depth
= outer_ctx
->depth
+ 1;
1705 ctx
->cb
.src_fn
= current_function_decl
;
1706 ctx
->cb
.dst_fn
= current_function_decl
;
1707 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1708 gcc_checking_assert (ctx
->cb
.src_node
);
1709 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1710 ctx
->cb
.src_cfun
= cfun
;
1711 ctx
->cb
.copy_decl
= omp_copy_decl
;
1712 ctx
->cb
.eh_lp_nr
= 0;
1713 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1717 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1722 static gimple_seq
maybe_catch_exception (gimple_seq
);
1724 /* Finalize task copyfn. */
1727 finalize_task_copyfn (gomp_task
*task_stmt
)
1729 struct function
*child_cfun
;
1731 gimple_seq seq
= NULL
, new_seq
;
1734 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1735 if (child_fn
== NULL_TREE
)
1738 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1739 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1741 push_cfun (child_cfun
);
1742 bind
= gimplify_body (child_fn
, false);
1743 gimple_seq_add_stmt (&seq
, bind
);
1744 new_seq
= maybe_catch_exception (seq
);
1747 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1749 gimple_seq_add_stmt (&seq
, bind
);
1751 gimple_set_body (child_fn
, seq
);
1754 /* Inform the callgraph about the new function. */
1755 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1756 node
->parallelized_function
= 1;
1757 cgraph_node::add_new_function (child_fn
, false);
1760 /* Destroy a omp_context data structures. Called through the splay tree
1761 value delete callback. */
1764 delete_omp_context (splay_tree_value value
)
1766 omp_context
*ctx
= (omp_context
*) value
;
1768 delete ctx
->cb
.decl_map
;
1771 splay_tree_delete (ctx
->field_map
);
1772 if (ctx
->sfield_map
)
1773 splay_tree_delete (ctx
->sfield_map
);
1775 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1776 it produces corrupt debug information. */
1777 if (ctx
->record_type
)
1780 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1781 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1783 if (ctx
->srecord_type
)
1786 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1787 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1790 if (is_task_ctx (ctx
))
1791 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1796 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1800 fixup_child_record_type (omp_context
*ctx
)
1802 tree f
, type
= ctx
->record_type
;
1804 if (!ctx
->receiver_decl
)
1806 /* ??? It isn't sufficient to just call remap_type here, because
1807 variably_modified_type_p doesn't work the way we expect for
1808 record types. Testing each field for whether it needs remapping
1809 and creating a new record by hand works, however. */
1810 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1811 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1815 tree name
, new_fields
= NULL
;
1817 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1818 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1819 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1820 TYPE_DECL
, name
, type
);
1821 TYPE_NAME (type
) = name
;
1823 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1825 tree new_f
= copy_node (f
);
1826 DECL_CONTEXT (new_f
) = type
;
1827 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1828 DECL_CHAIN (new_f
) = new_fields
;
1829 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1830 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1832 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1836 /* Arrange to be able to look up the receiver field
1837 given the sender field. */
1838 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1839 (splay_tree_value
) new_f
);
1841 TYPE_FIELDS (type
) = nreverse (new_fields
);
1845 /* In a target region we never modify any of the pointers in *.omp_data_i,
1846 so attempt to help the optimizers. */
1847 if (is_gimple_omp_offloaded (ctx
->stmt
))
1848 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1850 TREE_TYPE (ctx
->receiver_decl
)
1851 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1854 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1855 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1859 scan_sharing_clauses (tree clauses
, omp_context
*ctx
,
1860 bool base_pointers_restrict
= false)
1863 bool scan_array_reductions
= false;
1865 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1869 switch (OMP_CLAUSE_CODE (c
))
1871 case OMP_CLAUSE_PRIVATE
:
1872 decl
= OMP_CLAUSE_DECL (c
);
1873 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1875 else if (!is_variable_sized (decl
))
1876 install_var_local (decl
, ctx
);
1879 case OMP_CLAUSE_SHARED
:
1880 decl
= OMP_CLAUSE_DECL (c
);
1881 /* Ignore shared directives in teams construct. */
1882 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1884 /* Global variables don't need to be copied,
1885 the receiver side will use them directly. */
1886 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1887 if (is_global_var (odecl
))
1889 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1892 gcc_assert (is_taskreg_ctx (ctx
));
1893 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1894 || !is_variable_sized (decl
));
1895 /* Global variables don't need to be copied,
1896 the receiver side will use them directly. */
1897 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1899 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1901 use_pointer_for_field (decl
, ctx
);
1904 by_ref
= use_pointer_for_field (decl
, NULL
);
1905 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1906 || TREE_ADDRESSABLE (decl
)
1908 || is_reference (decl
))
1910 by_ref
= use_pointer_for_field (decl
, ctx
);
1911 install_var_field (decl
, by_ref
, 3, ctx
);
1912 install_var_local (decl
, ctx
);
1915 /* We don't need to copy const scalar vars back. */
1916 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1919 case OMP_CLAUSE_REDUCTION
:
1920 decl
= OMP_CLAUSE_DECL (c
);
1921 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1922 && TREE_CODE (decl
) == MEM_REF
)
1924 tree t
= TREE_OPERAND (decl
, 0);
1925 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1926 t
= TREE_OPERAND (t
, 0);
1927 if (TREE_CODE (t
) == INDIRECT_REF
1928 || TREE_CODE (t
) == ADDR_EXPR
)
1929 t
= TREE_OPERAND (t
, 0);
1930 install_var_local (t
, ctx
);
1931 if (is_taskreg_ctx (ctx
)
1932 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1933 && !is_variable_sized (t
))
1935 by_ref
= use_pointer_for_field (t
, ctx
);
1936 install_var_field (t
, by_ref
, 3, ctx
);
1942 case OMP_CLAUSE_LASTPRIVATE
:
1943 /* Let the corresponding firstprivate clause create
1945 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1949 case OMP_CLAUSE_FIRSTPRIVATE
:
1950 case OMP_CLAUSE_LINEAR
:
1951 decl
= OMP_CLAUSE_DECL (c
);
1953 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1954 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1955 && is_gimple_omp_offloaded (ctx
->stmt
))
1957 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1958 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1959 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1960 install_var_field (decl
, true, 3, ctx
);
1962 install_var_field (decl
, false, 3, ctx
);
1964 if (is_variable_sized (decl
))
1966 if (is_task_ctx (ctx
))
1967 install_var_field (decl
, false, 1, ctx
);
1970 else if (is_taskreg_ctx (ctx
))
1973 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1974 by_ref
= use_pointer_for_field (decl
, NULL
);
1976 if (is_task_ctx (ctx
)
1977 && (global
|| by_ref
|| is_reference (decl
)))
1979 install_var_field (decl
, false, 1, ctx
);
1981 install_var_field (decl
, by_ref
, 2, ctx
);
1984 install_var_field (decl
, by_ref
, 3, ctx
);
1986 install_var_local (decl
, ctx
);
1989 case OMP_CLAUSE_USE_DEVICE_PTR
:
1990 decl
= OMP_CLAUSE_DECL (c
);
1991 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1992 install_var_field (decl
, true, 3, ctx
);
1994 install_var_field (decl
, false, 3, ctx
);
1995 if (DECL_SIZE (decl
)
1996 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1998 tree decl2
= DECL_VALUE_EXPR (decl
);
1999 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2000 decl2
= TREE_OPERAND (decl2
, 0);
2001 gcc_assert (DECL_P (decl2
));
2002 install_var_local (decl2
, ctx
);
2004 install_var_local (decl
, ctx
);
2007 case OMP_CLAUSE_IS_DEVICE_PTR
:
2008 decl
= OMP_CLAUSE_DECL (c
);
2011 case OMP_CLAUSE__LOOPTEMP_
:
2012 gcc_assert (is_taskreg_ctx (ctx
));
2013 decl
= OMP_CLAUSE_DECL (c
);
2014 install_var_field (decl
, false, 3, ctx
);
2015 install_var_local (decl
, ctx
);
2018 case OMP_CLAUSE_COPYPRIVATE
:
2019 case OMP_CLAUSE_COPYIN
:
2020 decl
= OMP_CLAUSE_DECL (c
);
2021 by_ref
= use_pointer_for_field (decl
, NULL
);
2022 install_var_field (decl
, by_ref
, 3, ctx
);
2025 case OMP_CLAUSE_DEFAULT
:
2026 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2029 case OMP_CLAUSE_FINAL
:
2031 case OMP_CLAUSE_NUM_THREADS
:
2032 case OMP_CLAUSE_NUM_TEAMS
:
2033 case OMP_CLAUSE_THREAD_LIMIT
:
2034 case OMP_CLAUSE_DEVICE
:
2035 case OMP_CLAUSE_SCHEDULE
:
2036 case OMP_CLAUSE_DIST_SCHEDULE
:
2037 case OMP_CLAUSE_DEPEND
:
2038 case OMP_CLAUSE_PRIORITY
:
2039 case OMP_CLAUSE_GRAINSIZE
:
2040 case OMP_CLAUSE_NUM_TASKS
:
2041 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2042 case OMP_CLAUSE_NUM_GANGS
:
2043 case OMP_CLAUSE_NUM_WORKERS
:
2044 case OMP_CLAUSE_VECTOR_LENGTH
:
2046 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2050 case OMP_CLAUSE_FROM
:
2051 case OMP_CLAUSE_MAP
:
2053 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2054 decl
= OMP_CLAUSE_DECL (c
);
2055 /* Global variables with "omp declare target" attribute
2056 don't need to be copied, the receiver side will use them
2057 directly. However, global variables with "omp declare target link"
2058 attribute need to be copied. */
2059 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2061 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2062 && (OMP_CLAUSE_MAP_KIND (c
)
2063 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2064 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2065 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2066 && varpool_node::get_create (decl
)->offloadable
2067 && !lookup_attribute ("omp declare target link",
2068 DECL_ATTRIBUTES (decl
)))
2070 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2071 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2073 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2074 not offloaded; there is nothing to map for those. */
2075 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2076 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2077 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2080 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2081 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2082 || (OMP_CLAUSE_MAP_KIND (c
)
2083 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2085 if (TREE_CODE (decl
) == COMPONENT_REF
2086 || (TREE_CODE (decl
) == INDIRECT_REF
2087 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2088 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2089 == REFERENCE_TYPE
)))
2091 if (DECL_SIZE (decl
)
2092 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2094 tree decl2
= DECL_VALUE_EXPR (decl
);
2095 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2096 decl2
= TREE_OPERAND (decl2
, 0);
2097 gcc_assert (DECL_P (decl2
));
2098 install_var_local (decl2
, ctx
);
2100 install_var_local (decl
, ctx
);
2105 if (DECL_SIZE (decl
)
2106 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2108 tree decl2
= DECL_VALUE_EXPR (decl
);
2109 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2110 decl2
= TREE_OPERAND (decl2
, 0);
2111 gcc_assert (DECL_P (decl2
));
2112 install_var_field (decl2
, true, 3, ctx
);
2113 install_var_local (decl2
, ctx
);
2114 install_var_local (decl
, ctx
);
2118 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2119 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2120 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2121 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2122 install_var_field (decl
, true, 7, ctx
);
2124 install_var_field (decl
, true, 3, ctx
,
2125 base_pointers_restrict
);
2126 if (is_gimple_omp_offloaded (ctx
->stmt
)
2127 && !OMP_CLAUSE_MAP_IN_REDUCTION (c
))
2128 install_var_local (decl
, ctx
);
2133 tree base
= get_base_address (decl
);
2134 tree nc
= OMP_CLAUSE_CHAIN (c
);
2137 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2138 && OMP_CLAUSE_DECL (nc
) == base
2139 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2140 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2142 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2143 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2149 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2150 decl
= OMP_CLAUSE_DECL (c
);
2152 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2153 (splay_tree_key
) decl
));
2155 = build_decl (OMP_CLAUSE_LOCATION (c
),
2156 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2157 SET_DECL_ALIGN (field
, TYPE_ALIGN (ptr_type_node
));
2158 insert_field_into_struct (ctx
->record_type
, field
);
2159 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2160 (splay_tree_value
) field
);
2165 case OMP_CLAUSE__GRIDDIM_
:
2168 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
2169 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
2173 case OMP_CLAUSE_NOWAIT
:
2174 case OMP_CLAUSE_ORDERED
:
2175 case OMP_CLAUSE_COLLAPSE
:
2176 case OMP_CLAUSE_UNTIED
:
2177 case OMP_CLAUSE_MERGEABLE
:
2178 case OMP_CLAUSE_PROC_BIND
:
2179 case OMP_CLAUSE_SAFELEN
:
2180 case OMP_CLAUSE_SIMDLEN
:
2181 case OMP_CLAUSE_THREADS
:
2182 case OMP_CLAUSE_SIMD
:
2183 case OMP_CLAUSE_NOGROUP
:
2184 case OMP_CLAUSE_DEFAULTMAP
:
2185 case OMP_CLAUSE_ASYNC
:
2186 case OMP_CLAUSE_WAIT
:
2187 case OMP_CLAUSE_GANG
:
2188 case OMP_CLAUSE_WORKER
:
2189 case OMP_CLAUSE_VECTOR
:
2190 case OMP_CLAUSE_INDEPENDENT
:
2191 case OMP_CLAUSE_AUTO
:
2192 case OMP_CLAUSE_SEQ
:
2195 case OMP_CLAUSE_ALIGNED
:
2196 decl
= OMP_CLAUSE_DECL (c
);
2197 if (is_global_var (decl
)
2198 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2199 install_var_local (decl
, ctx
);
2202 case OMP_CLAUSE_TILE
:
2203 case OMP_CLAUSE__CACHE_
:
2209 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2211 switch (OMP_CLAUSE_CODE (c
))
2213 case OMP_CLAUSE_LASTPRIVATE
:
2214 /* Let the corresponding firstprivate clause create
2216 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2217 scan_array_reductions
= true;
2218 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2222 case OMP_CLAUSE_FIRSTPRIVATE
:
2223 case OMP_CLAUSE_PRIVATE
:
2224 case OMP_CLAUSE_LINEAR
:
2225 case OMP_CLAUSE_IS_DEVICE_PTR
:
2226 decl
= OMP_CLAUSE_DECL (c
);
2227 if (is_variable_sized (decl
))
2229 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2230 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2231 && is_gimple_omp_offloaded (ctx
->stmt
))
2233 tree decl2
= DECL_VALUE_EXPR (decl
);
2234 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2235 decl2
= TREE_OPERAND (decl2
, 0);
2236 gcc_assert (DECL_P (decl2
));
2237 install_var_local (decl2
, ctx
);
2238 fixup_remapped_decl (decl2
, ctx
, false);
2240 install_var_local (decl
, ctx
);
2242 fixup_remapped_decl (decl
, ctx
,
2243 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2244 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2245 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2246 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2247 scan_array_reductions
= true;
2250 case OMP_CLAUSE_REDUCTION
:
2251 decl
= OMP_CLAUSE_DECL (c
);
2252 if (TREE_CODE (decl
) != MEM_REF
)
2254 if (is_variable_sized (decl
))
2255 install_var_local (decl
, ctx
);
2256 fixup_remapped_decl (decl
, ctx
, false);
2258 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2259 scan_array_reductions
= true;
2262 case OMP_CLAUSE_SHARED
:
2263 /* Ignore shared directives in teams construct. */
2264 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2266 decl
= OMP_CLAUSE_DECL (c
);
2267 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2269 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2271 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2274 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2275 install_var_field (decl
, by_ref
, 11, ctx
);
2278 fixup_remapped_decl (decl
, ctx
, false);
2281 case OMP_CLAUSE_MAP
:
2282 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2284 decl
= OMP_CLAUSE_DECL (c
);
2286 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2287 && (OMP_CLAUSE_MAP_KIND (c
)
2288 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2289 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2290 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2291 && varpool_node::get_create (decl
)->offloadable
)
2295 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2296 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2297 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2298 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2300 tree new_decl
= lookup_decl (decl
, ctx
);
2301 TREE_TYPE (new_decl
)
2302 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2304 else if (DECL_SIZE (decl
)
2305 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2307 tree decl2
= DECL_VALUE_EXPR (decl
);
2308 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2309 decl2
= TREE_OPERAND (decl2
, 0);
2310 gcc_assert (DECL_P (decl2
));
2311 fixup_remapped_decl (decl2
, ctx
, false);
2312 fixup_remapped_decl (decl
, ctx
, true);
2315 fixup_remapped_decl (decl
, ctx
, false);
2319 case OMP_CLAUSE_COPYPRIVATE
:
2320 case OMP_CLAUSE_COPYIN
:
2321 case OMP_CLAUSE_DEFAULT
:
2323 case OMP_CLAUSE_NUM_THREADS
:
2324 case OMP_CLAUSE_NUM_TEAMS
:
2325 case OMP_CLAUSE_THREAD_LIMIT
:
2326 case OMP_CLAUSE_DEVICE
:
2327 case OMP_CLAUSE_SCHEDULE
:
2328 case OMP_CLAUSE_DIST_SCHEDULE
:
2329 case OMP_CLAUSE_NOWAIT
:
2330 case OMP_CLAUSE_ORDERED
:
2331 case OMP_CLAUSE_COLLAPSE
:
2332 case OMP_CLAUSE_UNTIED
:
2333 case OMP_CLAUSE_FINAL
:
2334 case OMP_CLAUSE_MERGEABLE
:
2335 case OMP_CLAUSE_PROC_BIND
:
2336 case OMP_CLAUSE_SAFELEN
:
2337 case OMP_CLAUSE_SIMDLEN
:
2338 case OMP_CLAUSE_ALIGNED
:
2339 case OMP_CLAUSE_DEPEND
:
2340 case OMP_CLAUSE__LOOPTEMP_
:
2342 case OMP_CLAUSE_FROM
:
2343 case OMP_CLAUSE_PRIORITY
:
2344 case OMP_CLAUSE_GRAINSIZE
:
2345 case OMP_CLAUSE_NUM_TASKS
:
2346 case OMP_CLAUSE_THREADS
:
2347 case OMP_CLAUSE_SIMD
:
2348 case OMP_CLAUSE_NOGROUP
:
2349 case OMP_CLAUSE_DEFAULTMAP
:
2350 case OMP_CLAUSE_USE_DEVICE_PTR
:
2351 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2352 case OMP_CLAUSE_ASYNC
:
2353 case OMP_CLAUSE_WAIT
:
2354 case OMP_CLAUSE_NUM_GANGS
:
2355 case OMP_CLAUSE_NUM_WORKERS
:
2356 case OMP_CLAUSE_VECTOR_LENGTH
:
2357 case OMP_CLAUSE_GANG
:
2358 case OMP_CLAUSE_WORKER
:
2359 case OMP_CLAUSE_VECTOR
:
2360 case OMP_CLAUSE_INDEPENDENT
:
2361 case OMP_CLAUSE_AUTO
:
2362 case OMP_CLAUSE_SEQ
:
2363 case OMP_CLAUSE__GRIDDIM_
:
2366 case OMP_CLAUSE_TILE
:
2367 case OMP_CLAUSE__CACHE_
:
2373 gcc_checking_assert (!scan_array_reductions
2374 || !is_gimple_omp_oacc (ctx
->stmt
));
2375 if (scan_array_reductions
)
2377 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2378 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2379 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2381 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2382 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2384 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2385 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2386 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2387 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2388 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2389 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2393 /* Create a new name for omp child function. Returns an identifier. If
2394 IS_CILK_FOR is true then the suffix for the child function is
2398 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2401 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2402 return clone_function_name (current_function_decl
,
2403 task_copy
? "_omp_cpyfn" : "_omp_fn");
2406 /* Returns the type of the induction variable for the child function for
2407 _Cilk_for and the types for _high and _low variables based on TYPE. */
2410 cilk_for_check_loop_diff_type (tree type
)
2412 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2414 if (TYPE_UNSIGNED (type
))
2415 return uint32_type_node
;
2417 return integer_type_node
;
2421 if (TYPE_UNSIGNED (type
))
2422 return uint64_type_node
;
2424 return long_long_integer_type_node
;
2428 /* Build a decl for the omp child function. It'll not contain a body
2429 yet, just the bare decl. */
2432 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2434 tree decl
, type
, name
, t
;
2437 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2438 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2439 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2440 tree cilk_var_type
= NULL_TREE
;
2442 name
= create_omp_child_function_name (task_copy
,
2443 cilk_for_count
!= NULL_TREE
);
2445 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2446 ptr_type_node
, NULL_TREE
);
2447 else if (cilk_for_count
)
2449 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2450 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2451 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2452 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2455 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2457 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2459 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2462 ctx
->cb
.dst_fn
= decl
;
2464 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2466 TREE_STATIC (decl
) = 1;
2467 TREE_USED (decl
) = 1;
2468 DECL_ARTIFICIAL (decl
) = 1;
2469 DECL_IGNORED_P (decl
) = 0;
2470 TREE_PUBLIC (decl
) = 0;
2471 DECL_UNINLINABLE (decl
) = 1;
2472 DECL_EXTERNAL (decl
) = 0;
2473 DECL_CONTEXT (decl
) = NULL_TREE
;
2474 DECL_INITIAL (decl
) = make_node (BLOCK
);
2475 BLOCK_SUPERCONTEXT (DECL_INITIAL (decl
)) = decl
;
2476 if (cgraph_node::get (current_function_decl
)->offloadable
)
2477 cgraph_node::get_create (decl
)->offloadable
= 1;
2481 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2482 if (is_gimple_omp_offloaded (octx
->stmt
))
2484 cgraph_node::get_create (decl
)->offloadable
= 1;
2485 if (ENABLE_OFFLOADING
)
2486 g
->have_offload
= true;
2492 if (cgraph_node::get_create (decl
)->offloadable
2493 && !lookup_attribute ("omp declare target",
2494 DECL_ATTRIBUTES (current_function_decl
)))
2495 DECL_ATTRIBUTES (decl
)
2496 = tree_cons (get_identifier ("omp target entrypoint"),
2497 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2499 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2500 RESULT_DECL
, NULL_TREE
, void_type_node
);
2501 DECL_ARTIFICIAL (t
) = 1;
2502 DECL_IGNORED_P (t
) = 1;
2503 DECL_CONTEXT (t
) = decl
;
2504 DECL_RESULT (decl
) = t
;
2506 /* _Cilk_for's child function requires two extra parameters called
2507 __low and __high that are set the by Cilk runtime when it calls this
2511 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2512 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2513 DECL_ARTIFICIAL (t
) = 1;
2514 DECL_NAMELESS (t
) = 1;
2515 DECL_ARG_TYPE (t
) = ptr_type_node
;
2516 DECL_CONTEXT (t
) = current_function_decl
;
2518 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2519 DECL_ARGUMENTS (decl
) = t
;
2521 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2522 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2523 DECL_ARTIFICIAL (t
) = 1;
2524 DECL_NAMELESS (t
) = 1;
2525 DECL_ARG_TYPE (t
) = ptr_type_node
;
2526 DECL_CONTEXT (t
) = current_function_decl
;
2528 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2529 DECL_ARGUMENTS (decl
) = t
;
2532 tree data_name
= get_identifier (".omp_data_i");
2533 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2535 DECL_ARTIFICIAL (t
) = 1;
2536 DECL_NAMELESS (t
) = 1;
2537 DECL_ARG_TYPE (t
) = ptr_type_node
;
2538 DECL_CONTEXT (t
) = current_function_decl
;
2540 TREE_READONLY (t
) = 1;
2542 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2543 DECL_ARGUMENTS (decl
) = t
;
2545 ctx
->receiver_decl
= t
;
2548 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2549 PARM_DECL
, get_identifier (".omp_data_o"),
2551 DECL_ARTIFICIAL (t
) = 1;
2552 DECL_NAMELESS (t
) = 1;
2553 DECL_ARG_TYPE (t
) = ptr_type_node
;
2554 DECL_CONTEXT (t
) = current_function_decl
;
2556 TREE_ADDRESSABLE (t
) = 1;
2557 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2558 DECL_ARGUMENTS (decl
) = t
;
2561 /* Allocate memory for the function structure. The call to
2562 allocate_struct_function clobbers CFUN, so we need to restore
2564 push_struct_function (decl
);
2565 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2566 init_tree_ssa (cfun
);
2570 /* Callback for walk_gimple_seq. Check if combined parallel
2571 contains gimple_omp_for_combined_into_p OMP_FOR. */
2574 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2575 bool *handled_ops_p
,
2576 struct walk_stmt_info
*wi
)
2578 gimple
*stmt
= gsi_stmt (*gsi_p
);
2580 *handled_ops_p
= true;
2581 switch (gimple_code (stmt
))
2585 case GIMPLE_OMP_FOR
:
2586 if (gimple_omp_for_combined_into_p (stmt
)
2587 && gimple_omp_for_kind (stmt
)
2588 == *(const enum gf_mask
*) (wi
->info
))
2591 return integer_zero_node
;
2600 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2603 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2604 omp_context
*outer_ctx
)
2606 struct walk_stmt_info wi
;
2608 memset (&wi
, 0, sizeof (wi
));
2610 wi
.info
= (void *) &msk
;
2611 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2612 if (wi
.info
!= (void *) &msk
)
2614 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2615 struct omp_for_data fd
;
2616 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2617 /* We need two temporaries with fd.loop.v type (istart/iend)
2618 and then (fd.collapse - 1) temporaries with the same
2619 type for count2 ... countN-1 vars if not constant. */
2620 size_t count
= 2, i
;
2621 tree type
= fd
.iter_type
;
2623 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2625 count
+= fd
.collapse
- 1;
2626 /* If there are lastprivate clauses on the inner
2627 GIMPLE_OMP_FOR, add one more temporaries for the total number
2628 of iterations (product of count1 ... countN-1). */
2629 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2630 OMP_CLAUSE_LASTPRIVATE
))
2632 else if (msk
== GF_OMP_FOR_KIND_FOR
2633 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2634 OMP_CLAUSE_LASTPRIVATE
))
2637 for (i
= 0; i
< count
; i
++)
2639 tree temp
= create_tmp_var (type
);
2640 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2641 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2642 OMP_CLAUSE_DECL (c
) = temp
;
2643 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2644 gimple_omp_taskreg_set_clauses (stmt
, c
);
2649 /* Scan an OpenMP parallel directive. */
2652 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2656 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2658 /* Ignore parallel directives with empty bodies, unless there
2659 are copyin clauses. */
2661 && empty_body_p (gimple_omp_body (stmt
))
2662 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2663 OMP_CLAUSE_COPYIN
) == NULL
)
2665 gsi_replace (gsi
, gimple_build_nop (), false);
2669 if (gimple_omp_parallel_combined_p (stmt
))
2670 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2672 ctx
= new_omp_context (stmt
, outer_ctx
);
2673 taskreg_contexts
.safe_push (ctx
);
2674 if (taskreg_nesting_level
> 1)
2675 ctx
->is_nested
= true;
2676 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2677 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2678 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2679 name
= create_tmp_var_name (".omp_data_s");
2680 name
= build_decl (gimple_location (stmt
),
2681 TYPE_DECL
, name
, ctx
->record_type
);
2682 DECL_ARTIFICIAL (name
) = 1;
2683 DECL_NAMELESS (name
) = 1;
2684 TYPE_NAME (ctx
->record_type
) = name
;
2685 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2686 if (!gimple_omp_parallel_grid_phony (stmt
))
2688 create_omp_child_function (ctx
, false);
2689 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2692 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2693 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2695 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2696 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2699 /* Scan an OpenMP task directive. */
2702 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2706 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2708 /* Ignore task directives with empty bodies. */
2710 && empty_body_p (gimple_omp_body (stmt
)))
2712 gsi_replace (gsi
, gimple_build_nop (), false);
2716 if (gimple_omp_task_taskloop_p (stmt
))
2717 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2719 ctx
= new_omp_context (stmt
, outer_ctx
);
2720 taskreg_contexts
.safe_push (ctx
);
2721 if (taskreg_nesting_level
> 1)
2722 ctx
->is_nested
= true;
2723 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2724 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2725 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2726 name
= create_tmp_var_name (".omp_data_s");
2727 name
= build_decl (gimple_location (stmt
),
2728 TYPE_DECL
, name
, ctx
->record_type
);
2729 DECL_ARTIFICIAL (name
) = 1;
2730 DECL_NAMELESS (name
) = 1;
2731 TYPE_NAME (ctx
->record_type
) = name
;
2732 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2733 create_omp_child_function (ctx
, false);
2734 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2736 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2738 if (ctx
->srecord_type
)
2740 name
= create_tmp_var_name (".omp_data_a");
2741 name
= build_decl (gimple_location (stmt
),
2742 TYPE_DECL
, name
, ctx
->srecord_type
);
2743 DECL_ARTIFICIAL (name
) = 1;
2744 DECL_NAMELESS (name
) = 1;
2745 TYPE_NAME (ctx
->srecord_type
) = name
;
2746 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2747 create_omp_child_function (ctx
, true);
2750 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2752 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2754 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2755 t
= build_int_cst (long_integer_type_node
, 0);
2756 gimple_omp_task_set_arg_size (stmt
, t
);
2757 t
= build_int_cst (long_integer_type_node
, 1);
2758 gimple_omp_task_set_arg_align (stmt
, t
);
2763 /* If any decls have been made addressable during scan_omp,
2764 adjust their fields if needed, and layout record types
2765 of parallel/task constructs. */
2768 finish_taskreg_scan (omp_context
*ctx
)
2770 if (ctx
->record_type
== NULL_TREE
)
2773 /* If any task_shared_vars were needed, verify all
2774 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2775 statements if use_pointer_for_field hasn't changed
2776 because of that. If it did, update field types now. */
2777 if (task_shared_vars
)
2781 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2782 c
; c
= OMP_CLAUSE_CHAIN (c
))
2783 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2784 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2786 tree decl
= OMP_CLAUSE_DECL (c
);
2788 /* Global variables don't need to be copied,
2789 the receiver side will use them directly. */
2790 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2792 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2793 || !use_pointer_for_field (decl
, ctx
))
2795 tree field
= lookup_field (decl
, ctx
);
2796 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2797 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2799 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2800 TREE_THIS_VOLATILE (field
) = 0;
2801 DECL_USER_ALIGN (field
) = 0;
2802 SET_DECL_ALIGN (field
, TYPE_ALIGN (TREE_TYPE (field
)));
2803 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2804 SET_TYPE_ALIGN (ctx
->record_type
, DECL_ALIGN (field
));
2805 if (ctx
->srecord_type
)
2807 tree sfield
= lookup_sfield (decl
, ctx
);
2808 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2809 TREE_THIS_VOLATILE (sfield
) = 0;
2810 DECL_USER_ALIGN (sfield
) = 0;
2811 SET_DECL_ALIGN (sfield
, DECL_ALIGN (field
));
2812 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2813 SET_TYPE_ALIGN (ctx
->srecord_type
, DECL_ALIGN (sfield
));
2818 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2820 layout_type (ctx
->record_type
);
2821 fixup_child_record_type (ctx
);
2825 location_t loc
= gimple_location (ctx
->stmt
);
2826 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2827 /* Move VLA fields to the end. */
2828 p
= &TYPE_FIELDS (ctx
->record_type
);
2830 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2831 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2834 *p
= TREE_CHAIN (*p
);
2835 TREE_CHAIN (*q
) = NULL_TREE
;
2836 q
= &TREE_CHAIN (*q
);
2839 p
= &DECL_CHAIN (*p
);
2841 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2843 /* Move fields corresponding to first and second _looptemp_
2844 clause first. There are filled by GOMP_taskloop
2845 and thus need to be in specific positions. */
2846 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2847 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2848 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2849 OMP_CLAUSE__LOOPTEMP_
);
2850 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2851 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2852 p
= &TYPE_FIELDS (ctx
->record_type
);
2854 if (*p
== f1
|| *p
== f2
)
2855 *p
= DECL_CHAIN (*p
);
2857 p
= &DECL_CHAIN (*p
);
2858 DECL_CHAIN (f1
) = f2
;
2859 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2860 TYPE_FIELDS (ctx
->record_type
) = f1
;
2861 if (ctx
->srecord_type
)
2863 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2864 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2865 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2867 if (*p
== f1
|| *p
== f2
)
2868 *p
= DECL_CHAIN (*p
);
2870 p
= &DECL_CHAIN (*p
);
2871 DECL_CHAIN (f1
) = f2
;
2872 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2873 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2876 layout_type (ctx
->record_type
);
2877 fixup_child_record_type (ctx
);
2878 if (ctx
->srecord_type
)
2879 layout_type (ctx
->srecord_type
);
2880 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2881 TYPE_SIZE_UNIT (ctx
->record_type
));
2882 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2883 t
= build_int_cst (long_integer_type_node
,
2884 TYPE_ALIGN_UNIT (ctx
->record_type
));
2885 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2889 /* Find the enclosing offload context. */
2891 static omp_context
*
2892 enclosing_target_ctx (omp_context
*ctx
)
2894 for (; ctx
; ctx
= ctx
->outer
)
2895 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2901 /* Return true if ctx is part of an oacc kernels region. */
2904 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2906 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2908 gimple
*stmt
= ctx
->stmt
;
2909 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2910 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2917 /* Check the parallelism clauses inside a kernels regions.
2918 Until kernels handling moves to use the same loop indirection
2919 scheme as parallel, we need to do this checking early. */
2922 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2924 bool checking
= true;
2925 unsigned outer_mask
= 0;
2926 unsigned this_mask
= 0;
2927 bool has_seq
= false, has_auto
= false;
2930 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2934 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2936 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2939 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2941 switch (OMP_CLAUSE_CODE (c
))
2943 case OMP_CLAUSE_GANG
:
2944 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2946 case OMP_CLAUSE_WORKER
:
2947 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2949 case OMP_CLAUSE_VECTOR
:
2950 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2952 case OMP_CLAUSE_SEQ
:
2955 case OMP_CLAUSE_AUTO
:
2965 if (has_seq
&& (this_mask
|| has_auto
))
2966 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2967 " OpenACC loop specifiers");
2968 else if (has_auto
&& this_mask
)
2969 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2970 " OpenACC loop specifiers");
2972 if (this_mask
& outer_mask
)
2973 error_at (gimple_location (stmt
), "inner loop uses same"
2974 " OpenACC parallelism as containing loop");
2977 return outer_mask
| this_mask
;
2980 /* Scan a GIMPLE_OMP_FOR. */
2983 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2987 tree clauses
= gimple_omp_for_clauses (stmt
);
2989 ctx
= new_omp_context (stmt
, outer_ctx
);
2991 if (is_gimple_omp_oacc (stmt
))
2993 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2995 if (!tgt
|| is_oacc_parallel (tgt
))
2996 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2998 char const *check
= NULL
;
3000 switch (OMP_CLAUSE_CODE (c
))
3002 case OMP_CLAUSE_GANG
:
3006 case OMP_CLAUSE_WORKER
:
3010 case OMP_CLAUSE_VECTOR
:
3018 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3019 error_at (gimple_location (stmt
),
3020 "argument not permitted on %qs clause in"
3021 " OpenACC %<parallel%>", check
);
3024 if (tgt
&& is_oacc_kernels (tgt
))
3026 /* Strip out reductions, as they are not handled yet. */
3027 tree
*prev_ptr
= &clauses
;
3029 while (tree probe
= *prev_ptr
)
3031 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3033 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3034 *prev_ptr
= *next_ptr
;
3036 prev_ptr
= next_ptr
;
3039 gimple_omp_for_set_clauses (stmt
, clauses
);
3040 check_oacc_kernel_gwv (stmt
, ctx
);
3044 scan_sharing_clauses (clauses
, ctx
);
3046 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3047 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3049 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3050 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3051 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3052 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3054 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3057 /* Scan an OpenMP sections directive. */
3060 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3064 ctx
= new_omp_context (stmt
, outer_ctx
);
3065 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3066 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3069 /* Scan an OpenMP single directive. */
3072 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3077 ctx
= new_omp_context (stmt
, outer_ctx
);
3078 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3079 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3080 name
= create_tmp_var_name (".omp_copy_s");
3081 name
= build_decl (gimple_location (stmt
),
3082 TYPE_DECL
, name
, ctx
->record_type
);
3083 TYPE_NAME (ctx
->record_type
) = name
;
3085 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3086 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3088 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3089 ctx
->record_type
= NULL
;
3091 layout_type (ctx
->record_type
);
3094 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3095 used in the corresponding offloaded function are restrict. */
3098 omp_target_base_pointers_restrict_p (tree clauses
)
3100 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3102 if (flag_openacc
== 0)
3105 /* I. Basic example:
3109 unsigned int a[2], b[2];
3111 #pragma acc kernels \
3120 After gimplification, we have:
3122 #pragma omp target oacc_kernels \
3123 map(force_from:a [len: 8]) \
3124 map(force_from:b [len: 8])
3130 Because both mappings have the force prefix, we know that they will be
3131 allocated when calling the corresponding offloaded function, which means we
3132 can mark the base pointers for a and b in the offloaded function as
3136 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3138 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
3141 switch (OMP_CLAUSE_MAP_KIND (c
))
3143 case GOMP_MAP_FORCE_ALLOC
:
3144 case GOMP_MAP_FORCE_TO
:
3145 case GOMP_MAP_FORCE_FROM
:
3146 case GOMP_MAP_FORCE_TOFROM
:
3156 /* Scan a GIMPLE_OMP_TARGET. */
3159 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3163 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3164 tree clauses
= gimple_omp_target_clauses (stmt
);
3166 ctx
= new_omp_context (stmt
, outer_ctx
);
3167 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3168 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3169 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3170 name
= create_tmp_var_name (".omp_data_t");
3171 name
= build_decl (gimple_location (stmt
),
3172 TYPE_DECL
, name
, ctx
->record_type
);
3173 DECL_ARTIFICIAL (name
) = 1;
3174 DECL_NAMELESS (name
) = 1;
3175 TYPE_NAME (ctx
->record_type
) = name
;
3176 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3178 bool base_pointers_restrict
= false;
3181 create_omp_child_function (ctx
, false);
3182 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3184 base_pointers_restrict
= omp_target_base_pointers_restrict_p (clauses
);
3185 if (base_pointers_restrict
3186 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3188 "Base pointers in offloaded function are restrict\n");
3191 scan_sharing_clauses (clauses
, ctx
, base_pointers_restrict
);
3192 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3194 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3195 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3198 TYPE_FIELDS (ctx
->record_type
)
3199 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3202 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3203 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3205 field
= DECL_CHAIN (field
))
3206 gcc_assert (DECL_ALIGN (field
) == align
);
3208 layout_type (ctx
->record_type
);
3210 fixup_child_record_type (ctx
);
3214 /* Scan an OpenMP teams directive. */
3217 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3219 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3220 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3221 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3224 /* Check nesting restrictions. */
3226 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3230 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
3231 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3232 the original copy of its contents. */
3235 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3236 inside an OpenACC CTX. */
3237 if (!(is_gimple_omp (stmt
)
3238 && is_gimple_omp_oacc (stmt
))
3239 /* Except for atomic codes that we share with OpenMP. */
3240 && !(gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3241 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3243 if (get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3245 error_at (gimple_location (stmt
),
3246 "non-OpenACC construct inside of OpenACC routine");
3250 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3251 if (is_gimple_omp (octx
->stmt
)
3252 && is_gimple_omp_oacc (octx
->stmt
))
3254 error_at (gimple_location (stmt
),
3255 "non-OpenACC construct inside of OpenACC region");
3262 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3263 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3266 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3268 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3269 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3271 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3272 && (ctx
->outer
== NULL
3273 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3274 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3275 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3276 != GF_OMP_FOR_KIND_FOR
)
3277 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3279 error_at (gimple_location (stmt
),
3280 "%<ordered simd threads%> must be closely "
3281 "nested inside of %<for simd%> region");
3287 error_at (gimple_location (stmt
),
3288 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3289 " may not be nested inside %<simd%> region");
3292 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3294 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3295 || (gimple_omp_for_kind (stmt
)
3296 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3297 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3299 error_at (gimple_location (stmt
),
3300 "only %<distribute%> or %<parallel%> regions are "
3301 "allowed to be strictly nested inside %<teams%> "
3307 switch (gimple_code (stmt
))
3309 case GIMPLE_OMP_FOR
:
3310 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3312 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3314 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3316 error_at (gimple_location (stmt
),
3317 "%<distribute%> region must be strictly nested "
3318 "inside %<teams%> construct");
3323 /* We split taskloop into task and nested taskloop in it. */
3324 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3326 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3331 switch (gimple_code (ctx
->stmt
))
3333 case GIMPLE_OMP_FOR
:
3334 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3335 == GF_OMP_FOR_KIND_OACC_LOOP
);
3338 case GIMPLE_OMP_TARGET
:
3339 switch (gimple_omp_target_kind (ctx
->stmt
))
3341 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3342 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3353 else if (get_oacc_fn_attrib (current_function_decl
))
3357 error_at (gimple_location (stmt
),
3358 "OpenACC loop directive must be associated with"
3359 " an OpenACC compute region");
3365 if (is_gimple_call (stmt
)
3366 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3367 == BUILT_IN_GOMP_CANCEL
3368 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3369 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3371 const char *bad
= NULL
;
3372 const char *kind
= NULL
;
3373 const char *construct
3374 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3375 == BUILT_IN_GOMP_CANCEL
)
3376 ? "#pragma omp cancel"
3377 : "#pragma omp cancellation point";
3380 error_at (gimple_location (stmt
), "orphaned %qs construct",
3384 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3385 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3389 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3390 bad
= "#pragma omp parallel";
3391 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3392 == BUILT_IN_GOMP_CANCEL
3393 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3394 ctx
->cancellable
= true;
3398 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3399 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3400 bad
= "#pragma omp for";
3401 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3402 == BUILT_IN_GOMP_CANCEL
3403 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3405 ctx
->cancellable
= true;
3406 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3408 warning_at (gimple_location (stmt
), 0,
3409 "%<#pragma omp cancel for%> inside "
3410 "%<nowait%> for construct");
3411 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3412 OMP_CLAUSE_ORDERED
))
3413 warning_at (gimple_location (stmt
), 0,
3414 "%<#pragma omp cancel for%> inside "
3415 "%<ordered%> for construct");
3420 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3421 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3422 bad
= "#pragma omp sections";
3423 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3424 == BUILT_IN_GOMP_CANCEL
3425 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3427 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3429 ctx
->cancellable
= true;
3430 if (find_omp_clause (gimple_omp_sections_clauses
3433 warning_at (gimple_location (stmt
), 0,
3434 "%<#pragma omp cancel sections%> inside "
3435 "%<nowait%> sections construct");
3439 gcc_assert (ctx
->outer
3440 && gimple_code (ctx
->outer
->stmt
)
3441 == GIMPLE_OMP_SECTIONS
);
3442 ctx
->outer
->cancellable
= true;
3443 if (find_omp_clause (gimple_omp_sections_clauses
3446 warning_at (gimple_location (stmt
), 0,
3447 "%<#pragma omp cancel sections%> inside "
3448 "%<nowait%> sections construct");
3454 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3455 bad
= "#pragma omp task";
3458 for (omp_context
*octx
= ctx
->outer
;
3459 octx
; octx
= octx
->outer
)
3461 switch (gimple_code (octx
->stmt
))
3463 case GIMPLE_OMP_TASKGROUP
:
3465 case GIMPLE_OMP_TARGET
:
3466 if (gimple_omp_target_kind (octx
->stmt
)
3467 != GF_OMP_TARGET_KIND_REGION
)
3470 case GIMPLE_OMP_PARALLEL
:
3471 case GIMPLE_OMP_TEAMS
:
3472 error_at (gimple_location (stmt
),
3473 "%<%s taskgroup%> construct not closely "
3474 "nested inside of %<taskgroup%> region",
3482 ctx
->cancellable
= true;
3487 error_at (gimple_location (stmt
), "invalid arguments");
3492 error_at (gimple_location (stmt
),
3493 "%<%s %s%> construct not closely nested inside of %qs",
3494 construct
, kind
, bad
);
3499 case GIMPLE_OMP_SECTIONS
:
3500 case GIMPLE_OMP_SINGLE
:
3501 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3502 switch (gimple_code (ctx
->stmt
))
3504 case GIMPLE_OMP_FOR
:
3505 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3506 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3509 case GIMPLE_OMP_SECTIONS
:
3510 case GIMPLE_OMP_SINGLE
:
3511 case GIMPLE_OMP_ORDERED
:
3512 case GIMPLE_OMP_MASTER
:
3513 case GIMPLE_OMP_TASK
:
3514 case GIMPLE_OMP_CRITICAL
:
3515 if (is_gimple_call (stmt
))
3517 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3518 != BUILT_IN_GOMP_BARRIER
)
3520 error_at (gimple_location (stmt
),
3521 "barrier region may not be closely nested inside "
3522 "of work-sharing, %<critical%>, %<ordered%>, "
3523 "%<master%>, explicit %<task%> or %<taskloop%> "
3527 error_at (gimple_location (stmt
),
3528 "work-sharing region may not be closely nested inside "
3529 "of work-sharing, %<critical%>, %<ordered%>, "
3530 "%<master%>, explicit %<task%> or %<taskloop%> region");
3532 case GIMPLE_OMP_PARALLEL
:
3533 case GIMPLE_OMP_TEAMS
:
3535 case GIMPLE_OMP_TARGET
:
3536 if (gimple_omp_target_kind (ctx
->stmt
)
3537 == GF_OMP_TARGET_KIND_REGION
)
3544 case GIMPLE_OMP_MASTER
:
3545 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3546 switch (gimple_code (ctx
->stmt
))
3548 case GIMPLE_OMP_FOR
:
3549 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3550 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3553 case GIMPLE_OMP_SECTIONS
:
3554 case GIMPLE_OMP_SINGLE
:
3555 case GIMPLE_OMP_TASK
:
3556 error_at (gimple_location (stmt
),
3557 "%<master%> region may not be closely nested inside "
3558 "of work-sharing, explicit %<task%> or %<taskloop%> "
3561 case GIMPLE_OMP_PARALLEL
:
3562 case GIMPLE_OMP_TEAMS
:
3564 case GIMPLE_OMP_TARGET
:
3565 if (gimple_omp_target_kind (ctx
->stmt
)
3566 == GF_OMP_TARGET_KIND_REGION
)
3573 case GIMPLE_OMP_TASK
:
3574 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3575 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3576 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3577 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3579 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3580 error_at (OMP_CLAUSE_LOCATION (c
),
3581 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3582 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3586 case GIMPLE_OMP_ORDERED
:
3587 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3588 c
; c
= OMP_CLAUSE_CHAIN (c
))
3590 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3592 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3593 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3596 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3597 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3598 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3601 /* Look for containing ordered(N) loop. */
3603 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3605 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3606 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3608 error_at (OMP_CLAUSE_LOCATION (c
),
3609 "%<ordered%> construct with %<depend%> clause "
3610 "must be closely nested inside an %<ordered%> "
3614 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3616 error_at (OMP_CLAUSE_LOCATION (c
),
3617 "%<ordered%> construct with %<depend%> clause "
3618 "must be closely nested inside a loop with "
3619 "%<ordered%> clause with a parameter");
3625 error_at (OMP_CLAUSE_LOCATION (c
),
3626 "invalid depend kind in omp %<ordered%> %<depend%>");
3630 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3631 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3633 /* ordered simd must be closely nested inside of simd region,
3634 and simd region must not encounter constructs other than
3635 ordered simd, therefore ordered simd may be either orphaned,
3636 or ctx->stmt must be simd. The latter case is handled already
3640 error_at (gimple_location (stmt
),
3641 "%<ordered%> %<simd%> must be closely nested inside "
3646 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3647 switch (gimple_code (ctx
->stmt
))
3649 case GIMPLE_OMP_CRITICAL
:
3650 case GIMPLE_OMP_TASK
:
3651 case GIMPLE_OMP_ORDERED
:
3652 ordered_in_taskloop
:
3653 error_at (gimple_location (stmt
),
3654 "%<ordered%> region may not be closely nested inside "
3655 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3656 "%<taskloop%> region");
3658 case GIMPLE_OMP_FOR
:
3659 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3660 goto ordered_in_taskloop
;
3661 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3662 OMP_CLAUSE_ORDERED
) == NULL
)
3664 error_at (gimple_location (stmt
),
3665 "%<ordered%> region must be closely nested inside "
3666 "a loop region with an %<ordered%> clause");
3670 case GIMPLE_OMP_TARGET
:
3671 if (gimple_omp_target_kind (ctx
->stmt
)
3672 != GF_OMP_TARGET_KIND_REGION
)
3675 case GIMPLE_OMP_PARALLEL
:
3676 case GIMPLE_OMP_TEAMS
:
3677 error_at (gimple_location (stmt
),
3678 "%<ordered%> region must be closely nested inside "
3679 "a loop region with an %<ordered%> clause");
3685 case GIMPLE_OMP_CRITICAL
:
3688 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3689 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3690 if (gomp_critical
*other_crit
3691 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3692 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3694 error_at (gimple_location (stmt
),
3695 "%<critical%> region may not be nested inside "
3696 "a %<critical%> region with the same name");
3701 case GIMPLE_OMP_TEAMS
:
3703 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3704 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3706 error_at (gimple_location (stmt
),
3707 "%<teams%> construct not closely nested inside of "
3708 "%<target%> construct");
3712 case GIMPLE_OMP_TARGET
:
3713 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3714 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3715 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3716 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3718 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3719 error_at (OMP_CLAUSE_LOCATION (c
),
3720 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3721 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3724 if (is_gimple_omp_offloaded (stmt
)
3725 && get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3727 error_at (gimple_location (stmt
),
3728 "OpenACC region inside of OpenACC routine, nested "
3729 "parallelism not supported yet");
3732 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3734 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3736 if (is_gimple_omp (stmt
)
3737 && is_gimple_omp_oacc (stmt
)
3738 && is_gimple_omp (ctx
->stmt
))
3740 error_at (gimple_location (stmt
),
3741 "OpenACC construct inside of non-OpenACC region");
3747 const char *stmt_name
, *ctx_stmt_name
;
3748 switch (gimple_omp_target_kind (stmt
))
3750 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3751 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3752 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3753 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3754 stmt_name
= "target enter data"; break;
3755 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3756 stmt_name
= "target exit data"; break;
3757 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3758 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3759 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3760 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3761 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3762 stmt_name
= "enter/exit data"; break;
3763 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3765 default: gcc_unreachable ();
3767 switch (gimple_omp_target_kind (ctx
->stmt
))
3769 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3770 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3771 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3772 ctx_stmt_name
= "parallel"; break;
3773 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3774 ctx_stmt_name
= "kernels"; break;
3775 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3776 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3777 ctx_stmt_name
= "host_data"; break;
3778 default: gcc_unreachable ();
3781 /* OpenACC/OpenMP mismatch? */
3782 if (is_gimple_omp_oacc (stmt
)
3783 != is_gimple_omp_oacc (ctx
->stmt
))
3785 error_at (gimple_location (stmt
),
3786 "%s %qs construct inside of %s %qs region",
3787 (is_gimple_omp_oacc (stmt
)
3788 ? "OpenACC" : "OpenMP"), stmt_name
,
3789 (is_gimple_omp_oacc (ctx
->stmt
)
3790 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3793 if (is_gimple_omp_offloaded (ctx
->stmt
))
3795 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3796 if (is_gimple_omp_oacc (ctx
->stmt
))
3798 error_at (gimple_location (stmt
),
3799 "%qs construct inside of %qs region",
3800 stmt_name
, ctx_stmt_name
);
3805 warning_at (gimple_location (stmt
), 0,
3806 "%qs construct inside of %qs region",
3807 stmt_name
, ctx_stmt_name
);
3819 /* Helper function scan_omp.
3821 Callback for walk_tree or operators in walk_gimple_stmt used to
3822 scan for OMP directives in TP. */
3825 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3827 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3828 omp_context
*ctx
= (omp_context
*) wi
->info
;
3831 switch (TREE_CODE (t
))
3839 tree repl
= remap_decl (t
, &ctx
->cb
);
3840 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3846 if (ctx
&& TYPE_P (t
))
3847 *tp
= remap_type (t
, &ctx
->cb
);
3848 else if (!DECL_P (t
))
3853 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3854 if (tem
!= TREE_TYPE (t
))
3856 if (TREE_CODE (t
) == INTEGER_CST
)
3857 *tp
= wide_int_to_tree (tem
, t
);
3859 TREE_TYPE (t
) = tem
;
3869 /* Return true if FNDECL is a setjmp or a longjmp. */
3872 setjmp_or_longjmp_p (const_tree fndecl
)
3874 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3875 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3876 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3879 tree declname
= DECL_NAME (fndecl
);
3882 const char *name
= IDENTIFIER_POINTER (declname
);
3883 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3887 /* Helper function for scan_omp.
3889 Callback for walk_gimple_stmt used to scan for OMP directives in
3890 the current statement in GSI. */
3893 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3894 struct walk_stmt_info
*wi
)
3896 gimple
*stmt
= gsi_stmt (*gsi
);
3897 omp_context
*ctx
= (omp_context
*) wi
->info
;
3899 if (gimple_has_location (stmt
))
3900 input_location
= gimple_location (stmt
);
3902 /* Check the nesting restrictions. */
3903 bool remove
= false;
3904 if (is_gimple_omp (stmt
))
3905 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3906 else if (is_gimple_call (stmt
))
3908 tree fndecl
= gimple_call_fndecl (stmt
);
3911 if (setjmp_or_longjmp_p (fndecl
)
3913 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3914 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3917 error_at (gimple_location (stmt
),
3918 "setjmp/longjmp inside simd construct");
3920 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3921 switch (DECL_FUNCTION_CODE (fndecl
))
3923 case BUILT_IN_GOMP_BARRIER
:
3924 case BUILT_IN_GOMP_CANCEL
:
3925 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3926 case BUILT_IN_GOMP_TASKYIELD
:
3927 case BUILT_IN_GOMP_TASKWAIT
:
3928 case BUILT_IN_GOMP_TASKGROUP_START
:
3929 case BUILT_IN_GOMP_TASKGROUP_END
:
3930 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3939 stmt
= gimple_build_nop ();
3940 gsi_replace (gsi
, stmt
, false);
3943 *handled_ops_p
= true;
3945 switch (gimple_code (stmt
))
3947 case GIMPLE_OMP_PARALLEL
:
3948 taskreg_nesting_level
++;
3949 scan_omp_parallel (gsi
, ctx
);
3950 taskreg_nesting_level
--;
3953 case GIMPLE_OMP_TASK
:
3954 taskreg_nesting_level
++;
3955 scan_omp_task (gsi
, ctx
);
3956 taskreg_nesting_level
--;
3959 case GIMPLE_OMP_FOR
:
3960 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3963 case GIMPLE_OMP_SECTIONS
:
3964 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3967 case GIMPLE_OMP_SINGLE
:
3968 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3971 case GIMPLE_OMP_SECTION
:
3972 case GIMPLE_OMP_MASTER
:
3973 case GIMPLE_OMP_TASKGROUP
:
3974 case GIMPLE_OMP_ORDERED
:
3975 case GIMPLE_OMP_CRITICAL
:
3976 case GIMPLE_OMP_GRID_BODY
:
3977 ctx
= new_omp_context (stmt
, ctx
);
3978 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3981 case GIMPLE_OMP_TARGET
:
3982 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3985 case GIMPLE_OMP_TEAMS
:
3986 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3993 *handled_ops_p
= false;
3995 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3997 var
= DECL_CHAIN (var
))
3998 insert_decl_map (&ctx
->cb
, var
, var
);
4002 *handled_ops_p
= false;
4010 /* Scan all the statements starting at the current statement. CTX
4011 contains context information about the OMP directives and
4012 clauses found during the scan. */
4015 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
4017 location_t saved_location
;
4018 struct walk_stmt_info wi
;
4020 memset (&wi
, 0, sizeof (wi
));
4022 wi
.want_locations
= true;
4024 saved_location
= input_location
;
4025 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
4026 input_location
= saved_location
;
4029 /* Re-gimplification and code generation routines. */
4031 /* Build a call to GOMP_barrier. */
4034 build_omp_barrier (tree lhs
)
4036 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
4037 : BUILT_IN_GOMP_BARRIER
);
4038 gcall
*g
= gimple_build_call (fndecl
, 0);
4040 gimple_call_set_lhs (g
, lhs
);
4044 /* If a context was created for STMT when it was scanned, return it. */
4046 static omp_context
*
4047 maybe_lookup_ctx (gimple
*stmt
)
4050 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
4051 return n
? (omp_context
*) n
->value
: NULL
;
4055 /* Find the mapping for DECL in CTX or the immediately enclosing
4056 context that has a mapping for DECL.
4058 If CTX is a nested parallel directive, we may have to use the decl
4059 mappings created in CTX's parent context. Suppose that we have the
4060 following parallel nesting (variable UIDs showed for clarity):
4063 #omp parallel shared(iD.1562) -> outer parallel
4064 iD.1562 = iD.1562 + 1;
4066 #omp parallel shared (iD.1562) -> inner parallel
4067 iD.1562 = iD.1562 - 1;
4069 Each parallel structure will create a distinct .omp_data_s structure
4070 for copying iD.1562 in/out of the directive:
4072 outer parallel .omp_data_s.1.i -> iD.1562
4073 inner parallel .omp_data_s.2.i -> iD.1562
4075 A shared variable mapping will produce a copy-out operation before
4076 the parallel directive and a copy-in operation after it. So, in
4077 this case we would have:
4080 .omp_data_o.1.i = iD.1562;
4081 #omp parallel shared(iD.1562) -> outer parallel
4082 .omp_data_i.1 = &.omp_data_o.1
4083 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4085 .omp_data_o.2.i = iD.1562; -> **
4086 #omp parallel shared(iD.1562) -> inner parallel
4087 .omp_data_i.2 = &.omp_data_o.2
4088 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4091 ** This is a problem. The symbol iD.1562 cannot be referenced
4092 inside the body of the outer parallel region. But since we are
4093 emitting this copy operation while expanding the inner parallel
4094 directive, we need to access the CTX structure of the outer
4095 parallel directive to get the correct mapping:
4097 .omp_data_o.2.i = .omp_data_i.1->i
4099 Since there may be other workshare or parallel directives enclosing
4100 the parallel directive, it may be necessary to walk up the context
4101 parent chain. This is not a problem in general because nested
4102 parallelism happens only rarely. */
4105 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4110 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4111 t
= maybe_lookup_decl (decl
, up
);
4113 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
4115 return t
? t
: decl
;
4119 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4120 in outer contexts. */
4123 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4128 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4129 t
= maybe_lookup_decl (decl
, up
);
4131 return t
? t
: decl
;
4135 /* Construct the initialization value for reduction operation OP. */
4138 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4147 case TRUTH_ORIF_EXPR
:
4148 case TRUTH_XOR_EXPR
:
4150 return build_zero_cst (type
);
4153 case TRUTH_AND_EXPR
:
4154 case TRUTH_ANDIF_EXPR
:
4156 return fold_convert_loc (loc
, type
, integer_one_node
);
4159 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4162 if (SCALAR_FLOAT_TYPE_P (type
))
4164 REAL_VALUE_TYPE max
, min
;
4165 if (HONOR_INFINITIES (type
))
4168 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4171 real_maxval (&min
, 1, TYPE_MODE (type
));
4172 return build_real (type
, min
);
4174 else if (POINTER_TYPE_P (type
))
4177 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4178 return wide_int_to_tree (type
, min
);
4182 gcc_assert (INTEGRAL_TYPE_P (type
));
4183 return TYPE_MIN_VALUE (type
);
4187 if (SCALAR_FLOAT_TYPE_P (type
))
4189 REAL_VALUE_TYPE max
;
4190 if (HONOR_INFINITIES (type
))
4193 real_maxval (&max
, 0, TYPE_MODE (type
));
4194 return build_real (type
, max
);
4196 else if (POINTER_TYPE_P (type
))
4199 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4200 return wide_int_to_tree (type
, max
);
4204 gcc_assert (INTEGRAL_TYPE_P (type
));
4205 return TYPE_MAX_VALUE (type
);
4213 /* Construct the initialization value for reduction CLAUSE. */
4216 omp_reduction_init (tree clause
, tree type
)
4218 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4219 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4222 /* Return alignment to be assumed for var in CLAUSE, which should be
4223 OMP_CLAUSE_ALIGNED. */
4226 omp_clause_aligned_alignment (tree clause
)
4228 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4229 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4231 /* Otherwise return implementation defined alignment. */
4232 unsigned int al
= 1;
4233 machine_mode mode
, vmode
;
4234 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4236 vs
= 1 << floor_log2 (vs
);
4237 static enum mode_class classes
[]
4238 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4239 for (int i
= 0; i
< 4; i
+= 2)
4240 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4242 mode
= GET_MODE_WIDER_MODE (mode
))
4244 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4245 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4248 && GET_MODE_SIZE (vmode
) < vs
4249 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4250 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4252 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4253 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4255 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4256 / GET_MODE_SIZE (mode
));
4257 if (TYPE_MODE (type
) != vmode
)
4259 if (TYPE_ALIGN_UNIT (type
) > al
)
4260 al
= TYPE_ALIGN_UNIT (type
);
4262 return build_int_cst (integer_type_node
, al
);
4265 /* Return maximum possible vectorization factor for the target. */
4272 || !flag_tree_loop_optimize
4273 || (!flag_tree_loop_vectorize
4274 && (global_options_set
.x_flag_tree_loop_vectorize
4275 || global_options_set
.x_flag_tree_vectorize
)))
4278 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4281 vs
= 1 << floor_log2 (vs
);
4284 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4285 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4286 return GET_MODE_NUNITS (vqimode
);
4290 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4294 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4295 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4299 max_vf
= omp_max_vf ();
4302 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4303 OMP_CLAUSE_SAFELEN
);
4304 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4306 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4308 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4312 idx
= create_tmp_var (unsigned_type_node
);
4313 lane
= create_tmp_var (unsigned_type_node
);
4319 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4320 tree avar
= create_tmp_var_raw (atype
);
4321 if (TREE_ADDRESSABLE (new_var
))
4322 TREE_ADDRESSABLE (avar
) = 1;
4323 DECL_ATTRIBUTES (avar
)
4324 = tree_cons (get_identifier ("omp simd array"), NULL
,
4325 DECL_ATTRIBUTES (avar
));
4326 gimple_add_tmp_var (avar
);
4327 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4328 NULL_TREE
, NULL_TREE
);
4329 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4330 NULL_TREE
, NULL_TREE
);
4331 if (DECL_P (new_var
))
4333 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4334 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4339 /* Helper function of lower_rec_input_clauses. For a reference
4340 in simd reduction, add an underlying variable it will reference. */
4343 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4345 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4346 if (TREE_CONSTANT (z
))
4348 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4349 get_name (new_vard
));
4350 gimple_add_tmp_var (z
);
4351 TREE_ADDRESSABLE (z
) = 1;
4352 z
= build_fold_addr_expr_loc (loc
, z
);
4353 gimplify_assign (new_vard
, z
, ilist
);
4357 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4358 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4359 private variables. Initialization statements go in ILIST, while calls
4360 to destructors go in DLIST. */
4363 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4364 omp_context
*ctx
, struct omp_for_data
*fd
)
4366 tree c
, dtor
, copyin_seq
, x
, ptr
;
4367 bool copyin_by_ref
= false;
4368 bool lastprivate_firstprivate
= false;
4369 bool reduction_omp_orig_ref
= false;
4371 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4372 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4374 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4375 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4376 gimple_seq llist
[2] = { NULL
, NULL
};
4380 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4381 with data sharing clauses referencing variable sized vars. That
4382 is unnecessarily hard to support and very unlikely to result in
4383 vectorized code anyway. */
4385 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4386 switch (OMP_CLAUSE_CODE (c
))
4388 case OMP_CLAUSE_LINEAR
:
4389 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4392 case OMP_CLAUSE_PRIVATE
:
4393 case OMP_CLAUSE_FIRSTPRIVATE
:
4394 case OMP_CLAUSE_LASTPRIVATE
:
4395 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4398 case OMP_CLAUSE_REDUCTION
:
4399 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4400 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4407 /* Do all the fixed sized types in the first pass, and the variable sized
4408 types in the second pass. This makes sure that the scalar arguments to
4409 the variable sized types are processed before we use them in the
4410 variable sized operations. */
4411 for (pass
= 0; pass
< 2; ++pass
)
4413 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4415 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4418 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4422 case OMP_CLAUSE_PRIVATE
:
4423 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4426 case OMP_CLAUSE_SHARED
:
4427 /* Ignore shared directives in teams construct. */
4428 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4430 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4432 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4433 || is_global_var (OMP_CLAUSE_DECL (c
)));
4436 case OMP_CLAUSE_FIRSTPRIVATE
:
4437 case OMP_CLAUSE_COPYIN
:
4439 case OMP_CLAUSE_LINEAR
:
4440 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4441 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4442 lastprivate_firstprivate
= true;
4444 case OMP_CLAUSE_REDUCTION
:
4445 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4446 reduction_omp_orig_ref
= true;
4448 case OMP_CLAUSE__LOOPTEMP_
:
4449 /* Handle _looptemp_ clauses only on parallel/task. */
4453 case OMP_CLAUSE_LASTPRIVATE
:
4454 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4456 lastprivate_firstprivate
= true;
4457 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4460 /* Even without corresponding firstprivate, if
4461 decl is Fortran allocatable, it needs outer var
4464 && lang_hooks
.decls
.omp_private_outer_ref
4465 (OMP_CLAUSE_DECL (c
)))
4466 lastprivate_firstprivate
= true;
4468 case OMP_CLAUSE_ALIGNED
:
4471 var
= OMP_CLAUSE_DECL (c
);
4472 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4473 && !is_global_var (var
))
4475 new_var
= maybe_lookup_decl (var
, ctx
);
4476 if (new_var
== NULL_TREE
)
4477 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4478 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4479 tree alarg
= omp_clause_aligned_alignment (c
);
4480 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4481 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
, alarg
);
4482 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4483 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4484 gimplify_and_add (x
, ilist
);
4486 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4487 && is_global_var (var
))
4489 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4490 new_var
= lookup_decl (var
, ctx
);
4491 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4492 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4493 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4494 tree alarg
= omp_clause_aligned_alignment (c
);
4495 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4496 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
, alarg
);
4497 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4498 x
= create_tmp_var (ptype
);
4499 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4500 gimplify_and_add (t
, ilist
);
4501 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4502 SET_DECL_VALUE_EXPR (new_var
, t
);
4503 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4510 new_var
= var
= OMP_CLAUSE_DECL (c
);
4511 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4513 var
= TREE_OPERAND (var
, 0);
4514 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4515 var
= TREE_OPERAND (var
, 0);
4516 if (TREE_CODE (var
) == INDIRECT_REF
4517 || TREE_CODE (var
) == ADDR_EXPR
)
4518 var
= TREE_OPERAND (var
, 0);
4519 if (is_variable_sized (var
))
4521 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4522 var
= DECL_VALUE_EXPR (var
);
4523 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4524 var
= TREE_OPERAND (var
, 0);
4525 gcc_assert (DECL_P (var
));
4529 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4530 new_var
= lookup_decl (var
, ctx
);
4532 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4537 /* C/C++ array section reductions. */
4538 else if (c_kind
== OMP_CLAUSE_REDUCTION
4539 && var
!= OMP_CLAUSE_DECL (c
))
4544 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4545 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4546 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4548 tree b
= TREE_OPERAND (orig_var
, 1);
4549 b
= maybe_lookup_decl (b
, ctx
);
4552 b
= TREE_OPERAND (orig_var
, 1);
4553 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4555 if (integer_zerop (bias
))
4559 bias
= fold_convert_loc (clause_loc
,
4560 TREE_TYPE (b
), bias
);
4561 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4562 TREE_TYPE (b
), b
, bias
);
4564 orig_var
= TREE_OPERAND (orig_var
, 0);
4566 if (TREE_CODE (orig_var
) == INDIRECT_REF
4567 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4568 orig_var
= TREE_OPERAND (orig_var
, 0);
4569 tree d
= OMP_CLAUSE_DECL (c
);
4570 tree type
= TREE_TYPE (d
);
4571 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4572 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4573 const char *name
= get_name (orig_var
);
4574 if (TREE_CONSTANT (v
))
4576 x
= create_tmp_var_raw (type
, name
);
4577 gimple_add_tmp_var (x
);
4578 TREE_ADDRESSABLE (x
) = 1;
4579 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4584 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4585 tree t
= maybe_lookup_decl (v
, ctx
);
4589 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4590 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4591 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4593 build_int_cst (TREE_TYPE (v
), 1));
4594 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4596 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4597 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4598 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4601 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4602 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4603 tree y
= create_tmp_var (ptype
, name
);
4604 gimplify_assign (y
, x
, ilist
);
4608 if (!integer_zerop (bias
))
4610 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4612 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4614 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4615 pointer_sized_int_node
, yb
, bias
);
4616 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4617 yb
= create_tmp_var (ptype
, name
);
4618 gimplify_assign (yb
, x
, ilist
);
4622 d
= TREE_OPERAND (d
, 0);
4623 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4624 d
= TREE_OPERAND (d
, 0);
4625 if (TREE_CODE (d
) == ADDR_EXPR
)
4627 if (orig_var
!= var
)
4629 gcc_assert (is_variable_sized (orig_var
));
4630 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4632 gimplify_assign (new_var
, x
, ilist
);
4633 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4634 tree t
= build_fold_indirect_ref (new_var
);
4635 DECL_IGNORED_P (new_var
) = 0;
4636 TREE_THIS_NOTRAP (t
);
4637 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4638 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4642 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4643 build_int_cst (ptype
, 0));
4644 SET_DECL_VALUE_EXPR (new_var
, x
);
4645 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4650 gcc_assert (orig_var
== var
);
4651 if (TREE_CODE (d
) == INDIRECT_REF
)
4653 x
= create_tmp_var (ptype
, name
);
4654 TREE_ADDRESSABLE (x
) = 1;
4655 gimplify_assign (x
, yb
, ilist
);
4656 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4658 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4659 gimplify_assign (new_var
, x
, ilist
);
4661 tree y1
= create_tmp_var (ptype
, NULL
);
4662 gimplify_assign (y1
, y
, ilist
);
4663 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4664 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4665 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4666 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4668 y2
= create_tmp_var (ptype
, NULL
);
4669 gimplify_assign (y2
, y
, ilist
);
4670 tree ref
= build_outer_var_ref (var
, ctx
);
4671 /* For ref build_outer_var_ref already performs this. */
4672 if (TREE_CODE (d
) == INDIRECT_REF
)
4673 gcc_assert (is_reference (var
));
4674 else if (TREE_CODE (d
) == ADDR_EXPR
)
4675 ref
= build_fold_addr_expr (ref
);
4676 else if (is_reference (var
))
4677 ref
= build_fold_addr_expr (ref
);
4678 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4679 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4680 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4682 y3
= create_tmp_var (ptype
, NULL
);
4683 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4687 y4
= create_tmp_var (ptype
, NULL
);
4688 gimplify_assign (y4
, ref
, dlist
);
4691 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4692 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4693 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4694 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4695 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4698 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4699 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4700 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4701 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4702 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4704 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4706 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4707 tree decl_placeholder
4708 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4709 SET_DECL_VALUE_EXPR (decl_placeholder
,
4710 build_simple_mem_ref (y1
));
4711 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4712 SET_DECL_VALUE_EXPR (placeholder
,
4713 y3
? build_simple_mem_ref (y3
)
4715 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4716 x
= lang_hooks
.decls
.omp_clause_default_ctor
4717 (c
, build_simple_mem_ref (y1
),
4718 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4720 gimplify_and_add (x
, ilist
);
4721 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4723 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4724 lower_omp (&tseq
, ctx
);
4725 gimple_seq_add_seq (ilist
, tseq
);
4727 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4730 SET_DECL_VALUE_EXPR (decl_placeholder
,
4731 build_simple_mem_ref (y2
));
4732 SET_DECL_VALUE_EXPR (placeholder
,
4733 build_simple_mem_ref (y4
));
4734 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4735 lower_omp (&tseq
, ctx
);
4736 gimple_seq_add_seq (dlist
, tseq
);
4737 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4739 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4740 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4741 x
= lang_hooks
.decls
.omp_clause_dtor
4742 (c
, build_simple_mem_ref (y2
));
4745 gimple_seq tseq
= NULL
;
4747 gimplify_stmt (&dtor
, &tseq
);
4748 gimple_seq_add_seq (dlist
, tseq
);
4753 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4754 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4756 /* reduction(-:var) sums up the partial results, so it
4757 acts identically to reduction(+:var). */
4758 if (code
== MINUS_EXPR
)
4761 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4764 x
= build2 (code
, TREE_TYPE (type
),
4765 build_simple_mem_ref (y4
),
4766 build_simple_mem_ref (y2
));
4767 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4771 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4772 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4773 gimple_seq_add_stmt (ilist
, g
);
4776 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4777 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4778 gimple_seq_add_stmt (ilist
, g
);
4780 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4781 build_int_cst (TREE_TYPE (i
), 1));
4782 gimple_seq_add_stmt (ilist
, g
);
4783 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4784 gimple_seq_add_stmt (ilist
, g
);
4785 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4788 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4789 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4790 gimple_seq_add_stmt (dlist
, g
);
4793 g
= gimple_build_assign
4794 (y4
, POINTER_PLUS_EXPR
, y4
,
4795 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4796 gimple_seq_add_stmt (dlist
, g
);
4798 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4799 build_int_cst (TREE_TYPE (i2
), 1));
4800 gimple_seq_add_stmt (dlist
, g
);
4801 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4802 gimple_seq_add_stmt (dlist
, g
);
4803 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4807 else if (is_variable_sized (var
))
4809 /* For variable sized types, we need to allocate the
4810 actual storage here. Call alloca and store the
4811 result in the pointer decl that we created elsewhere. */
4815 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4820 ptr
= DECL_VALUE_EXPR (new_var
);
4821 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4822 ptr
= TREE_OPERAND (ptr
, 0);
4823 gcc_assert (DECL_P (ptr
));
4824 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4826 /* void *tmp = __builtin_alloca */
4827 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4828 stmt
= gimple_build_call (atmp
, 2, x
,
4829 size_int (DECL_ALIGN (var
)));
4830 tmp
= create_tmp_var_raw (ptr_type_node
);
4831 gimple_add_tmp_var (tmp
);
4832 gimple_call_set_lhs (stmt
, tmp
);
4834 gimple_seq_add_stmt (ilist
, stmt
);
4836 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4837 gimplify_assign (ptr
, x
, ilist
);
4840 else if (is_reference (var
))
4842 /* For references that are being privatized for Fortran,
4843 allocate new backing storage for the new pointer
4844 variable. This allows us to avoid changing all the
4845 code that expects a pointer to something that expects
4846 a direct variable. */
4850 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4851 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4853 x
= build_receiver_ref (var
, false, ctx
);
4854 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4856 else if (TREE_CONSTANT (x
))
4858 /* For reduction in SIMD loop, defer adding the
4859 initialization of the reference, because if we decide
4860 to use SIMD array for it, the initilization could cause
4862 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4866 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4868 gimple_add_tmp_var (x
);
4869 TREE_ADDRESSABLE (x
) = 1;
4870 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4876 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4877 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4878 tree al
= size_int (TYPE_ALIGN (rtype
));
4879 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4884 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4885 gimplify_assign (new_var
, x
, ilist
);
4888 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4890 else if (c_kind
== OMP_CLAUSE_REDUCTION
4891 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4899 switch (OMP_CLAUSE_CODE (c
))
4901 case OMP_CLAUSE_SHARED
:
4902 /* Ignore shared directives in teams construct. */
4903 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4905 /* Shared global vars are just accessed directly. */
4906 if (is_global_var (new_var
))
4908 /* For taskloop firstprivate/lastprivate, represented
4909 as firstprivate and shared clause on the task, new_var
4910 is the firstprivate var. */
4911 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4913 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4914 needs to be delayed until after fixup_child_record_type so
4915 that we get the correct type during the dereference. */
4916 by_ref
= use_pointer_for_field (var
, ctx
);
4917 x
= build_receiver_ref (var
, by_ref
, ctx
);
4918 SET_DECL_VALUE_EXPR (new_var
, x
);
4919 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4921 /* ??? If VAR is not passed by reference, and the variable
4922 hasn't been initialized yet, then we'll get a warning for
4923 the store into the omp_data_s structure. Ideally, we'd be
4924 able to notice this and not store anything at all, but
4925 we're generating code too early. Suppress the warning. */
4927 TREE_NO_WARNING (var
) = 1;
4930 case OMP_CLAUSE_LASTPRIVATE
:
4931 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4935 case OMP_CLAUSE_PRIVATE
:
4936 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4937 x
= build_outer_var_ref (var
, ctx
);
4938 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4940 if (is_task_ctx (ctx
))
4941 x
= build_receiver_ref (var
, false, ctx
);
4943 x
= build_outer_var_ref (var
, ctx
);
4949 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4950 (c
, unshare_expr (new_var
), x
);
4953 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4954 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4955 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4956 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4957 idx
, lane
, ivar
, lvar
))
4960 x
= lang_hooks
.decls
.omp_clause_default_ctor
4961 (c
, unshare_expr (ivar
), x
);
4963 gimplify_and_add (x
, &llist
[0]);
4966 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4969 gimple_seq tseq
= NULL
;
4972 gimplify_stmt (&dtor
, &tseq
);
4973 gimple_seq_add_seq (&llist
[1], tseq
);
4980 gimplify_and_add (nx
, ilist
);
4984 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4987 gimple_seq tseq
= NULL
;
4990 gimplify_stmt (&dtor
, &tseq
);
4991 gimple_seq_add_seq (dlist
, tseq
);
4995 case OMP_CLAUSE_LINEAR
:
4996 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4997 goto do_firstprivate
;
4998 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
5001 x
= build_outer_var_ref (var
, ctx
);
5004 case OMP_CLAUSE_FIRSTPRIVATE
:
5005 if (is_task_ctx (ctx
))
5007 if (is_reference (var
) || is_variable_sized (var
))
5009 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
5011 || use_pointer_for_field (var
, NULL
))
5013 x
= build_receiver_ref (var
, false, ctx
);
5014 SET_DECL_VALUE_EXPR (new_var
, x
);
5015 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5020 x
= build_outer_var_ref (var
, ctx
);
5023 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5024 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5026 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5027 tree stept
= TREE_TYPE (t
);
5028 tree ct
= find_omp_clause (clauses
,
5029 OMP_CLAUSE__LOOPTEMP_
);
5031 tree l
= OMP_CLAUSE_DECL (ct
);
5032 tree n1
= fd
->loop
.n1
;
5033 tree step
= fd
->loop
.step
;
5034 tree itype
= TREE_TYPE (l
);
5035 if (POINTER_TYPE_P (itype
))
5036 itype
= signed_type_for (itype
);
5037 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5038 if (TYPE_UNSIGNED (itype
)
5039 && fd
->loop
.cond_code
== GT_EXPR
)
5040 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5041 fold_build1 (NEGATE_EXPR
, itype
, l
),
5042 fold_build1 (NEGATE_EXPR
,
5045 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5046 t
= fold_build2 (MULT_EXPR
, stept
,
5047 fold_convert (stept
, l
), t
);
5049 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5051 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5053 gimplify_and_add (x
, ilist
);
5057 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5058 x
= fold_build2 (POINTER_PLUS_EXPR
,
5059 TREE_TYPE (x
), x
, t
);
5061 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5064 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5065 || TREE_ADDRESSABLE (new_var
))
5066 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5067 idx
, lane
, ivar
, lvar
))
5069 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5071 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5072 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5073 gimplify_and_add (x
, ilist
);
5074 gimple_stmt_iterator gsi
5075 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5077 = gimple_build_assign (unshare_expr (lvar
), iv
);
5078 gsi_insert_before_without_update (&gsi
, g
,
5080 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5081 enum tree_code code
= PLUS_EXPR
;
5082 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5083 code
= POINTER_PLUS_EXPR
;
5084 g
= gimple_build_assign (iv
, code
, iv
, t
);
5085 gsi_insert_before_without_update (&gsi
, g
,
5089 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5090 (c
, unshare_expr (ivar
), x
);
5091 gimplify_and_add (x
, &llist
[0]);
5092 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5095 gimple_seq tseq
= NULL
;
5098 gimplify_stmt (&dtor
, &tseq
);
5099 gimple_seq_add_seq (&llist
[1], tseq
);
5104 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5105 (c
, unshare_expr (new_var
), x
);
5106 gimplify_and_add (x
, ilist
);
5109 case OMP_CLAUSE__LOOPTEMP_
:
5110 gcc_assert (is_taskreg_ctx (ctx
));
5111 x
= build_outer_var_ref (var
, ctx
);
5112 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5113 gimplify_and_add (x
, ilist
);
5116 case OMP_CLAUSE_COPYIN
:
5117 by_ref
= use_pointer_for_field (var
, NULL
);
5118 x
= build_receiver_ref (var
, by_ref
, ctx
);
5119 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5120 append_to_statement_list (x
, ©in_seq
);
5121 copyin_by_ref
|= by_ref
;
5124 case OMP_CLAUSE_REDUCTION
:
5125 /* OpenACC reductions are initialized using the
5126 GOACC_REDUCTION internal function. */
5127 if (is_gimple_omp_oacc (ctx
->stmt
))
5129 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5131 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5133 x
= build_outer_var_ref (var
, ctx
);
5135 if (is_reference (var
)
5136 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5138 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5139 SET_DECL_VALUE_EXPR (placeholder
, x
);
5140 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5141 tree new_vard
= new_var
;
5142 if (is_reference (var
))
5144 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5145 new_vard
= TREE_OPERAND (new_var
, 0);
5146 gcc_assert (DECL_P (new_vard
));
5149 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5150 idx
, lane
, ivar
, lvar
))
5152 if (new_vard
== new_var
)
5154 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5155 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5159 SET_DECL_VALUE_EXPR (new_vard
,
5160 build_fold_addr_expr (ivar
));
5161 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5163 x
= lang_hooks
.decls
.omp_clause_default_ctor
5164 (c
, unshare_expr (ivar
),
5165 build_outer_var_ref (var
, ctx
));
5167 gimplify_and_add (x
, &llist
[0]);
5168 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5170 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5171 lower_omp (&tseq
, ctx
);
5172 gimple_seq_add_seq (&llist
[0], tseq
);
5174 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5175 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5176 lower_omp (&tseq
, ctx
);
5177 gimple_seq_add_seq (&llist
[1], tseq
);
5178 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5179 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5180 if (new_vard
== new_var
)
5181 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5183 SET_DECL_VALUE_EXPR (new_vard
,
5184 build_fold_addr_expr (lvar
));
5185 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5190 gimplify_stmt (&dtor
, &tseq
);
5191 gimple_seq_add_seq (&llist
[1], tseq
);
5195 /* If this is a reference to constant size reduction var
5196 with placeholder, we haven't emitted the initializer
5197 for it because it is undesirable if SIMD arrays are used.
5198 But if they aren't used, we need to emit the deferred
5199 initialization now. */
5200 else if (is_reference (var
) && is_simd
)
5201 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5202 x
= lang_hooks
.decls
.omp_clause_default_ctor
5203 (c
, unshare_expr (new_var
),
5204 build_outer_var_ref (var
, ctx
));
5206 gimplify_and_add (x
, ilist
);
5207 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5209 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5210 lower_omp (&tseq
, ctx
);
5211 gimple_seq_add_seq (ilist
, tseq
);
5213 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5216 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5217 lower_omp (&tseq
, ctx
);
5218 gimple_seq_add_seq (dlist
, tseq
);
5219 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5221 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5226 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5227 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5228 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5230 /* reduction(-:var) sums up the partial results, so it
5231 acts identically to reduction(+:var). */
5232 if (code
== MINUS_EXPR
)
5235 tree new_vard
= new_var
;
5236 if (is_simd
&& is_reference (var
))
5238 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5239 new_vard
= TREE_OPERAND (new_var
, 0);
5240 gcc_assert (DECL_P (new_vard
));
5243 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5244 idx
, lane
, ivar
, lvar
))
5246 tree ref
= build_outer_var_ref (var
, ctx
);
5248 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5250 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5251 ref
= build_outer_var_ref (var
, ctx
);
5252 gimplify_assign (ref
, x
, &llist
[1]);
5254 if (new_vard
!= new_var
)
5256 SET_DECL_VALUE_EXPR (new_vard
,
5257 build_fold_addr_expr (lvar
));
5258 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5263 if (is_reference (var
) && is_simd
)
5264 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5265 gimplify_assign (new_var
, x
, ilist
);
5268 tree ref
= build_outer_var_ref (var
, ctx
);
5270 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5271 ref
= build_outer_var_ref (var
, ctx
);
5272 gimplify_assign (ref
, x
, dlist
);
5286 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5287 /* Don't want uninit warnings on simduid, it is always uninitialized,
5288 but we use it not for the value, but for the DECL_UID only. */
5289 TREE_NO_WARNING (uid
) = 1;
5291 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5292 gimple_call_set_lhs (g
, lane
);
5293 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5294 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5295 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5296 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5297 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5298 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5299 g
= gimple_build_assign (lane
, INTEGER_CST
,
5300 build_int_cst (unsigned_type_node
, 0));
5301 gimple_seq_add_stmt (ilist
, g
);
5302 for (int i
= 0; i
< 2; i
++)
5305 tree vf
= create_tmp_var (unsigned_type_node
);
5306 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5307 gimple_call_set_lhs (g
, vf
);
5308 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5309 gimple_seq_add_stmt (seq
, g
);
5310 tree t
= build_int_cst (unsigned_type_node
, 0);
5311 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5312 gimple_seq_add_stmt (seq
, g
);
5313 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5314 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5315 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5316 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5317 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5318 gimple_seq_add_seq (seq
, llist
[i
]);
5319 t
= build_int_cst (unsigned_type_node
, 1);
5320 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5321 gimple_seq_add_stmt (seq
, g
);
5322 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5323 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5324 gimple_seq_add_stmt (seq
, g
);
5325 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5329 /* The copyin sequence is not to be executed by the main thread, since
5330 that would result in self-copies. Perhaps not visible to scalars,
5331 but it certainly is to C++ operator=. */
5334 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5336 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5337 build_int_cst (TREE_TYPE (x
), 0));
5338 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5339 gimplify_and_add (x
, ilist
);
5342 /* If any copyin variable is passed by reference, we must ensure the
5343 master thread doesn't modify it before it is copied over in all
5344 threads. Similarly for variables in both firstprivate and
5345 lastprivate clauses we need to ensure the lastprivate copying
5346 happens after firstprivate copying in all threads. And similarly
5347 for UDRs if initializer expression refers to omp_orig. */
5348 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5350 /* Don't add any barrier for #pragma omp simd or
5351 #pragma omp distribute. */
5352 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5353 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5354 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5357 /* If max_vf is non-zero, then we can use only a vectorization factor
5358 up to the max_vf we chose. So stick it into the safelen clause. */
5361 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5362 OMP_CLAUSE_SAFELEN
);
5364 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5365 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5368 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5369 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5371 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5372 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5378 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5379 both parallel and workshare constructs. PREDICATE may be NULL if it's
5383 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5386 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5387 bool par_clauses
= false;
5388 tree simduid
= NULL
, lastlane
= NULL
;
5390 /* Early exit if there are no lastprivate or linear clauses. */
5391 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5392 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5393 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5394 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5396 if (clauses
== NULL
)
5398 /* If this was a workshare clause, see if it had been combined
5399 with its parallel. In that case, look for the clauses on the
5400 parallel statement itself. */
5401 if (is_parallel_ctx (ctx
))
5405 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5408 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5409 OMP_CLAUSE_LASTPRIVATE
);
5410 if (clauses
== NULL
)
5418 tree label_true
, arm1
, arm2
;
5420 label
= create_artificial_label (UNKNOWN_LOCATION
);
5421 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5422 arm1
= TREE_OPERAND (predicate
, 0);
5423 arm2
= TREE_OPERAND (predicate
, 1);
5424 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5425 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5426 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5428 gimple_seq_add_stmt (stmt_list
, stmt
);
5429 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5432 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5433 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5435 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5437 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5440 for (c
= clauses
; c
;)
5443 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5445 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5446 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5447 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5449 var
= OMP_CLAUSE_DECL (c
);
5450 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5451 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5452 && is_taskloop_ctx (ctx
))
5454 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5455 new_var
= lookup_decl (var
, ctx
->outer
);
5458 new_var
= lookup_decl (var
, ctx
);
5460 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5462 tree val
= DECL_VALUE_EXPR (new_var
);
5463 if (TREE_CODE (val
) == ARRAY_REF
5464 && VAR_P (TREE_OPERAND (val
, 0))
5465 && lookup_attribute ("omp simd array",
5466 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5469 if (lastlane
== NULL
)
5471 lastlane
= create_tmp_var (unsigned_type_node
);
5473 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5475 TREE_OPERAND (val
, 1));
5476 gimple_call_set_lhs (g
, lastlane
);
5477 gimple_seq_add_stmt (stmt_list
, g
);
5479 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5480 TREE_OPERAND (val
, 0), lastlane
,
5481 NULL_TREE
, NULL_TREE
);
5485 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5486 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5488 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5489 gimple_seq_add_seq (stmt_list
,
5490 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5491 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5493 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5494 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5496 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5497 gimple_seq_add_seq (stmt_list
,
5498 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5499 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5503 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5504 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5506 gcc_checking_assert (is_taskloop_ctx (ctx
));
5507 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5509 if (is_global_var (ovar
))
5513 x
= build_outer_var_ref (var
, ctx
, true);
5514 if (is_reference (var
))
5515 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5516 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5517 gimplify_and_add (x
, stmt_list
);
5519 c
= OMP_CLAUSE_CHAIN (c
);
5520 if (c
== NULL
&& !par_clauses
)
5522 /* If this was a workshare clause, see if it had been combined
5523 with its parallel. In that case, continue looking for the
5524 clauses also on the parallel statement itself. */
5525 if (is_parallel_ctx (ctx
))
5529 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5532 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5533 OMP_CLAUSE_LASTPRIVATE
);
5539 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5542 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5543 (which might be a placeholder). INNER is true if this is an inner
5544 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5545 join markers. Generate the before-loop forking sequence in
5546 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5547 general form of these sequences is
5549 GOACC_REDUCTION_SETUP
5551 GOACC_REDUCTION_INIT
5553 GOACC_REDUCTION_FINI
5555 GOACC_REDUCTION_TEARDOWN. */
5558 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5559 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5560 gimple_seq
*join_seq
, omp_context
*ctx
)
5562 gimple_seq before_fork
= NULL
;
5563 gimple_seq after_fork
= NULL
;
5564 gimple_seq before_join
= NULL
;
5565 gimple_seq after_join
= NULL
;
5566 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5567 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5568 unsigned offset
= 0;
5570 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5571 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5573 tree orig
= OMP_CLAUSE_DECL (c
);
5574 tree var
= maybe_lookup_decl (orig
, ctx
);
5575 tree ref_to_res
= NULL_TREE
;
5576 tree incoming
, outgoing
, v1
, v2
, v3
;
5577 bool is_private
= false;
5579 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5580 if (rcode
== MINUS_EXPR
)
5582 else if (rcode
== TRUTH_ANDIF_EXPR
)
5583 rcode
= BIT_AND_EXPR
;
5584 else if (rcode
== TRUTH_ORIF_EXPR
)
5585 rcode
= BIT_IOR_EXPR
;
5586 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5591 incoming
= outgoing
= var
;
5595 /* See if an outer construct also reduces this variable. */
5596 omp_context
*outer
= ctx
;
5598 while (omp_context
*probe
= outer
->outer
)
5600 enum gimple_code type
= gimple_code (probe
->stmt
);
5605 case GIMPLE_OMP_FOR
:
5606 cls
= gimple_omp_for_clauses (probe
->stmt
);
5609 case GIMPLE_OMP_TARGET
:
5610 if (gimple_omp_target_kind (probe
->stmt
)
5611 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5614 cls
= gimple_omp_target_clauses (probe
->stmt
);
5622 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5623 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5624 && orig
== OMP_CLAUSE_DECL (cls
))
5626 incoming
= outgoing
= lookup_decl (orig
, probe
);
5627 goto has_outer_reduction
;
5629 else if ((OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_FIRSTPRIVATE
5630 || OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_PRIVATE
)
5631 && orig
== OMP_CLAUSE_DECL (cls
))
5639 /* This is the outermost construct with this reduction,
5640 see if there's a mapping for it. */
5641 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5642 && maybe_lookup_field (orig
, outer
) && !is_private
)
5644 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5645 if (is_reference (orig
))
5646 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5648 tree type
= TREE_TYPE (var
);
5649 if (POINTER_TYPE_P (type
))
5650 type
= TREE_TYPE (type
);
5653 incoming
= omp_reduction_init_op (loc
, rcode
, type
);
5655 else if (ctx
->outer
)
5656 incoming
= outgoing
= lookup_decl (orig
, ctx
->outer
);
5658 incoming
= outgoing
= orig
;
5660 has_outer_reduction
:;
5664 ref_to_res
= integer_zero_node
;
5666 if (is_reference (orig
))
5668 tree type
= TREE_TYPE (var
);
5669 const char *id
= IDENTIFIER_POINTER (DECL_NAME (var
));
5673 tree x
= create_tmp_var (TREE_TYPE (type
), id
);
5674 gimplify_assign (var
, build_fold_addr_expr (x
), fork_seq
);
5677 v1
= create_tmp_var (type
, id
);
5678 v2
= create_tmp_var (type
, id
);
5679 v3
= create_tmp_var (type
, id
);
5681 gimplify_assign (v1
, var
, fork_seq
);
5682 gimplify_assign (v2
, var
, fork_seq
);
5683 gimplify_assign (v3
, var
, fork_seq
);
5685 var
= build_simple_mem_ref (var
);
5686 v1
= build_simple_mem_ref (v1
);
5687 v2
= build_simple_mem_ref (v2
);
5688 v3
= build_simple_mem_ref (v3
);
5689 outgoing
= build_simple_mem_ref (outgoing
);
5691 if (!TREE_CONSTANT (incoming
))
5692 incoming
= build_simple_mem_ref (incoming
);
5697 /* Determine position in reduction buffer, which may be used
5699 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5700 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5701 offset
= (offset
+ align
- 1) & ~(align
- 1);
5702 tree off
= build_int_cst (sizetype
, offset
);
5703 offset
+= GET_MODE_SIZE (mode
);
5707 init_code
= build_int_cst (integer_type_node
,
5708 IFN_GOACC_REDUCTION_INIT
);
5709 fini_code
= build_int_cst (integer_type_node
,
5710 IFN_GOACC_REDUCTION_FINI
);
5711 setup_code
= build_int_cst (integer_type_node
,
5712 IFN_GOACC_REDUCTION_SETUP
);
5713 teardown_code
= build_int_cst (integer_type_node
,
5714 IFN_GOACC_REDUCTION_TEARDOWN
);
5718 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5719 TREE_TYPE (var
), 6, setup_code
,
5720 unshare_expr (ref_to_res
),
5721 incoming
, level
, op
, off
);
5723 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5724 TREE_TYPE (var
), 6, init_code
,
5725 unshare_expr (ref_to_res
),
5726 v1
, level
, op
, off
);
5728 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5729 TREE_TYPE (var
), 6, fini_code
,
5730 unshare_expr (ref_to_res
),
5731 v2
, level
, op
, off
);
5733 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5734 TREE_TYPE (var
), 6, teardown_code
,
5735 ref_to_res
, v3
, level
, op
, off
);
5737 gimplify_assign (v1
, setup_call
, &before_fork
);
5738 gimplify_assign (v2
, init_call
, &after_fork
);
5739 gimplify_assign (v3
, fini_call
, &before_join
);
5740 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5743 /* Now stitch things together. */
5744 gimple_seq_add_seq (fork_seq
, before_fork
);
5746 gimple_seq_add_stmt (fork_seq
, fork
);
5747 gimple_seq_add_seq (fork_seq
, after_fork
);
5749 gimple_seq_add_seq (join_seq
, before_join
);
5751 gimple_seq_add_stmt (join_seq
, join
);
5752 gimple_seq_add_seq (join_seq
, after_join
);
5755 /* Generate code to implement the REDUCTION clauses. */
5758 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5760 gimple_seq sub_seq
= NULL
;
5765 /* OpenACC loop reductions are handled elsewhere. */
5766 if (is_gimple_omp_oacc (ctx
->stmt
))
5769 /* SIMD reductions are handled in lower_rec_input_clauses. */
5770 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5771 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5774 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5775 update in that case, otherwise use a lock. */
5776 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5777 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5779 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5780 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5782 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5792 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5794 tree var
, ref
, new_var
, orig_var
;
5795 enum tree_code code
;
5796 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5798 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5801 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5802 if (TREE_CODE (var
) == MEM_REF
)
5804 var
= TREE_OPERAND (var
, 0);
5805 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5806 var
= TREE_OPERAND (var
, 0);
5807 if (TREE_CODE (var
) == INDIRECT_REF
5808 || TREE_CODE (var
) == ADDR_EXPR
)
5809 var
= TREE_OPERAND (var
, 0);
5811 if (is_variable_sized (var
))
5813 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5814 var
= DECL_VALUE_EXPR (var
);
5815 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5816 var
= TREE_OPERAND (var
, 0);
5817 gcc_assert (DECL_P (var
));
5820 new_var
= lookup_decl (var
, ctx
);
5821 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5822 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5823 ref
= build_outer_var_ref (var
, ctx
);
5824 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5826 /* reduction(-:var) sums up the partial results, so it acts
5827 identically to reduction(+:var). */
5828 if (code
== MINUS_EXPR
)
5833 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5835 addr
= save_expr (addr
);
5836 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5837 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5838 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5839 gimplify_and_add (x
, stmt_seqp
);
5842 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5844 tree d
= OMP_CLAUSE_DECL (c
);
5845 tree type
= TREE_TYPE (d
);
5846 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5847 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5848 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5849 tree bias
= TREE_OPERAND (d
, 1);
5850 d
= TREE_OPERAND (d
, 0);
5851 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5853 tree b
= TREE_OPERAND (d
, 1);
5854 b
= maybe_lookup_decl (b
, ctx
);
5857 b
= TREE_OPERAND (d
, 1);
5858 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5860 if (integer_zerop (bias
))
5864 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5865 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5866 TREE_TYPE (b
), b
, bias
);
5868 d
= TREE_OPERAND (d
, 0);
5870 /* For ref build_outer_var_ref already performs this, so
5871 only new_var needs a dereference. */
5872 if (TREE_CODE (d
) == INDIRECT_REF
)
5874 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5875 gcc_assert (is_reference (var
) && var
== orig_var
);
5877 else if (TREE_CODE (d
) == ADDR_EXPR
)
5879 if (orig_var
== var
)
5881 new_var
= build_fold_addr_expr (new_var
);
5882 ref
= build_fold_addr_expr (ref
);
5887 gcc_assert (orig_var
== var
);
5888 if (is_reference (var
))
5889 ref
= build_fold_addr_expr (ref
);
5893 tree t
= maybe_lookup_decl (v
, ctx
);
5897 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5898 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5900 if (!integer_zerop (bias
))
5902 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5903 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5904 TREE_TYPE (new_var
), new_var
,
5905 unshare_expr (bias
));
5906 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5907 TREE_TYPE (ref
), ref
, bias
);
5909 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5910 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5911 tree m
= create_tmp_var (ptype
, NULL
);
5912 gimplify_assign (m
, new_var
, stmt_seqp
);
5914 m
= create_tmp_var (ptype
, NULL
);
5915 gimplify_assign (m
, ref
, stmt_seqp
);
5917 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5918 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5919 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5920 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5921 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5922 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5923 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5925 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5926 tree decl_placeholder
5927 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5928 SET_DECL_VALUE_EXPR (placeholder
, out
);
5929 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5930 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5931 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5932 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5933 gimple_seq_add_seq (&sub_seq
,
5934 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5935 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5936 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5937 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5941 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5942 out
= unshare_expr (out
);
5943 gimplify_assign (out
, x
, &sub_seq
);
5945 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5946 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5947 gimple_seq_add_stmt (&sub_seq
, g
);
5948 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5949 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5950 gimple_seq_add_stmt (&sub_seq
, g
);
5951 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5952 build_int_cst (TREE_TYPE (i
), 1));
5953 gimple_seq_add_stmt (&sub_seq
, g
);
5954 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5955 gimple_seq_add_stmt (&sub_seq
, g
);
5956 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5958 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5960 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5962 if (is_reference (var
)
5963 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5965 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5966 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5967 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5968 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5969 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5970 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5971 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5975 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5976 ref
= build_outer_var_ref (var
, ctx
);
5977 gimplify_assign (ref
, x
, &sub_seq
);
5981 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5983 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5985 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5987 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5989 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5993 /* Generate code to implement the COPYPRIVATE clauses. */
5996 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
6001 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6003 tree var
, new_var
, ref
, x
;
6005 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6007 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
6010 var
= OMP_CLAUSE_DECL (c
);
6011 by_ref
= use_pointer_for_field (var
, NULL
);
6013 ref
= build_sender_ref (var
, ctx
);
6014 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
6017 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
6018 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
6020 gimplify_assign (ref
, x
, slist
);
6022 ref
= build_receiver_ref (var
, false, ctx
);
6025 ref
= fold_convert_loc (clause_loc
,
6026 build_pointer_type (TREE_TYPE (new_var
)),
6028 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
6030 if (is_reference (var
))
6032 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
6033 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
6034 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
6036 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
6037 gimplify_and_add (x
, rlist
);
6042 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
6043 and REDUCTION from the sender (aka parent) side. */
6046 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
6050 int ignored_looptemp
= 0;
6051 bool is_taskloop
= false;
6053 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
6054 by GOMP_taskloop. */
6055 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
6057 ignored_looptemp
= 2;
6061 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6063 tree val
, ref
, x
, var
;
6064 bool by_ref
, do_in
= false, do_out
= false;
6065 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6067 switch (OMP_CLAUSE_CODE (c
))
6069 case OMP_CLAUSE_PRIVATE
:
6070 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
6073 case OMP_CLAUSE_FIRSTPRIVATE
:
6074 case OMP_CLAUSE_COPYIN
:
6075 case OMP_CLAUSE_LASTPRIVATE
:
6076 case OMP_CLAUSE_REDUCTION
:
6078 case OMP_CLAUSE_SHARED
:
6079 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6082 case OMP_CLAUSE__LOOPTEMP_
:
6083 if (ignored_looptemp
)
6093 val
= OMP_CLAUSE_DECL (c
);
6094 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6095 && TREE_CODE (val
) == MEM_REF
)
6097 val
= TREE_OPERAND (val
, 0);
6098 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
6099 val
= TREE_OPERAND (val
, 0);
6100 if (TREE_CODE (val
) == INDIRECT_REF
6101 || TREE_CODE (val
) == ADDR_EXPR
)
6102 val
= TREE_OPERAND (val
, 0);
6103 if (is_variable_sized (val
))
6107 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6108 outer taskloop region. */
6109 omp_context
*ctx_for_o
= ctx
;
6111 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
6112 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6113 ctx_for_o
= ctx
->outer
;
6115 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
6117 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
6118 && is_global_var (var
))
6121 t
= omp_member_access_dummy_var (var
);
6124 var
= DECL_VALUE_EXPR (var
);
6125 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
6127 var
= unshare_and_remap (var
, t
, o
);
6129 var
= unshare_expr (var
);
6132 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
6134 /* Handle taskloop firstprivate/lastprivate, where the
6135 lastprivate on GIMPLE_OMP_TASK is represented as
6136 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6137 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
6138 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
6139 if (use_pointer_for_field (val
, ctx
))
6140 var
= build_fold_addr_expr (var
);
6141 gimplify_assign (x
, var
, ilist
);
6142 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
6146 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6147 || val
== OMP_CLAUSE_DECL (c
))
6148 && is_variable_sized (val
))
6150 by_ref
= use_pointer_for_field (val
, NULL
);
6152 switch (OMP_CLAUSE_CODE (c
))
6154 case OMP_CLAUSE_FIRSTPRIVATE
:
6155 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
)
6157 && is_task_ctx (ctx
))
6158 TREE_NO_WARNING (var
) = 1;
6162 case OMP_CLAUSE_PRIVATE
:
6163 case OMP_CLAUSE_COPYIN
:
6164 case OMP_CLAUSE__LOOPTEMP_
:
6168 case OMP_CLAUSE_LASTPRIVATE
:
6169 if (by_ref
|| is_reference (val
))
6171 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
6178 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
6183 case OMP_CLAUSE_REDUCTION
:
6185 if (val
== OMP_CLAUSE_DECL (c
))
6186 do_out
= !(by_ref
|| is_reference (val
));
6188 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
6197 ref
= build_sender_ref (val
, ctx
);
6198 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
6199 gimplify_assign (ref
, x
, ilist
);
6200 if (is_task_ctx (ctx
))
6201 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
6206 ref
= build_sender_ref (val
, ctx
);
6207 gimplify_assign (var
, ref
, olist
);
6212 /* Generate code to implement SHARED from the sender (aka parent)
6213 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6214 list things that got automatically shared. */
6217 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6219 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6221 if (ctx
->record_type
== NULL
)
6224 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6225 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6227 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6228 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6231 nvar
= maybe_lookup_decl (ovar
, ctx
);
6232 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6235 /* If CTX is a nested parallel directive. Find the immediately
6236 enclosing parallel or workshare construct that contains a
6237 mapping for OVAR. */
6238 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6240 t
= omp_member_access_dummy_var (var
);
6243 var
= DECL_VALUE_EXPR (var
);
6244 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6246 var
= unshare_and_remap (var
, t
, o
);
6248 var
= unshare_expr (var
);
6251 if (use_pointer_for_field (ovar
, ctx
))
6253 x
= build_sender_ref (ovar
, ctx
);
6254 var
= build_fold_addr_expr (var
);
6255 gimplify_assign (x
, var
, ilist
);
6259 x
= build_sender_ref (ovar
, ctx
);
6260 gimplify_assign (x
, var
, ilist
);
6262 if (!TREE_READONLY (var
)
6263 /* We don't need to receive a new reference to a result
6264 or parm decl. In fact we may not store to it as we will
6265 invalidate any pending RSO and generate wrong gimple
6267 && !((TREE_CODE (var
) == RESULT_DECL
6268 || TREE_CODE (var
) == PARM_DECL
)
6269 && DECL_BY_REFERENCE (var
)))
6271 x
= build_sender_ref (ovar
, ctx
);
6272 gimplify_assign (var
, x
, olist
);
6278 /* Emit an OpenACC head marker call, encapulating the partitioning and
6279 other information that must be processed by the target compiler.
6280 Return the maximum number of dimensions the associated loop might
6281 be partitioned over. */
6284 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6285 gimple_seq
*seq
, omp_context
*ctx
)
6287 unsigned levels
= 0;
6289 tree gang_static
= NULL_TREE
;
6290 auto_vec
<tree
, 5> args
;
6292 args
.quick_push (build_int_cst
6293 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6294 args
.quick_push (ddvar
);
6295 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6297 switch (OMP_CLAUSE_CODE (c
))
6299 case OMP_CLAUSE_GANG
:
6300 tag
|= OLF_DIM_GANG
;
6301 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6302 /* static:* is represented by -1, and we can ignore it, as
6303 scheduling is always static. */
6304 if (gang_static
&& integer_minus_onep (gang_static
))
6305 gang_static
= NULL_TREE
;
6309 case OMP_CLAUSE_WORKER
:
6310 tag
|= OLF_DIM_WORKER
;
6314 case OMP_CLAUSE_VECTOR
:
6315 tag
|= OLF_DIM_VECTOR
;
6319 case OMP_CLAUSE_SEQ
:
6323 case OMP_CLAUSE_AUTO
:
6327 case OMP_CLAUSE_INDEPENDENT
:
6328 tag
|= OLF_INDEPENDENT
;
6338 if (DECL_P (gang_static
))
6339 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6340 tag
|= OLF_GANG_STATIC
;
6343 /* In a parallel region, loops are implicitly INDEPENDENT. */
6344 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6345 if (!tgt
|| is_oacc_parallel (tgt
))
6346 tag
|= OLF_INDEPENDENT
;
6348 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6349 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6353 /* Ensure at least one level. */
6357 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6358 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6360 args
.quick_push (gang_static
);
6362 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6363 gimple_set_location (call
, loc
);
6364 gimple_set_lhs (call
, ddvar
);
6365 gimple_seq_add_stmt (seq
, call
);
6370 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6371 partitioning level of the enclosed region. */
6374 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6375 tree tofollow
, gimple_seq
*seq
)
6377 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6378 : IFN_UNIQUE_OACC_TAIL_MARK
);
6379 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6380 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6381 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6382 marker
, ddvar
, tofollow
);
6383 gimple_set_location (call
, loc
);
6384 gimple_set_lhs (call
, ddvar
);
6385 gimple_seq_add_stmt (seq
, call
);
6388 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6389 the loop clauses, from which we extract reductions. Initialize
6393 lower_oacc_head_tail (location_t loc
, tree clauses
,
6394 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6397 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6398 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6400 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6401 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6402 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6405 for (unsigned done
= 1; count
; count
--, done
++)
6407 gimple_seq fork_seq
= NULL
;
6408 gimple_seq join_seq
= NULL
;
6410 tree place
= build_int_cst (integer_type_node
, -1);
6411 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6412 fork_kind
, ddvar
, place
);
6413 gimple_set_location (fork
, loc
);
6414 gimple_set_lhs (fork
, ddvar
);
6416 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6417 join_kind
, ddvar
, place
);
6418 gimple_set_location (join
, loc
);
6419 gimple_set_lhs (join
, ddvar
);
6421 /* Mark the beginning of this level sequence. */
6423 lower_oacc_loop_marker (loc
, ddvar
, true,
6424 build_int_cst (integer_type_node
, count
),
6426 lower_oacc_loop_marker (loc
, ddvar
, false,
6427 build_int_cst (integer_type_node
, done
),
6430 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6431 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6433 /* Append this level to head. */
6434 gimple_seq_add_seq (head
, fork_seq
);
6435 /* Prepend it to tail. */
6436 gimple_seq_add_seq (&join_seq
, *tail
);
6442 /* Mark the end of the sequence. */
6443 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6444 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6447 /* A convenience function to build an empty GIMPLE_COND with just the
6451 gimple_build_cond_empty (tree cond
)
6453 enum tree_code pred_code
;
6456 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6457 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6460 /* Return true if a parallel REGION is within a declare target function or
6461 within a target region and is not a part of a gridified target. */
6464 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
6466 bool indirect
= false;
6467 for (region
= region
->outer
; region
; region
= region
->outer
)
6469 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6471 else if (region
->type
== GIMPLE_OMP_TARGET
)
6473 gomp_target
*tgt_stmt
6474 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
6476 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
6477 OMP_CLAUSE__GRIDDIM_
))
6484 if (lookup_attribute ("omp declare target",
6485 DECL_ATTRIBUTES (current_function_decl
)))
6491 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6494 /* Build the function calls to GOMP_parallel_start etc to actually
6495 generate the parallel operation. REGION is the parallel region
6496 being expanded. BB is the block where to insert the code. WS_ARGS
6497 will be set if this is a call to a combined parallel+workshare
6498 construct, it contains the list of additional arguments needed by
6499 the workshare construct. */
6502 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6503 gomp_parallel
*entry_stmt
,
6504 vec
<tree
, va_gc
> *ws_args
)
6506 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6507 gimple_stmt_iterator gsi
;
6509 enum built_in_function start_ix
;
6511 location_t clause_loc
;
6512 vec
<tree
, va_gc
> *args
;
6514 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6516 /* Determine what flavor of GOMP_parallel we will be
6518 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6519 if (is_combined_parallel (region
))
6521 switch (region
->inner
->type
)
6523 case GIMPLE_OMP_FOR
:
6524 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6525 switch (region
->inner
->sched_kind
)
6527 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6530 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6531 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6532 if (region
->inner
->sched_modifiers
6533 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6535 start_ix2
= 3 + region
->inner
->sched_kind
;
6540 start_ix2
= region
->inner
->sched_kind
;
6543 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6544 start_ix
= (enum built_in_function
) start_ix2
;
6546 case GIMPLE_OMP_SECTIONS
:
6547 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6554 /* By default, the value of NUM_THREADS is zero (selected at run time)
6555 and there is no conditional. */
6557 val
= build_int_cst (unsigned_type_node
, 0);
6558 flags
= build_int_cst (unsigned_type_node
, 0);
6560 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6562 cond
= OMP_CLAUSE_IF_EXPR (c
);
6564 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6567 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6568 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6571 clause_loc
= gimple_location (entry_stmt
);
6573 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6575 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6577 /* Ensure 'val' is of the correct type. */
6578 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6580 /* If we found the clause 'if (cond)', build either
6581 (cond != 0) or (cond ? val : 1u). */
6584 cond
= gimple_boolify (cond
);
6586 if (integer_zerop (val
))
6587 val
= fold_build2_loc (clause_loc
,
6588 EQ_EXPR
, unsigned_type_node
, cond
,
6589 build_int_cst (TREE_TYPE (cond
), 0));
6592 basic_block cond_bb
, then_bb
, else_bb
;
6593 edge e
, e_then
, e_else
;
6594 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6596 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6597 if (gimple_in_ssa_p (cfun
))
6599 tmp_then
= make_ssa_name (tmp_var
);
6600 tmp_else
= make_ssa_name (tmp_var
);
6601 tmp_join
= make_ssa_name (tmp_var
);
6610 e
= split_block_after_labels (bb
);
6615 then_bb
= create_empty_bb (cond_bb
);
6616 else_bb
= create_empty_bb (then_bb
);
6617 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6618 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6620 stmt
= gimple_build_cond_empty (cond
);
6621 gsi
= gsi_start_bb (cond_bb
);
6622 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6624 gsi
= gsi_start_bb (then_bb
);
6625 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6627 gsi
= gsi_start_bb (else_bb
);
6628 expand_omp_build_assign (&gsi
, tmp_else
,
6629 build_int_cst (unsigned_type_node
, 1),
6632 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6633 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6634 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6635 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6636 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6637 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6639 if (gimple_in_ssa_p (cfun
))
6641 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6642 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6643 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6649 gsi
= gsi_start_bb (bb
);
6650 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6651 false, GSI_CONTINUE_LINKING
);
6654 gsi
= gsi_last_bb (bb
);
6655 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6657 t1
= null_pointer_node
;
6659 t1
= build_fold_addr_expr (t
);
6660 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
6661 t2
= build_fold_addr_expr (child_fndecl
);
6663 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6664 args
->quick_push (t2
);
6665 args
->quick_push (t1
);
6666 args
->quick_push (val
);
6668 args
->splice (*ws_args
);
6669 args
->quick_push (flags
);
6671 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6672 builtin_decl_explicit (start_ix
), args
);
6674 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6675 false, GSI_CONTINUE_LINKING
);
6677 if (hsa_gen_requested_p ()
6678 && parallel_needs_hsa_kernel_p (region
))
6680 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
6681 hsa_register_kernel (child_cnode
);
6685 /* Insert a function call whose name is FUNC_NAME with the information from
6686 ENTRY_STMT into the basic_block BB. */
6689 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6690 vec
<tree
, va_gc
> *ws_args
)
6693 gimple_stmt_iterator gsi
;
6694 vec
<tree
, va_gc
> *args
;
6696 gcc_assert (vec_safe_length (ws_args
) == 2);
6697 tree func_name
= (*ws_args
)[0];
6698 tree grain
= (*ws_args
)[1];
6700 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6701 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6702 gcc_assert (count
!= NULL_TREE
);
6703 count
= OMP_CLAUSE_OPERAND (count
, 0);
6705 gsi
= gsi_last_bb (bb
);
6706 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6708 t1
= null_pointer_node
;
6710 t1
= build_fold_addr_expr (t
);
6711 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6713 vec_alloc (args
, 4);
6714 args
->quick_push (t2
);
6715 args
->quick_push (t1
);
6716 args
->quick_push (count
);
6717 args
->quick_push (grain
);
6718 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6720 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6721 GSI_CONTINUE_LINKING
);
6724 /* Build the function call to GOMP_task to actually
6725 generate the task operation. BB is the block where to insert the code. */
6728 expand_task_call (struct omp_region
*region
, basic_block bb
,
6729 gomp_task
*entry_stmt
)
6732 gimple_stmt_iterator gsi
;
6733 location_t loc
= gimple_location (entry_stmt
);
6735 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6737 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6738 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6739 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6740 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6741 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6742 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6745 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6746 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6747 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6749 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6750 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6751 tree num_tasks
= NULL_TREE
;
6755 gimple
*g
= last_stmt (region
->outer
->entry
);
6756 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6757 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6758 struct omp_for_data fd
;
6759 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6760 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6761 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6762 OMP_CLAUSE__LOOPTEMP_
);
6763 startvar
= OMP_CLAUSE_DECL (startvar
);
6764 endvar
= OMP_CLAUSE_DECL (endvar
);
6765 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6766 if (fd
.loop
.cond_code
== LT_EXPR
)
6767 iflags
|= GOMP_TASK_FLAG_UP
;
6768 tree tclauses
= gimple_omp_for_clauses (g
);
6769 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6771 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6774 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6777 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6778 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6781 num_tasks
= integer_zero_node
;
6783 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6784 if (ifc
== NULL_TREE
)
6785 iflags
|= GOMP_TASK_FLAG_IF
;
6786 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6787 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6788 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6791 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6793 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6795 tree cond
= boolean_true_node
;
6800 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6801 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6802 build_int_cst (unsigned_type_node
,
6804 build_int_cst (unsigned_type_node
, 0));
6805 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6809 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6814 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6815 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6816 build_int_cst (unsigned_type_node
,
6817 GOMP_TASK_FLAG_FINAL
),
6818 build_int_cst (unsigned_type_node
, 0));
6819 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6822 depend
= OMP_CLAUSE_DECL (depend
);
6824 depend
= build_int_cst (ptr_type_node
, 0);
6826 priority
= fold_convert (integer_type_node
,
6827 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6829 priority
= integer_zero_node
;
6831 gsi
= gsi_last_bb (bb
);
6832 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6834 t2
= null_pointer_node
;
6836 t2
= build_fold_addr_expr_loc (loc
, t
);
6837 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6838 t
= gimple_omp_task_copy_fn (entry_stmt
);
6840 t3
= null_pointer_node
;
6842 t3
= build_fold_addr_expr_loc (loc
, t
);
6845 t
= build_call_expr (ull
6846 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6847 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6849 gimple_omp_task_arg_size (entry_stmt
),
6850 gimple_omp_task_arg_align (entry_stmt
), flags
,
6851 num_tasks
, priority
, startvar
, endvar
, step
);
6853 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6855 gimple_omp_task_arg_size (entry_stmt
),
6856 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6859 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6860 false, GSI_CONTINUE_LINKING
);
6864 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6865 catch handler and return it. This prevents programs from violating the
6866 structured block semantics with throws. */
6869 maybe_catch_exception (gimple_seq body
)
6874 if (!flag_exceptions
)
6877 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6878 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6880 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6882 g
= gimple_build_eh_must_not_throw (decl
);
6883 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6886 return gimple_seq_alloc_with_stmt (g
);
6889 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6892 vec2chain (vec
<tree
, va_gc
> *v
)
6894 tree chain
= NULL_TREE
, t
;
6897 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6899 DECL_CHAIN (t
) = chain
;
6907 /* Remove barriers in REGION->EXIT's block. Note that this is only
6908 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6909 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6910 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6914 remove_exit_barrier (struct omp_region
*region
)
6916 gimple_stmt_iterator gsi
;
6917 basic_block exit_bb
;
6921 int any_addressable_vars
= -1;
6923 exit_bb
= region
->exit
;
6925 /* If the parallel region doesn't return, we don't have REGION->EXIT
6930 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6931 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6932 statements that can appear in between are extremely limited -- no
6933 memory operations at all. Here, we allow nothing at all, so the
6934 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6935 gsi
= gsi_last_bb (exit_bb
);
6936 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6938 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6941 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6943 gsi
= gsi_last_bb (e
->src
);
6944 if (gsi_end_p (gsi
))
6946 stmt
= gsi_stmt (gsi
);
6947 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6948 && !gimple_omp_return_nowait_p (stmt
))
6950 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6951 in many cases. If there could be tasks queued, the barrier
6952 might be needed to let the tasks run before some local
6953 variable of the parallel that the task uses as shared
6954 runs out of scope. The task can be spawned either
6955 from within current function (this would be easy to check)
6956 or from some function it calls and gets passed an address
6957 of such a variable. */
6958 if (any_addressable_vars
< 0)
6960 gomp_parallel
*parallel_stmt
6961 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6962 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6963 tree local_decls
, block
, decl
;
6966 any_addressable_vars
= 0;
6967 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6968 if (TREE_ADDRESSABLE (decl
))
6970 any_addressable_vars
= 1;
6973 for (block
= gimple_block (stmt
);
6974 !any_addressable_vars
6976 && TREE_CODE (block
) == BLOCK
;
6977 block
= BLOCK_SUPERCONTEXT (block
))
6979 for (local_decls
= BLOCK_VARS (block
);
6981 local_decls
= DECL_CHAIN (local_decls
))
6982 if (TREE_ADDRESSABLE (local_decls
))
6984 any_addressable_vars
= 1;
6987 if (block
== gimple_block (parallel_stmt
))
6991 if (!any_addressable_vars
)
6992 gimple_omp_return_set_nowait (stmt
);
6998 remove_exit_barriers (struct omp_region
*region
)
7000 if (region
->type
== GIMPLE_OMP_PARALLEL
)
7001 remove_exit_barrier (region
);
7005 region
= region
->inner
;
7006 remove_exit_barriers (region
);
7007 while (region
->next
)
7009 region
= region
->next
;
7010 remove_exit_barriers (region
);
7015 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
7016 calls. These can't be declared as const functions, but
7017 within one parallel body they are constant, so they can be
7018 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
7019 which are declared const. Similarly for task body, except
7020 that in untied task omp_get_thread_num () can change at any task
7021 scheduling point. */
7024 optimize_omp_library_calls (gimple
*entry_stmt
)
7027 gimple_stmt_iterator gsi
;
7028 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7029 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
7030 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7031 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
7032 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
7033 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
7034 OMP_CLAUSE_UNTIED
) != NULL
);
7036 FOR_EACH_BB_FN (bb
, cfun
)
7037 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
7039 gimple
*call
= gsi_stmt (gsi
);
7042 if (is_gimple_call (call
)
7043 && (decl
= gimple_call_fndecl (call
))
7044 && DECL_EXTERNAL (decl
)
7045 && TREE_PUBLIC (decl
)
7046 && DECL_INITIAL (decl
) == NULL
)
7050 if (DECL_NAME (decl
) == thr_num_id
)
7052 /* In #pragma omp task untied omp_get_thread_num () can change
7053 during the execution of the task region. */
7056 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7058 else if (DECL_NAME (decl
) == num_thr_id
)
7059 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7063 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
7064 || gimple_call_num_args (call
) != 0)
7067 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
7070 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
7071 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
7072 TREE_TYPE (TREE_TYPE (built_in
))))
7075 gimple_call_set_fndecl (call
, built_in
);
7080 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7084 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
7088 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7089 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
7092 if (TREE_CODE (t
) == ADDR_EXPR
)
7093 recompute_tree_invariant_for_addr_expr (t
);
7095 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
7099 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7102 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
7105 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
7106 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
7107 !after
, after
? GSI_CONTINUE_LINKING
7109 gimple
*stmt
= gimple_build_assign (to
, from
);
7111 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
7113 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
7114 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
7115 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
7117 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
7118 gimple_regimplify_operands (stmt
, &gsi
);
7122 /* Expand the OpenMP parallel or task directive starting at REGION. */
7125 expand_omp_taskreg (struct omp_region
*region
)
7127 basic_block entry_bb
, exit_bb
, new_bb
;
7128 struct function
*child_cfun
;
7129 tree child_fn
, block
, t
;
7130 gimple_stmt_iterator gsi
;
7131 gimple
*entry_stmt
, *stmt
;
7133 vec
<tree
, va_gc
> *ws_args
;
7135 entry_stmt
= last_stmt (region
->entry
);
7136 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
7137 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7139 entry_bb
= region
->entry
;
7140 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
7141 exit_bb
= region
->cont
;
7143 exit_bb
= region
->exit
;
7147 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
7148 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
7149 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
7152 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7153 and the inner statement contains the name of the built-in function
7155 ws_args
= region
->inner
->ws_args
;
7156 else if (is_combined_parallel (region
))
7157 ws_args
= region
->ws_args
;
7161 if (child_cfun
->cfg
)
7163 /* Due to inlining, it may happen that we have already outlined
7164 the region, in which case all we need to do is make the
7165 sub-graph unreachable and emit the parallel call. */
7166 edge entry_succ_e
, exit_succ_e
;
7168 entry_succ_e
= single_succ_edge (entry_bb
);
7170 gsi
= gsi_last_bb (entry_bb
);
7171 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
7172 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
7173 gsi_remove (&gsi
, true);
7178 exit_succ_e
= single_succ_edge (exit_bb
);
7179 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
7181 remove_edge_and_dominated_blocks (entry_succ_e
);
7185 unsigned srcidx
, dstidx
, num
;
7187 /* If the parallel region needs data sent from the parent
7188 function, then the very first statement (except possible
7189 tree profile counter updates) of the parallel body
7190 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7191 &.OMP_DATA_O is passed as an argument to the child function,
7192 we need to replace it with the argument as seen by the child
7195 In most cases, this will end up being the identity assignment
7196 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7197 a function call that has been inlined, the original PARM_DECL
7198 .OMP_DATA_I may have been converted into a different local
7199 variable. In which case, we need to keep the assignment. */
7200 if (gimple_omp_taskreg_data_arg (entry_stmt
))
7202 basic_block entry_succ_bb
7203 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
7204 : FALLTHRU_EDGE (entry_bb
)->dest
;
7206 gimple
*parcopy_stmt
= NULL
;
7208 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7212 gcc_assert (!gsi_end_p (gsi
));
7213 stmt
= gsi_stmt (gsi
);
7214 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7217 if (gimple_num_ops (stmt
) == 2)
7219 tree arg
= gimple_assign_rhs1 (stmt
);
7221 /* We're ignore the subcode because we're
7222 effectively doing a STRIP_NOPS. */
7224 if (TREE_CODE (arg
) == ADDR_EXPR
7225 && TREE_OPERAND (arg
, 0)
7226 == gimple_omp_taskreg_data_arg (entry_stmt
))
7228 parcopy_stmt
= stmt
;
7234 gcc_assert (parcopy_stmt
!= NULL
);
7235 arg
= DECL_ARGUMENTS (child_fn
);
7237 if (!gimple_in_ssa_p (cfun
))
7239 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
7240 gsi_remove (&gsi
, true);
7243 /* ?? Is setting the subcode really necessary ?? */
7244 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
7245 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7250 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
7251 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
7252 /* We'd like to set the rhs to the default def in the child_fn,
7253 but it's too early to create ssa names in the child_fn.
7254 Instead, we set the rhs to the parm. In
7255 move_sese_region_to_fn, we introduce a default def for the
7256 parm, map the parm to it's default def, and once we encounter
7257 this stmt, replace the parm with the default def. */
7258 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7259 update_stmt (parcopy_stmt
);
7263 /* Declare local variables needed in CHILD_CFUN. */
7264 block
= DECL_INITIAL (child_fn
);
7265 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7266 /* The gimplifier could record temporaries in parallel/task block
7267 rather than in containing function's local_decls chain,
7268 which would mean cgraph missed finalizing them. Do it now. */
7269 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7270 if (TREE_CODE (t
) == VAR_DECL
7272 && !DECL_EXTERNAL (t
))
7273 varpool_node::finalize_decl (t
);
7274 DECL_SAVED_TREE (child_fn
) = NULL
;
7275 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7276 gimple_set_body (child_fn
, NULL
);
7277 TREE_USED (block
) = 1;
7279 /* Reset DECL_CONTEXT on function arguments. */
7280 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7281 DECL_CONTEXT (t
) = child_fn
;
7283 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7284 so that it can be moved to the child function. */
7285 gsi
= gsi_last_bb (entry_bb
);
7286 stmt
= gsi_stmt (gsi
);
7287 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7288 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7289 e
= split_block (entry_bb
, stmt
);
7290 gsi_remove (&gsi
, true);
7293 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7294 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7297 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7298 gcc_assert (e2
->dest
== region
->exit
);
7299 remove_edge (BRANCH_EDGE (entry_bb
));
7300 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7301 gsi
= gsi_last_bb (region
->exit
);
7302 gcc_assert (!gsi_end_p (gsi
)
7303 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7304 gsi_remove (&gsi
, true);
7307 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7310 gsi
= gsi_last_bb (exit_bb
);
7311 gcc_assert (!gsi_end_p (gsi
)
7312 && (gimple_code (gsi_stmt (gsi
))
7313 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7314 stmt
= gimple_build_return (NULL
);
7315 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7316 gsi_remove (&gsi
, true);
7319 /* Move the parallel region into CHILD_CFUN. */
7321 if (gimple_in_ssa_p (cfun
))
7323 init_tree_ssa (child_cfun
);
7324 init_ssa_operands (child_cfun
);
7325 child_cfun
->gimple_df
->in_ssa_p
= true;
7329 block
= gimple_block (entry_stmt
);
7331 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7333 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7336 basic_block dest_bb
= e2
->dest
;
7338 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7340 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7342 /* When the OMP expansion process cannot guarantee an up-to-date
7343 loop tree arrange for the child function to fixup loops. */
7344 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7345 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7347 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7348 num
= vec_safe_length (child_cfun
->local_decls
);
7349 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7351 t
= (*child_cfun
->local_decls
)[srcidx
];
7352 if (DECL_CONTEXT (t
) == cfun
->decl
)
7354 if (srcidx
!= dstidx
)
7355 (*child_cfun
->local_decls
)[dstidx
] = t
;
7359 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7361 /* Inform the callgraph about the new function. */
7362 child_cfun
->curr_properties
= cfun
->curr_properties
;
7363 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7364 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7365 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7366 node
->parallelized_function
= 1;
7367 cgraph_node::add_new_function (child_fn
, true);
7369 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7370 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7372 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7373 fixed in a following pass. */
7374 push_cfun (child_cfun
);
7376 assign_assembler_name_if_neeeded (child_fn
);
7379 optimize_omp_library_calls (entry_stmt
);
7380 cgraph_edge::rebuild_edges ();
7382 /* Some EH regions might become dead, see PR34608. If
7383 pass_cleanup_cfg isn't the first pass to happen with the
7384 new child, these dead EH edges might cause problems.
7385 Clean them up now. */
7386 if (flag_exceptions
)
7389 bool changed
= false;
7391 FOR_EACH_BB_FN (bb
, cfun
)
7392 changed
|= gimple_purge_dead_eh_edges (bb
);
7394 cleanup_tree_cfg ();
7396 if (gimple_in_ssa_p (cfun
))
7397 update_ssa (TODO_update_ssa
);
7398 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7399 verify_loop_structure ();
7402 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7404 omp_any_child_fn_dumped
= true;
7405 dump_function_header (dump_file
, child_fn
, dump_flags
);
7406 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7410 /* Emit a library call to launch the children threads. */
7412 expand_cilk_for_call (new_bb
,
7413 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7414 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7415 expand_parallel_call (region
, new_bb
,
7416 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7418 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7419 if (gimple_in_ssa_p (cfun
))
7420 update_ssa (TODO_update_ssa_only_virtuals
);
7423 /* Information about members of an OpenACC collapsed loop nest. */
7425 struct oacc_collapse
7427 tree base
; /* Base value. */
7428 tree iters
; /* Number of steps. */
7429 tree step
; /* step size. */
7432 /* Helper for expand_oacc_for. Determine collapsed loop information.
7433 Fill in COUNTS array. Emit any initialization code before GSI.
7434 Return the calculated outer loop bound of BOUND_TYPE. */
7437 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7438 gimple_stmt_iterator
*gsi
,
7439 oacc_collapse
*counts
, tree bound_type
)
7441 tree total
= build_int_cst (bound_type
, 1);
7444 gcc_assert (integer_onep (fd
->loop
.step
));
7445 gcc_assert (integer_zerop (fd
->loop
.n1
));
7447 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7449 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7451 tree iter_type
= TREE_TYPE (loop
->v
);
7452 tree diff_type
= iter_type
;
7453 tree plus_type
= iter_type
;
7455 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7457 if (POINTER_TYPE_P (iter_type
))
7458 plus_type
= sizetype
;
7459 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7460 diff_type
= signed_type_for (diff_type
);
7464 tree s
= loop
->step
;
7465 bool up
= loop
->cond_code
== LT_EXPR
;
7466 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7470 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7471 true, GSI_SAME_STMT
);
7472 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7473 true, GSI_SAME_STMT
);
7475 /* Convert the step, avoiding possible unsigned->signed overflow. */
7476 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7478 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7479 s
= fold_convert (diff_type
, s
);
7481 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7482 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7483 true, GSI_SAME_STMT
);
7485 /* Determine the range, avoiding possible unsigned->signed overflow. */
7486 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7487 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7488 fold_convert (plus_type
, negating
? b
: e
),
7489 fold_convert (plus_type
, negating
? e
: b
));
7490 expr
= fold_convert (diff_type
, expr
);
7492 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7493 tree range
= force_gimple_operand_gsi
7494 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7496 /* Determine number of iterations. */
7497 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7498 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7499 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7501 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7502 true, GSI_SAME_STMT
);
7504 counts
[ix
].base
= b
;
7505 counts
[ix
].iters
= iters
;
7506 counts
[ix
].step
= s
;
7508 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7509 fold_convert (bound_type
, iters
));
7515 /* Emit initializers for collapsed loop members. IVAR is the outer
7516 loop iteration variable, from which collapsed loop iteration values
7517 are calculated. COUNTS array has been initialized by
7518 expand_oacc_collapse_inits. */
7521 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7522 gimple_stmt_iterator
*gsi
,
7523 const oacc_collapse
*counts
, tree ivar
)
7525 tree ivar_type
= TREE_TYPE (ivar
);
7527 /* The most rapidly changing iteration variable is the innermost
7529 for (int ix
= fd
->collapse
; ix
--;)
7531 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7532 const oacc_collapse
*collapse
= &counts
[ix
];
7533 tree iter_type
= TREE_TYPE (loop
->v
);
7534 tree diff_type
= TREE_TYPE (collapse
->step
);
7535 tree plus_type
= iter_type
;
7536 enum tree_code plus_code
= PLUS_EXPR
;
7539 if (POINTER_TYPE_P (iter_type
))
7541 plus_code
= POINTER_PLUS_EXPR
;
7542 plus_type
= sizetype
;
7545 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7546 fold_convert (ivar_type
, collapse
->iters
));
7547 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7549 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7550 fold_convert (plus_type
, expr
));
7551 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7552 true, GSI_SAME_STMT
);
7553 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7554 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7558 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7559 fold_convert (ivar_type
, collapse
->iters
));
7560 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7561 true, GSI_SAME_STMT
);
7567 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7568 of the combined collapse > 1 loop constructs, generate code like:
7569 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7574 count3 = (adj + N32 - N31) / STEP3;
7575 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7580 count2 = (adj + N22 - N21) / STEP2;
7581 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7586 count1 = (adj + N12 - N11) / STEP1;
7587 count = count1 * count2 * count3;
7588 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7590 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7591 of the combined loop constructs, just initialize COUNTS array
7592 from the _looptemp_ clauses. */
7594 /* NOTE: It *could* be better to moosh all of the BBs together,
7595 creating one larger BB with all the computation and the unexpected
7596 jump at the end. I.e.
7598 bool zero3, zero2, zero1, zero;
7601 count3 = (N32 - N31) /[cl] STEP3;
7603 count2 = (N22 - N21) /[cl] STEP2;
7605 count1 = (N12 - N11) /[cl] STEP1;
7606 zero = zero3 || zero2 || zero1;
7607 count = count1 * count2 * count3;
7608 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7610 After all, we expect the zero=false, and thus we expect to have to
7611 evaluate all of the comparison expressions, so short-circuiting
7612 oughtn't be a win. Since the condition isn't protecting a
7613 denominator, we're not concerned about divide-by-zero, so we can
7614 fully evaluate count even if a numerator turned out to be wrong.
7616 It seems like putting this all together would create much better
7617 scheduling opportunities, and less pressure on the chip's branch
7621 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7622 basic_block
&entry_bb
, tree
*counts
,
7623 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7624 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7625 basic_block
&l2_dom_bb
)
7627 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7631 /* Collapsed loops need work for expansion into SSA form. */
7632 gcc_assert (!gimple_in_ssa_p (cfun
));
7634 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7635 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7637 gcc_assert (fd
->ordered
== 0);
7638 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7639 isn't supposed to be handled, as the inner loop doesn't
7641 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7642 OMP_CLAUSE__LOOPTEMP_
);
7643 gcc_assert (innerc
);
7644 for (i
= 0; i
< fd
->collapse
; i
++)
7646 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7647 OMP_CLAUSE__LOOPTEMP_
);
7648 gcc_assert (innerc
);
7650 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7652 counts
[0] = NULL_TREE
;
7657 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7659 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7660 counts
[i
] = NULL_TREE
;
7661 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7662 fold_convert (itype
, fd
->loops
[i
].n1
),
7663 fold_convert (itype
, fd
->loops
[i
].n2
));
7664 if (t
&& integer_zerop (t
))
7666 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7667 counts
[i
] = build_int_cst (type
, 0);
7671 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7673 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7675 if (i
>= fd
->collapse
&& counts
[i
])
7677 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7678 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7679 fold_convert (itype
, fd
->loops
[i
].n1
),
7680 fold_convert (itype
, fd
->loops
[i
].n2
)))
7681 == NULL_TREE
|| !integer_onep (t
)))
7685 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7686 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7687 true, GSI_SAME_STMT
);
7688 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7689 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7690 true, GSI_SAME_STMT
);
7691 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7692 NULL_TREE
, NULL_TREE
);
7693 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7694 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7695 expand_omp_regimplify_p
, NULL
, NULL
)
7696 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7697 expand_omp_regimplify_p
, NULL
, NULL
))
7699 *gsi
= gsi_for_stmt (cond_stmt
);
7700 gimple_regimplify_operands (cond_stmt
, gsi
);
7702 e
= split_block (entry_bb
, cond_stmt
);
7703 basic_block
&zero_iter_bb
7704 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7705 int &first_zero_iter
7706 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7707 if (zero_iter_bb
== NULL
)
7709 gassign
*assign_stmt
;
7710 first_zero_iter
= i
;
7711 zero_iter_bb
= create_empty_bb (entry_bb
);
7712 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7713 *gsi
= gsi_after_labels (zero_iter_bb
);
7714 if (i
< fd
->collapse
)
7715 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7716 build_zero_cst (type
));
7719 counts
[i
] = create_tmp_reg (type
, ".count");
7721 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7723 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7724 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7727 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7728 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7729 e
->flags
= EDGE_TRUE_VALUE
;
7730 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7731 if (l2_dom_bb
== NULL
)
7732 l2_dom_bb
= entry_bb
;
7734 *gsi
= gsi_last_bb (entry_bb
);
7737 if (POINTER_TYPE_P (itype
))
7738 itype
= signed_type_for (itype
);
7739 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7741 t
= fold_build2 (PLUS_EXPR
, itype
,
7742 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7743 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7744 fold_convert (itype
, fd
->loops
[i
].n2
));
7745 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7746 fold_convert (itype
, fd
->loops
[i
].n1
));
7747 /* ?? We could probably use CEIL_DIV_EXPR instead of
7748 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7749 generate the same code in the end because generically we
7750 don't know that the values involved must be negative for
7752 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7753 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7754 fold_build1 (NEGATE_EXPR
, itype
, t
),
7755 fold_build1 (NEGATE_EXPR
, itype
,
7756 fold_convert (itype
,
7757 fd
->loops
[i
].step
)));
7759 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7760 fold_convert (itype
, fd
->loops
[i
].step
));
7761 t
= fold_convert (type
, t
);
7762 if (TREE_CODE (t
) == INTEGER_CST
)
7766 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7767 counts
[i
] = create_tmp_reg (type
, ".count");
7768 expand_omp_build_assign (gsi
, counts
[i
], t
);
7770 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7775 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7776 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7782 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7784 V3 = N31 + (T % count3) * STEP3;
7786 V2 = N21 + (T % count2) * STEP2;
7788 V1 = N11 + T * STEP1;
7789 if this loop doesn't have an inner loop construct combined with it.
7790 If it does have an inner loop construct combined with it and the
7791 iteration count isn't known constant, store values from counts array
7792 into its _looptemp_ temporaries instead. */
7795 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7796 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7799 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7801 /* If fd->loop.n2 is constant, then no propagation of the counts
7802 is needed, they are constant. */
7803 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7806 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7807 ? gimple_omp_taskreg_clauses (inner_stmt
)
7808 : gimple_omp_for_clauses (inner_stmt
);
7809 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7810 isn't supposed to be handled, as the inner loop doesn't
7812 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7813 gcc_assert (innerc
);
7814 for (i
= 0; i
< fd
->collapse
; i
++)
7816 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7817 OMP_CLAUSE__LOOPTEMP_
);
7818 gcc_assert (innerc
);
7821 tree tem
= OMP_CLAUSE_DECL (innerc
);
7822 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7823 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7824 false, GSI_CONTINUE_LINKING
);
7825 gassign
*stmt
= gimple_build_assign (tem
, t
);
7826 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7832 tree type
= TREE_TYPE (fd
->loop
.v
);
7833 tree tem
= create_tmp_reg (type
, ".tem");
7834 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7835 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7837 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7839 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7841 if (POINTER_TYPE_P (vtype
))
7842 itype
= signed_type_for (vtype
);
7844 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7847 t
= fold_convert (itype
, t
);
7848 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7849 fold_convert (itype
, fd
->loops
[i
].step
));
7850 if (POINTER_TYPE_P (vtype
))
7851 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7853 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7854 t
= force_gimple_operand_gsi (gsi
, t
,
7855 DECL_P (fd
->loops
[i
].v
)
7856 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7858 GSI_CONTINUE_LINKING
);
7859 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7860 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7863 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7864 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7865 false, GSI_CONTINUE_LINKING
);
7866 stmt
= gimple_build_assign (tem
, t
);
7867 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7873 /* Helper function for expand_omp_for_*. Generate code like:
7876 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7880 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7887 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7888 basic_block body_bb
)
7890 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7892 gimple_stmt_iterator gsi
;
7898 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7900 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7902 bb
= create_empty_bb (last_bb
);
7903 add_bb_to_loop (bb
, last_bb
->loop_father
);
7904 gsi
= gsi_start_bb (bb
);
7906 if (i
< fd
->collapse
- 1)
7908 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7909 e
->probability
= REG_BR_PROB_BASE
/ 8;
7911 t
= fd
->loops
[i
+ 1].n1
;
7912 t
= force_gimple_operand_gsi (&gsi
, t
,
7913 DECL_P (fd
->loops
[i
+ 1].v
)
7914 && TREE_ADDRESSABLE (fd
->loops
[i
7917 GSI_CONTINUE_LINKING
);
7918 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7919 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7924 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7926 if (POINTER_TYPE_P (vtype
))
7927 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7929 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7930 t
= force_gimple_operand_gsi (&gsi
, t
,
7931 DECL_P (fd
->loops
[i
].v
)
7932 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7933 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7934 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7935 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7939 t
= fd
->loops
[i
].n2
;
7940 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7941 false, GSI_CONTINUE_LINKING
);
7942 tree v
= fd
->loops
[i
].v
;
7943 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7944 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7945 false, GSI_CONTINUE_LINKING
);
7946 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7947 stmt
= gimple_build_cond_empty (t
);
7948 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7949 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7950 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7953 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7961 /* Expand #pragma omp ordered depend(source). */
7964 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7965 tree
*counts
, location_t loc
)
7967 enum built_in_function source_ix
7968 = fd
->iter_type
== long_integer_type_node
7969 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7971 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7972 build_fold_addr_expr (counts
[fd
->ordered
]));
7973 gimple_set_location (g
, loc
);
7974 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7977 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7980 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7981 tree
*counts
, tree c
, location_t loc
)
7983 auto_vec
<tree
, 10> args
;
7984 enum built_in_function sink_ix
7985 = fd
->iter_type
== long_integer_type_node
7986 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7987 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7989 gimple_stmt_iterator gsi2
= *gsi
;
7990 bool warned_step
= false;
7992 for (i
= 0; i
< fd
->ordered
; i
++)
7994 off
= TREE_PURPOSE (deps
);
7995 if (!integer_zerop (off
))
7997 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7998 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7999 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
8000 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8001 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
8002 "lexically later iteration");
8005 deps
= TREE_CHAIN (deps
);
8007 /* If all offsets corresponding to the collapsed loops are zero,
8008 this depend clause can be ignored. FIXME: but there is still a
8009 flush needed. We need to emit one __sync_synchronize () for it
8010 though (perhaps conditionally)? Solve this together with the
8011 conservative dependence folding optimization.
8012 if (i >= fd->collapse)
8015 deps
= OMP_CLAUSE_DECL (c
);
8017 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
8018 edge e2
= split_block_after_labels (e1
->dest
);
8020 *gsi
= gsi_after_labels (e1
->dest
);
8021 for (i
= 0; i
< fd
->ordered
; i
++)
8023 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
8024 if (POINTER_TYPE_P (itype
))
8027 deps
= TREE_CHAIN (deps
);
8028 off
= TREE_PURPOSE (deps
);
8029 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
8031 if (integer_zerop (off
))
8032 t
= boolean_true_node
;
8036 tree co
= fold_convert_loc (loc
, itype
, off
);
8037 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
8039 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8040 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
8041 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
8042 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
8045 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8046 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8047 fd
->loops
[i
].v
, co
);
8049 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8050 fd
->loops
[i
].v
, co
);
8051 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
8053 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8054 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
8057 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
8060 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8061 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
8064 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
8068 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
8072 off
= fold_convert_loc (loc
, itype
, off
);
8074 if (fd
->loops
[i
].cond_code
== LT_EXPR
8075 ? !integer_onep (fd
->loops
[i
].step
)
8076 : !integer_minus_onep (fd
->loops
[i
].step
))
8078 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8079 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
8080 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8083 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
8084 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
8085 build_int_cst (itype
, 0));
8086 if (integer_zerop (t
) && !warned_step
)
8088 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
8089 "in the iteration space");
8092 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
8096 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8102 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8103 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8104 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
8106 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8107 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
8108 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8111 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
8112 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8113 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
8114 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
8115 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8118 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
8120 if (i
< fd
->collapse
- 1)
8122 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
8127 off
= unshare_expr (off
);
8128 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
8129 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
8130 true, GSI_SAME_STMT
);
8133 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
8134 gimple_set_location (g
, loc
);
8135 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
8137 *gsi
= gsi_last_bb (e1
->src
);
8138 cond
= unshare_expr (cond
);
8139 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
8140 GSI_CONTINUE_LINKING
);
8141 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
8142 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
8143 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8144 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8145 e1
->flags
= EDGE_TRUE_VALUE
;
8146 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
8148 *gsi
= gsi_after_labels (e2
->dest
);
8151 /* Expand all #pragma omp ordered depend(source) and
8152 #pragma omp ordered depend(sink:...) constructs in the current
8153 #pragma omp for ordered(n) region. */
8156 expand_omp_ordered_source_sink (struct omp_region
*region
,
8157 struct omp_for_data
*fd
, tree
*counts
,
8158 basic_block cont_bb
)
8160 struct omp_region
*inner
;
8162 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
8163 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
8164 counts
[i
] = NULL_TREE
;
8165 else if (i
>= fd
->collapse
&& !cont_bb
)
8166 counts
[i
] = build_zero_cst (fd
->iter_type
);
8167 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
8168 && integer_onep (fd
->loops
[i
].step
))
8169 counts
[i
] = NULL_TREE
;
8171 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
8173 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
8174 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
8175 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
8177 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
8178 if (inner
->type
== GIMPLE_OMP_ORDERED
)
8180 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
8181 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
8182 location_t loc
= gimple_location (ord_stmt
);
8184 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8185 c
; c
= OMP_CLAUSE_CHAIN (c
))
8186 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
8189 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
8190 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8191 c
; c
= OMP_CLAUSE_CHAIN (c
))
8192 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8193 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
8194 gsi_remove (&gsi
, true);
8198 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8202 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
8203 basic_block cont_bb
, basic_block body_bb
,
8204 bool ordered_lastprivate
)
8206 if (fd
->ordered
== fd
->collapse
)
8211 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8212 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8214 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8215 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
8216 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
8217 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8218 size_int (i
- fd
->collapse
+ 1),
8219 NULL_TREE
, NULL_TREE
);
8220 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8225 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
8227 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
8228 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8229 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8230 fold_convert (type
, fd
->loops
[i
].n1
));
8232 expand_omp_build_assign (&gsi
, counts
[i
],
8233 build_zero_cst (fd
->iter_type
));
8234 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8235 size_int (i
- fd
->collapse
+ 1),
8236 NULL_TREE
, NULL_TREE
);
8237 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8238 if (!gsi_end_p (gsi
))
8241 gsi
= gsi_last_bb (body_bb
);
8242 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
8243 basic_block new_body
= e1
->dest
;
8244 if (body_bb
== cont_bb
)
8247 basic_block new_header
;
8248 if (EDGE_COUNT (cont_bb
->preds
) > 0)
8250 gsi
= gsi_last_bb (cont_bb
);
8251 if (POINTER_TYPE_P (type
))
8252 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8253 fold_convert (sizetype
,
8254 fd
->loops
[i
].step
));
8256 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8257 fold_convert (type
, fd
->loops
[i
].step
));
8258 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8261 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8262 build_int_cst (fd
->iter_type
, 1));
8263 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8268 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8269 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8270 t
= fold_convert (fd
->iter_type
, t
);
8271 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8272 true, GSI_SAME_STMT
);
8274 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8275 size_int (i
- fd
->collapse
+ 1),
8276 NULL_TREE
, NULL_TREE
);
8277 expand_omp_build_assign (&gsi
, aref
, t
);
8279 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8280 new_header
= e2
->dest
;
8283 new_header
= cont_bb
;
8284 gsi
= gsi_after_labels (new_header
);
8285 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8286 true, GSI_SAME_STMT
);
8288 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8289 true, NULL_TREE
, true, GSI_SAME_STMT
);
8290 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8291 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8292 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8295 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8296 e3
->flags
= EDGE_FALSE_VALUE
;
8297 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8298 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8299 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8301 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8302 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8306 struct loop
*loop
= alloc_loop ();
8307 loop
->header
= new_header
;
8308 loop
->latch
= e2
->src
;
8309 add_loop (loop
, body_bb
->loop_father
);
8313 /* If there are any lastprivate clauses and it is possible some loops
8314 might have zero iterations, ensure all the decls are initialized,
8315 otherwise we could crash evaluating C++ class iterators with lastprivate
8317 bool need_inits
= false;
8318 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8321 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8322 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8323 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8324 fold_convert (type
, fd
->loops
[i
].n1
));
8328 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8329 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8331 fold_convert (type
, fd
->loops
[i
].n1
),
8332 fold_convert (type
, fd
->loops
[i
].n2
));
8333 if (!integer_onep (this_cond
))
8341 /* A subroutine of expand_omp_for. Generate code for a parallel
8342 loop with any schedule. Given parameters:
8344 for (V = N1; V cond N2; V += STEP) BODY;
8346 where COND is "<" or ">", we generate pseudocode
8348 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8349 if (more) goto L0; else goto L3;
8356 if (V cond iend) goto L1; else goto L2;
8358 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8361 If this is a combined omp parallel loop, instead of the call to
8362 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8363 If this is gimple_omp_for_combined_p loop, then instead of assigning
8364 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8365 inner GIMPLE_OMP_FOR and V += STEP; and
8366 if (V cond iend) goto L1; else goto L2; are removed.
8368 For collapsed loops, given parameters:
8370 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8371 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8372 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8375 we generate pseudocode
8377 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8382 count3 = (adj + N32 - N31) / STEP3;
8383 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8388 count2 = (adj + N22 - N21) / STEP2;
8389 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8394 count1 = (adj + N12 - N11) / STEP1;
8395 count = count1 * count2 * count3;
8400 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8401 if (more) goto L0; else goto L3;
8405 V3 = N31 + (T % count3) * STEP3;
8407 V2 = N21 + (T % count2) * STEP2;
8409 V1 = N11 + T * STEP1;
8414 if (V < iend) goto L10; else goto L2;
8417 if (V3 cond3 N32) goto L1; else goto L11;
8421 if (V2 cond2 N22) goto L1; else goto L12;
8427 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8433 expand_omp_for_generic (struct omp_region
*region
,
8434 struct omp_for_data
*fd
,
8435 enum built_in_function start_fn
,
8436 enum built_in_function next_fn
,
8439 tree type
, istart0
, iend0
, iend
;
8440 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8441 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8442 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8443 gimple_stmt_iterator gsi
;
8444 gassign
*assign_stmt
;
8445 bool in_combined_parallel
= is_combined_parallel (region
);
8446 bool broken_loop
= region
->cont
== NULL
;
8448 tree
*counts
= NULL
;
8450 bool ordered_lastprivate
= false;
8452 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8453 gcc_assert (fd
->iter_type
== long_integer_type_node
8454 || !in_combined_parallel
);
8456 entry_bb
= region
->entry
;
8457 cont_bb
= region
->cont
;
8459 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8460 gcc_assert (broken_loop
8461 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8462 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8463 l1_bb
= single_succ (l0_bb
);
8466 l2_bb
= create_empty_bb (cont_bb
);
8467 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8468 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8470 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8474 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8475 exit_bb
= region
->exit
;
8477 gsi
= gsi_last_bb (entry_bb
);
8479 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8481 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8482 OMP_CLAUSE_LASTPRIVATE
))
8483 ordered_lastprivate
= false;
8484 if (fd
->collapse
> 1 || fd
->ordered
)
8486 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8487 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8489 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8490 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8491 zero_iter1_bb
, first_zero_iter1
,
8492 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8496 /* Some counts[i] vars might be uninitialized if
8497 some loop has zero iterations. But the body shouldn't
8498 be executed in that case, so just avoid uninit warnings. */
8499 for (i
= first_zero_iter1
;
8500 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8501 if (SSA_VAR_P (counts
[i
]))
8502 TREE_NO_WARNING (counts
[i
]) = 1;
8504 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8506 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8507 gsi
= gsi_last_bb (entry_bb
);
8508 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8509 get_immediate_dominator (CDI_DOMINATORS
,
8514 /* Some counts[i] vars might be uninitialized if
8515 some loop has zero iterations. But the body shouldn't
8516 be executed in that case, so just avoid uninit warnings. */
8517 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8518 if (SSA_VAR_P (counts
[i
]))
8519 TREE_NO_WARNING (counts
[i
]) = 1;
8521 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8525 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8527 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8528 gsi
= gsi_last_bb (entry_bb
);
8529 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8530 get_immediate_dominator
8531 (CDI_DOMINATORS
, zero_iter2_bb
));
8534 if (fd
->collapse
== 1)
8536 counts
[0] = fd
->loop
.n2
;
8537 fd
->loop
= fd
->loops
[0];
8541 type
= TREE_TYPE (fd
->loop
.v
);
8542 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8543 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8544 TREE_ADDRESSABLE (istart0
) = 1;
8545 TREE_ADDRESSABLE (iend0
) = 1;
8547 /* See if we need to bias by LLONG_MIN. */
8548 if (fd
->iter_type
== long_long_unsigned_type_node
8549 && TREE_CODE (type
) == INTEGER_TYPE
8550 && !TYPE_UNSIGNED (type
)
8551 && fd
->ordered
== 0)
8555 if (fd
->loop
.cond_code
== LT_EXPR
)
8558 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8562 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8565 if (TREE_CODE (n1
) != INTEGER_CST
8566 || TREE_CODE (n2
) != INTEGER_CST
8567 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8568 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8571 gimple_stmt_iterator gsif
= gsi
;
8574 tree arr
= NULL_TREE
;
8575 if (in_combined_parallel
)
8577 gcc_assert (fd
->ordered
== 0);
8578 /* In a combined parallel loop, emit a call to
8579 GOMP_loop_foo_next. */
8580 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8581 build_fold_addr_expr (istart0
),
8582 build_fold_addr_expr (iend0
));
8586 tree t0
, t1
, t2
, t3
, t4
;
8587 /* If this is not a combined parallel loop, emit a call to
8588 GOMP_loop_foo_start in ENTRY_BB. */
8589 t4
= build_fold_addr_expr (iend0
);
8590 t3
= build_fold_addr_expr (istart0
);
8593 t0
= build_int_cst (unsigned_type_node
,
8594 fd
->ordered
- fd
->collapse
+ 1);
8595 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8597 - fd
->collapse
+ 1),
8599 DECL_NAMELESS (arr
) = 1;
8600 TREE_ADDRESSABLE (arr
) = 1;
8601 TREE_STATIC (arr
) = 1;
8602 vec
<constructor_elt
, va_gc
> *v
;
8603 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8606 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8609 if (idx
== 0 && fd
->collapse
> 1)
8612 c
= counts
[idx
+ fd
->collapse
- 1];
8613 tree purpose
= size_int (idx
);
8614 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8615 if (TREE_CODE (c
) != INTEGER_CST
)
8616 TREE_STATIC (arr
) = 0;
8619 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8620 if (!TREE_STATIC (arr
))
8621 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8622 void_type_node
, arr
),
8623 true, NULL_TREE
, true, GSI_SAME_STMT
);
8624 t1
= build_fold_addr_expr (arr
);
8629 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8632 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8635 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8636 OMP_CLAUSE__LOOPTEMP_
);
8637 gcc_assert (innerc
);
8638 t0
= OMP_CLAUSE_DECL (innerc
);
8639 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8640 OMP_CLAUSE__LOOPTEMP_
);
8641 gcc_assert (innerc
);
8642 t1
= OMP_CLAUSE_DECL (innerc
);
8644 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8645 && TYPE_PRECISION (TREE_TYPE (t0
))
8646 != TYPE_PRECISION (fd
->iter_type
))
8648 /* Avoid casting pointers to integer of a different size. */
8649 tree itype
= signed_type_for (type
);
8650 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8651 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8655 t1
= fold_convert (fd
->iter_type
, t1
);
8656 t0
= fold_convert (fd
->iter_type
, t0
);
8660 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8661 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8664 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8668 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8669 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8671 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8672 5, t0
, t1
, t
, t3
, t4
);
8674 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8675 6, t0
, t1
, t2
, t
, t3
, t4
);
8677 else if (fd
->ordered
)
8678 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8681 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8682 5, t0
, t1
, t2
, t3
, t4
);
8690 /* The GOMP_loop_ull_*start functions have additional boolean
8691 argument, true for < loops and false for > loops.
8692 In Fortran, the C bool type can be different from
8693 boolean_type_node. */
8694 bfn_decl
= builtin_decl_explicit (start_fn
);
8695 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8696 t5
= build_int_cst (c_bool_type
,
8697 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8700 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8701 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8702 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8703 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8706 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8707 6, t5
, t0
, t1
, t2
, t3
, t4
);
8710 if (TREE_TYPE (t
) != boolean_type_node
)
8711 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8712 t
, build_int_cst (TREE_TYPE (t
), 0));
8713 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8714 true, GSI_SAME_STMT
);
8715 if (arr
&& !TREE_STATIC (arr
))
8717 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8718 TREE_THIS_VOLATILE (clobber
) = 1;
8719 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8722 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8724 /* Remove the GIMPLE_OMP_FOR statement. */
8725 gsi_remove (&gsi
, true);
8727 if (gsi_end_p (gsif
))
8728 gsif
= gsi_after_labels (gsi_bb (gsif
));
8731 /* Iteration setup for sequential loop goes in L0_BB. */
8732 tree startvar
= fd
->loop
.v
;
8733 tree endvar
= NULL_TREE
;
8735 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8737 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8738 && gimple_omp_for_kind (inner_stmt
)
8739 == GF_OMP_FOR_KIND_SIMD
);
8740 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8741 OMP_CLAUSE__LOOPTEMP_
);
8742 gcc_assert (innerc
);
8743 startvar
= OMP_CLAUSE_DECL (innerc
);
8744 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8745 OMP_CLAUSE__LOOPTEMP_
);
8746 gcc_assert (innerc
);
8747 endvar
= OMP_CLAUSE_DECL (innerc
);
8750 gsi
= gsi_start_bb (l0_bb
);
8752 if (fd
->ordered
&& fd
->collapse
== 1)
8753 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8754 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8756 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8757 if (fd
->ordered
&& fd
->collapse
== 1)
8759 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8760 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8761 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8764 t
= fold_convert (TREE_TYPE (startvar
), t
);
8765 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8771 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8772 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8773 t
= fold_convert (TREE_TYPE (startvar
), t
);
8775 t
= force_gimple_operand_gsi (&gsi
, t
,
8777 && TREE_ADDRESSABLE (startvar
),
8778 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8779 assign_stmt
= gimple_build_assign (startvar
, t
);
8780 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8783 if (fd
->ordered
&& fd
->collapse
== 1)
8784 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8785 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8787 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8788 if (fd
->ordered
&& fd
->collapse
== 1)
8790 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8791 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8792 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8795 t
= fold_convert (TREE_TYPE (startvar
), t
);
8796 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8802 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8803 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8804 t
= fold_convert (TREE_TYPE (startvar
), t
);
8806 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8807 false, GSI_CONTINUE_LINKING
);
8810 assign_stmt
= gimple_build_assign (endvar
, iend
);
8811 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8812 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8813 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8815 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8816 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8818 /* Handle linear clause adjustments. */
8819 tree itercnt
= NULL_TREE
;
8820 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8821 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8822 c
; c
= OMP_CLAUSE_CHAIN (c
))
8823 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8824 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8826 tree d
= OMP_CLAUSE_DECL (c
);
8827 bool is_ref
= is_reference (d
);
8828 tree t
= d
, a
, dest
;
8830 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8831 tree type
= TREE_TYPE (t
);
8832 if (POINTER_TYPE_P (type
))
8834 dest
= unshare_expr (t
);
8835 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8836 expand_omp_build_assign (&gsif
, v
, t
);
8837 if (itercnt
== NULL_TREE
)
8840 tree n1
= fd
->loop
.n1
;
8841 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8844 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8846 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8848 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8850 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8851 itercnt
, fd
->loop
.step
);
8852 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8854 GSI_CONTINUE_LINKING
);
8856 a
= fold_build2 (MULT_EXPR
, type
,
8857 fold_convert (type
, itercnt
),
8858 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8859 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8860 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8861 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8862 false, GSI_CONTINUE_LINKING
);
8863 assign_stmt
= gimple_build_assign (dest
, t
);
8864 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8866 if (fd
->collapse
> 1)
8867 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8871 /* Until now, counts array contained number of iterations or
8872 variable containing it for ith loop. From now on, we need
8873 those counts only for collapsed loops, and only for the 2nd
8874 till the last collapsed one. Move those one element earlier,
8875 we'll use counts[fd->collapse - 1] for the first source/sink
8876 iteration counter and so on and counts[fd->ordered]
8877 as the array holding the current counter values for
8879 if (fd
->collapse
> 1)
8880 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8884 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8886 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8888 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8889 fold_convert (type
, fd
->loops
[i
].n1
),
8890 fold_convert (type
, fd
->loops
[i
].n2
));
8891 if (!integer_onep (this_cond
))
8894 if (i
< fd
->ordered
)
8897 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8898 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8899 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8900 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8901 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8902 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8903 make_edge (cont_bb
, l1_bb
, 0);
8904 l2_bb
= create_empty_bb (cont_bb
);
8905 broken_loop
= false;
8908 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8909 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8910 ordered_lastprivate
);
8911 if (counts
[fd
->collapse
- 1])
8913 gcc_assert (fd
->collapse
== 1);
8914 gsi
= gsi_last_bb (l0_bb
);
8915 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8917 gsi
= gsi_last_bb (cont_bb
);
8918 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8919 build_int_cst (fd
->iter_type
, 1));
8920 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8921 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8922 size_zero_node
, NULL_TREE
, NULL_TREE
);
8923 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8924 t
= counts
[fd
->collapse
- 1];
8926 else if (fd
->collapse
> 1)
8930 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8931 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8932 t
= fold_convert (fd
->iter_type
, t
);
8934 gsi
= gsi_last_bb (l0_bb
);
8935 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8936 size_zero_node
, NULL_TREE
, NULL_TREE
);
8937 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8938 false, GSI_CONTINUE_LINKING
);
8939 expand_omp_build_assign (&gsi
, aref
, t
, true);
8944 /* Code to control the increment and predicate for the sequential
8945 loop goes in the CONT_BB. */
8946 gsi
= gsi_last_bb (cont_bb
);
8947 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8948 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8949 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8950 vback
= gimple_omp_continue_control_def (cont_stmt
);
8952 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8954 if (POINTER_TYPE_P (type
))
8955 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8957 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8958 t
= force_gimple_operand_gsi (&gsi
, t
,
8960 && TREE_ADDRESSABLE (vback
),
8961 NULL_TREE
, true, GSI_SAME_STMT
);
8962 assign_stmt
= gimple_build_assign (vback
, t
);
8963 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8965 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8967 if (fd
->collapse
> 1)
8971 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8972 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8973 t
= fold_convert (fd
->iter_type
, t
);
8975 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8976 counts
[fd
->ordered
], size_zero_node
,
8977 NULL_TREE
, NULL_TREE
);
8978 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8979 true, GSI_SAME_STMT
);
8980 expand_omp_build_assign (&gsi
, aref
, t
);
8983 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8984 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8986 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8987 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8990 /* Remove GIMPLE_OMP_CONTINUE. */
8991 gsi_remove (&gsi
, true);
8993 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8994 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8996 /* Emit code to get the next parallel iteration in L2_BB. */
8997 gsi
= gsi_start_bb (l2_bb
);
8999 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
9000 build_fold_addr_expr (istart0
),
9001 build_fold_addr_expr (iend0
));
9002 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9003 false, GSI_CONTINUE_LINKING
);
9004 if (TREE_TYPE (t
) != boolean_type_node
)
9005 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
9006 t
, build_int_cst (TREE_TYPE (t
), 0));
9007 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9008 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
9011 /* Add the loop cleanup function. */
9012 gsi
= gsi_last_bb (exit_bb
);
9013 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9014 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
9015 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9016 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
9018 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
9019 gcall
*call_stmt
= gimple_build_call (t
, 0);
9020 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9021 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
9022 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
9025 tree arr
= counts
[fd
->ordered
];
9026 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
9027 TREE_THIS_VOLATILE (clobber
) = 1;
9028 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
9031 gsi_remove (&gsi
, true);
9033 /* Connect the new blocks. */
9034 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
9035 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
9041 e
= find_edge (cont_bb
, l3_bb
);
9042 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
9044 phis
= phi_nodes (l3_bb
);
9045 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9047 gimple
*phi
= gsi_stmt (gsi
);
9048 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
9049 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
9053 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
9054 e
= find_edge (cont_bb
, l1_bb
);
9057 e
= BRANCH_EDGE (cont_bb
);
9058 gcc_assert (single_succ (e
->dest
) == l1_bb
);
9060 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9065 else if (fd
->collapse
> 1)
9068 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9071 e
->flags
= EDGE_TRUE_VALUE
;
9074 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9075 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
9079 e
= find_edge (cont_bb
, l2_bb
);
9080 e
->flags
= EDGE_FALLTHRU
;
9082 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9084 if (gimple_in_ssa_p (cfun
))
9086 /* Add phis to the outer loop that connect to the phis in the inner,
9087 original loop, and move the loop entry value of the inner phi to
9088 the loop entry value of the outer phi. */
9090 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
9092 source_location locus
;
9094 gphi
*exit_phi
= psi
.phi ();
9096 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
9097 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
9099 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
9100 edge latch_to_l1
= find_edge (latch
, l1_bb
);
9102 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
9104 tree t
= gimple_phi_result (exit_phi
);
9105 tree new_res
= copy_ssa_name (t
, NULL
);
9106 nphi
= create_phi_node (new_res
, l0_bb
);
9108 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
9109 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
9110 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
9111 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
9112 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
9114 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
9115 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
9117 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
9121 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
9122 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
9123 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
9124 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
9125 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
9126 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
9127 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
9128 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
9130 /* We enter expand_omp_for_generic with a loop. This original loop may
9131 have its own loop struct, or it may be part of an outer loop struct
9132 (which may be the fake loop). */
9133 struct loop
*outer_loop
= entry_bb
->loop_father
;
9134 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
9136 add_bb_to_loop (l2_bb
, outer_loop
);
9138 /* We've added a new loop around the original loop. Allocate the
9139 corresponding loop struct. */
9140 struct loop
*new_loop
= alloc_loop ();
9141 new_loop
->header
= l0_bb
;
9142 new_loop
->latch
= l2_bb
;
9143 add_loop (new_loop
, outer_loop
);
9145 /* Allocate a loop structure for the original loop unless we already
9147 if (!orig_loop_has_loop_struct
9148 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9150 struct loop
*orig_loop
= alloc_loop ();
9151 orig_loop
->header
= l1_bb
;
9152 /* The loop may have multiple latches. */
9153 add_loop (orig_loop
, new_loop
);
9159 /* A subroutine of expand_omp_for. Generate code for a parallel
9160 loop with static schedule and no specified chunk size. Given
9163 for (V = N1; V cond N2; V += STEP) BODY;
9165 where COND is "<" or ">", we generate pseudocode
9167 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9172 if ((__typeof (V)) -1 > 0 && cond is >)
9173 n = -(adj + N2 - N1) / -STEP;
9175 n = (adj + N2 - N1) / STEP;
9178 if (threadid < tt) goto L3; else goto L4;
9183 s0 = q * threadid + tt;
9186 if (s0 >= e0) goto L2; else goto L0;
9192 if (V cond e) goto L1;
9197 expand_omp_for_static_nochunk (struct omp_region
*region
,
9198 struct omp_for_data
*fd
,
9201 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
9202 tree type
, itype
, vmain
, vback
;
9203 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
9204 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
9206 gimple_stmt_iterator gsi
;
9208 bool broken_loop
= region
->cont
== NULL
;
9209 tree
*counts
= NULL
;
9212 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9213 if (POINTER_TYPE_P (type
))
9214 itype
= signed_type_for (type
);
9216 entry_bb
= region
->entry
;
9217 cont_bb
= region
->cont
;
9218 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9219 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9220 gcc_assert (broken_loop
9221 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
9222 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
9223 body_bb
= single_succ (seq_start_bb
);
9226 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9227 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9228 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9230 exit_bb
= region
->exit
;
9232 /* Iteration space partitioning goes in ENTRY_BB. */
9233 gsi
= gsi_last_bb (entry_bb
);
9234 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9236 if (fd
->collapse
> 1)
9238 int first_zero_iter
= -1, dummy
= -1;
9239 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9241 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9242 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9243 fin_bb
, first_zero_iter
,
9244 dummy_bb
, dummy
, l2_dom_bb
);
9247 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9248 t
= integer_one_node
;
9250 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9251 fold_convert (type
, fd
->loop
.n1
),
9252 fold_convert (type
, fd
->loop
.n2
));
9253 if (fd
->collapse
== 1
9254 && TYPE_UNSIGNED (type
)
9255 && (t
== NULL_TREE
|| !integer_onep (t
)))
9257 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9258 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9259 true, GSI_SAME_STMT
);
9260 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9261 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9262 true, GSI_SAME_STMT
);
9263 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9264 NULL_TREE
, NULL_TREE
);
9265 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9266 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9267 expand_omp_regimplify_p
, NULL
, NULL
)
9268 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9269 expand_omp_regimplify_p
, NULL
, NULL
))
9271 gsi
= gsi_for_stmt (cond_stmt
);
9272 gimple_regimplify_operands (cond_stmt
, &gsi
);
9274 ep
= split_block (entry_bb
, cond_stmt
);
9275 ep
->flags
= EDGE_TRUE_VALUE
;
9276 entry_bb
= ep
->dest
;
9277 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9278 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9279 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9280 if (gimple_in_ssa_p (cfun
))
9282 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9283 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9284 !gsi_end_p (gpi
); gsi_next (&gpi
))
9286 gphi
*phi
= gpi
.phi ();
9287 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9288 ep
, UNKNOWN_LOCATION
);
9291 gsi
= gsi_last_bb (entry_bb
);
9294 switch (gimple_omp_for_kind (fd
->for_stmt
))
9296 case GF_OMP_FOR_KIND_FOR
:
9297 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9298 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9300 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9301 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9302 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9307 nthreads
= build_call_expr (nthreads
, 0);
9308 nthreads
= fold_convert (itype
, nthreads
);
9309 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9310 true, GSI_SAME_STMT
);
9311 threadid
= build_call_expr (threadid
, 0);
9312 threadid
= fold_convert (itype
, threadid
);
9313 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9314 true, GSI_SAME_STMT
);
9318 step
= fd
->loop
.step
;
9319 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9321 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9322 OMP_CLAUSE__LOOPTEMP_
);
9323 gcc_assert (innerc
);
9324 n1
= OMP_CLAUSE_DECL (innerc
);
9325 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9326 OMP_CLAUSE__LOOPTEMP_
);
9327 gcc_assert (innerc
);
9328 n2
= OMP_CLAUSE_DECL (innerc
);
9330 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9331 true, NULL_TREE
, true, GSI_SAME_STMT
);
9332 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9333 true, NULL_TREE
, true, GSI_SAME_STMT
);
9334 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9335 true, NULL_TREE
, true, GSI_SAME_STMT
);
9337 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9338 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9339 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9340 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9341 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9342 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9343 fold_build1 (NEGATE_EXPR
, itype
, t
),
9344 fold_build1 (NEGATE_EXPR
, itype
, step
));
9346 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9347 t
= fold_convert (itype
, t
);
9348 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9350 q
= create_tmp_reg (itype
, "q");
9351 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9352 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9353 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9355 tt
= create_tmp_reg (itype
, "tt");
9356 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9357 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9358 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9360 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9361 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9362 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9364 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9365 gsi
= gsi_last_bb (second_bb
);
9366 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9368 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9370 gassign
*assign_stmt
9371 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9372 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9374 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9375 gsi
= gsi_last_bb (third_bb
);
9376 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9378 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9379 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9380 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9382 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9383 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9385 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9386 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9388 /* Remove the GIMPLE_OMP_FOR statement. */
9389 gsi_remove (&gsi
, true);
9391 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9392 gsi
= gsi_start_bb (seq_start_bb
);
9394 tree startvar
= fd
->loop
.v
;
9395 tree endvar
= NULL_TREE
;
9397 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9399 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9400 ? gimple_omp_parallel_clauses (inner_stmt
)
9401 : gimple_omp_for_clauses (inner_stmt
);
9402 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9403 gcc_assert (innerc
);
9404 startvar
= OMP_CLAUSE_DECL (innerc
);
9405 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9406 OMP_CLAUSE__LOOPTEMP_
);
9407 gcc_assert (innerc
);
9408 endvar
= OMP_CLAUSE_DECL (innerc
);
9409 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9410 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9413 for (i
= 1; i
< fd
->collapse
; i
++)
9415 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9416 OMP_CLAUSE__LOOPTEMP_
);
9417 gcc_assert (innerc
);
9419 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9420 OMP_CLAUSE__LOOPTEMP_
);
9423 /* If needed (distribute parallel for with lastprivate),
9424 propagate down the total number of iterations. */
9425 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9427 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9428 GSI_CONTINUE_LINKING
);
9429 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9430 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9434 t
= fold_convert (itype
, s0
);
9435 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9436 if (POINTER_TYPE_P (type
))
9437 t
= fold_build_pointer_plus (n1
, t
);
9439 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9440 t
= fold_convert (TREE_TYPE (startvar
), t
);
9441 t
= force_gimple_operand_gsi (&gsi
, t
,
9443 && TREE_ADDRESSABLE (startvar
),
9444 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9445 assign_stmt
= gimple_build_assign (startvar
, t
);
9446 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9448 t
= fold_convert (itype
, e0
);
9449 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9450 if (POINTER_TYPE_P (type
))
9451 t
= fold_build_pointer_plus (n1
, t
);
9453 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9454 t
= fold_convert (TREE_TYPE (startvar
), t
);
9455 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9456 false, GSI_CONTINUE_LINKING
);
9459 assign_stmt
= gimple_build_assign (endvar
, e
);
9460 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9461 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9462 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9464 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9465 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9467 /* Handle linear clause adjustments. */
9468 tree itercnt
= NULL_TREE
;
9469 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9470 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9471 c
; c
= OMP_CLAUSE_CHAIN (c
))
9472 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9473 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9475 tree d
= OMP_CLAUSE_DECL (c
);
9476 bool is_ref
= is_reference (d
);
9477 tree t
= d
, a
, dest
;
9479 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9480 if (itercnt
== NULL_TREE
)
9482 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9484 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9485 fold_convert (itype
, n1
),
9486 fold_convert (itype
, fd
->loop
.n1
));
9487 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9488 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9489 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9491 GSI_CONTINUE_LINKING
);
9496 tree type
= TREE_TYPE (t
);
9497 if (POINTER_TYPE_P (type
))
9499 a
= fold_build2 (MULT_EXPR
, type
,
9500 fold_convert (type
, itercnt
),
9501 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9502 dest
= unshare_expr (t
);
9503 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9504 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9505 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9506 false, GSI_CONTINUE_LINKING
);
9507 assign_stmt
= gimple_build_assign (dest
, t
);
9508 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9510 if (fd
->collapse
> 1)
9511 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9515 /* The code controlling the sequential loop replaces the
9516 GIMPLE_OMP_CONTINUE. */
9517 gsi
= gsi_last_bb (cont_bb
);
9518 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9519 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9520 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9521 vback
= gimple_omp_continue_control_def (cont_stmt
);
9523 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9525 if (POINTER_TYPE_P (type
))
9526 t
= fold_build_pointer_plus (vmain
, step
);
9528 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9529 t
= force_gimple_operand_gsi (&gsi
, t
,
9531 && TREE_ADDRESSABLE (vback
),
9532 NULL_TREE
, true, GSI_SAME_STMT
);
9533 assign_stmt
= gimple_build_assign (vback
, t
);
9534 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9536 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9537 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9539 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9542 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9543 gsi_remove (&gsi
, true);
9545 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9546 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9549 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9550 gsi
= gsi_last_bb (exit_bb
);
9551 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9553 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9554 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9556 gsi_remove (&gsi
, true);
9558 /* Connect all the blocks. */
9559 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9560 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9561 ep
= find_edge (entry_bb
, second_bb
);
9562 ep
->flags
= EDGE_TRUE_VALUE
;
9563 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9564 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9565 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9569 ep
= find_edge (cont_bb
, body_bb
);
9572 ep
= BRANCH_EDGE (cont_bb
);
9573 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9575 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9580 else if (fd
->collapse
> 1)
9583 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9586 ep
->flags
= EDGE_TRUE_VALUE
;
9587 find_edge (cont_bb
, fin_bb
)->flags
9588 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9591 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9592 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9593 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9595 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9596 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9597 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9598 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9600 struct loop
*loop
= body_bb
->loop_father
;
9601 if (loop
!= entry_bb
->loop_father
)
9603 gcc_assert (loop
->header
== body_bb
);
9604 gcc_assert (broken_loop
9605 || loop
->latch
== region
->cont
9606 || single_pred (loop
->latch
) == region
->cont
);
9610 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9612 loop
= alloc_loop ();
9613 loop
->header
= body_bb
;
9614 if (collapse_bb
== NULL
)
9615 loop
->latch
= cont_bb
;
9616 add_loop (loop
, body_bb
->loop_father
);
9620 /* Return phi in E->DEST with ARG on edge E. */
9623 find_phi_with_arg_on_edge (tree arg
, edge e
)
9625 basic_block bb
= e
->dest
;
9627 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9631 gphi
*phi
= gpi
.phi ();
9632 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9639 /* A subroutine of expand_omp_for. Generate code for a parallel
9640 loop with static schedule and a specified chunk size. Given
9643 for (V = N1; V cond N2; V += STEP) BODY;
9645 where COND is "<" or ">", we generate pseudocode
9647 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9652 if ((__typeof (V)) -1 > 0 && cond is >)
9653 n = -(adj + N2 - N1) / -STEP;
9655 n = (adj + N2 - N1) / STEP;
9657 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9658 here so that V is defined
9659 if the loop is not entered
9661 s0 = (trip * nthreads + threadid) * CHUNK;
9662 e0 = min(s0 + CHUNK, n);
9663 if (s0 < n) goto L1; else goto L4;
9670 if (V cond e) goto L2; else goto L3;
9678 expand_omp_for_static_chunk (struct omp_region
*region
,
9679 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9681 tree n
, s0
, e0
, e
, t
;
9682 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9683 tree type
, itype
, vmain
, vback
, vextra
;
9684 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9685 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9686 gimple_stmt_iterator gsi
;
9688 bool broken_loop
= region
->cont
== NULL
;
9689 tree
*counts
= NULL
;
9692 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9693 if (POINTER_TYPE_P (type
))
9694 itype
= signed_type_for (type
);
9696 entry_bb
= region
->entry
;
9697 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9699 iter_part_bb
= se
->dest
;
9700 cont_bb
= region
->cont
;
9701 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9702 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9703 gcc_assert (broken_loop
9704 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9705 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9706 body_bb
= single_succ (seq_start_bb
);
9709 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9710 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9711 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9712 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9714 exit_bb
= region
->exit
;
9716 /* Trip and adjustment setup goes in ENTRY_BB. */
9717 gsi
= gsi_last_bb (entry_bb
);
9718 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9720 if (fd
->collapse
> 1)
9722 int first_zero_iter
= -1, dummy
= -1;
9723 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9725 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9726 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9727 fin_bb
, first_zero_iter
,
9728 dummy_bb
, dummy
, l2_dom_bb
);
9731 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9732 t
= integer_one_node
;
9734 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9735 fold_convert (type
, fd
->loop
.n1
),
9736 fold_convert (type
, fd
->loop
.n2
));
9737 if (fd
->collapse
== 1
9738 && TYPE_UNSIGNED (type
)
9739 && (t
== NULL_TREE
|| !integer_onep (t
)))
9741 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9742 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9743 true, GSI_SAME_STMT
);
9744 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9745 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9746 true, GSI_SAME_STMT
);
9747 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9748 NULL_TREE
, NULL_TREE
);
9749 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9750 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9751 expand_omp_regimplify_p
, NULL
, NULL
)
9752 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9753 expand_omp_regimplify_p
, NULL
, NULL
))
9755 gsi
= gsi_for_stmt (cond_stmt
);
9756 gimple_regimplify_operands (cond_stmt
, &gsi
);
9758 se
= split_block (entry_bb
, cond_stmt
);
9759 se
->flags
= EDGE_TRUE_VALUE
;
9760 entry_bb
= se
->dest
;
9761 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9762 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9763 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9764 if (gimple_in_ssa_p (cfun
))
9766 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9767 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9768 !gsi_end_p (gpi
); gsi_next (&gpi
))
9770 gphi
*phi
= gpi
.phi ();
9771 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9772 se
, UNKNOWN_LOCATION
);
9775 gsi
= gsi_last_bb (entry_bb
);
9778 switch (gimple_omp_for_kind (fd
->for_stmt
))
9780 case GF_OMP_FOR_KIND_FOR
:
9781 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9782 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9784 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9785 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9786 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9791 nthreads
= build_call_expr (nthreads
, 0);
9792 nthreads
= fold_convert (itype
, nthreads
);
9793 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9794 true, GSI_SAME_STMT
);
9795 threadid
= build_call_expr (threadid
, 0);
9796 threadid
= fold_convert (itype
, threadid
);
9797 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9798 true, GSI_SAME_STMT
);
9802 step
= fd
->loop
.step
;
9803 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9805 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9806 OMP_CLAUSE__LOOPTEMP_
);
9807 gcc_assert (innerc
);
9808 n1
= OMP_CLAUSE_DECL (innerc
);
9809 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9810 OMP_CLAUSE__LOOPTEMP_
);
9811 gcc_assert (innerc
);
9812 n2
= OMP_CLAUSE_DECL (innerc
);
9814 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9815 true, NULL_TREE
, true, GSI_SAME_STMT
);
9816 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9817 true, NULL_TREE
, true, GSI_SAME_STMT
);
9818 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9819 true, NULL_TREE
, true, GSI_SAME_STMT
);
9820 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9821 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9823 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9826 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9827 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9828 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9829 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9830 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9831 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9832 fold_build1 (NEGATE_EXPR
, itype
, t
),
9833 fold_build1 (NEGATE_EXPR
, itype
, step
));
9835 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9836 t
= fold_convert (itype
, t
);
9837 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9838 true, GSI_SAME_STMT
);
9840 trip_var
= create_tmp_reg (itype
, ".trip");
9841 if (gimple_in_ssa_p (cfun
))
9843 trip_init
= make_ssa_name (trip_var
);
9844 trip_main
= make_ssa_name (trip_var
);
9845 trip_back
= make_ssa_name (trip_var
);
9849 trip_init
= trip_var
;
9850 trip_main
= trip_var
;
9851 trip_back
= trip_var
;
9854 gassign
*assign_stmt
9855 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9856 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9858 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9859 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9860 if (POINTER_TYPE_P (type
))
9861 t
= fold_build_pointer_plus (n1
, t
);
9863 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9864 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9865 true, GSI_SAME_STMT
);
9867 /* Remove the GIMPLE_OMP_FOR. */
9868 gsi_remove (&gsi
, true);
9870 gimple_stmt_iterator gsif
= gsi
;
9872 /* Iteration space partitioning goes in ITER_PART_BB. */
9873 gsi
= gsi_last_bb (iter_part_bb
);
9875 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9876 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9877 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9878 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9879 false, GSI_CONTINUE_LINKING
);
9881 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9882 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9883 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9884 false, GSI_CONTINUE_LINKING
);
9886 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9887 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9889 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9890 gsi
= gsi_start_bb (seq_start_bb
);
9892 tree startvar
= fd
->loop
.v
;
9893 tree endvar
= NULL_TREE
;
9895 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9897 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9898 ? gimple_omp_parallel_clauses (inner_stmt
)
9899 : gimple_omp_for_clauses (inner_stmt
);
9900 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9901 gcc_assert (innerc
);
9902 startvar
= OMP_CLAUSE_DECL (innerc
);
9903 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9904 OMP_CLAUSE__LOOPTEMP_
);
9905 gcc_assert (innerc
);
9906 endvar
= OMP_CLAUSE_DECL (innerc
);
9907 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9908 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9911 for (i
= 1; i
< fd
->collapse
; i
++)
9913 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9914 OMP_CLAUSE__LOOPTEMP_
);
9915 gcc_assert (innerc
);
9917 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9918 OMP_CLAUSE__LOOPTEMP_
);
9921 /* If needed (distribute parallel for with lastprivate),
9922 propagate down the total number of iterations. */
9923 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9925 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9926 GSI_CONTINUE_LINKING
);
9927 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9928 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9933 t
= fold_convert (itype
, s0
);
9934 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9935 if (POINTER_TYPE_P (type
))
9936 t
= fold_build_pointer_plus (n1
, t
);
9938 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9939 t
= fold_convert (TREE_TYPE (startvar
), t
);
9940 t
= force_gimple_operand_gsi (&gsi
, t
,
9942 && TREE_ADDRESSABLE (startvar
),
9943 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9944 assign_stmt
= gimple_build_assign (startvar
, t
);
9945 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9947 t
= fold_convert (itype
, e0
);
9948 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9949 if (POINTER_TYPE_P (type
))
9950 t
= fold_build_pointer_plus (n1
, t
);
9952 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9953 t
= fold_convert (TREE_TYPE (startvar
), t
);
9954 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9955 false, GSI_CONTINUE_LINKING
);
9958 assign_stmt
= gimple_build_assign (endvar
, e
);
9959 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9960 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9961 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9963 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9964 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9966 /* Handle linear clause adjustments. */
9967 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9968 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9969 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9970 c
; c
= OMP_CLAUSE_CHAIN (c
))
9971 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9972 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9974 tree d
= OMP_CLAUSE_DECL (c
);
9975 bool is_ref
= is_reference (d
);
9976 tree t
= d
, a
, dest
;
9978 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9979 tree type
= TREE_TYPE (t
);
9980 if (POINTER_TYPE_P (type
))
9982 dest
= unshare_expr (t
);
9983 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9984 expand_omp_build_assign (&gsif
, v
, t
);
9985 if (itercnt
== NULL_TREE
)
9987 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9990 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9991 fold_convert (itype
, fd
->loop
.n1
));
9992 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9995 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9998 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9999 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
10001 GSI_CONTINUE_LINKING
);
10006 a
= fold_build2 (MULT_EXPR
, type
,
10007 fold_convert (type
, itercnt
),
10008 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
10009 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
10010 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
10011 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10012 false, GSI_CONTINUE_LINKING
);
10013 assign_stmt
= gimple_build_assign (dest
, t
);
10014 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10016 if (fd
->collapse
> 1)
10017 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10021 /* The code controlling the sequential loop goes in CONT_BB,
10022 replacing the GIMPLE_OMP_CONTINUE. */
10023 gsi
= gsi_last_bb (cont_bb
);
10024 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10025 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10026 vback
= gimple_omp_continue_control_def (cont_stmt
);
10028 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10030 if (POINTER_TYPE_P (type
))
10031 t
= fold_build_pointer_plus (vmain
, step
);
10033 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10034 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
10035 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10036 true, GSI_SAME_STMT
);
10037 assign_stmt
= gimple_build_assign (vback
, t
);
10038 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10040 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
10041 t
= build2 (EQ_EXPR
, boolean_type_node
,
10042 build_int_cst (itype
, 0),
10043 build_int_cst (itype
, 1));
10045 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10046 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10048 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10051 /* Remove GIMPLE_OMP_CONTINUE. */
10052 gsi_remove (&gsi
, true);
10054 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10055 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10057 /* Trip update code goes into TRIP_UPDATE_BB. */
10058 gsi
= gsi_start_bb (trip_update_bb
);
10060 t
= build_int_cst (itype
, 1);
10061 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
10062 assign_stmt
= gimple_build_assign (trip_back
, t
);
10063 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10066 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10067 gsi
= gsi_last_bb (exit_bb
);
10068 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
10070 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
10071 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
10073 gsi_remove (&gsi
, true);
10075 /* Connect the new blocks. */
10076 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
10077 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
10081 se
= find_edge (cont_bb
, body_bb
);
10084 se
= BRANCH_EDGE (cont_bb
);
10085 gcc_assert (single_succ (se
->dest
) == body_bb
);
10087 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10092 else if (fd
->collapse
> 1)
10095 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10098 se
->flags
= EDGE_TRUE_VALUE
;
10099 find_edge (cont_bb
, trip_update_bb
)->flags
10100 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10102 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
10105 if (gimple_in_ssa_p (cfun
))
10113 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
10115 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10116 remove arguments of the phi nodes in fin_bb. We need to create
10117 appropriate phi nodes in iter_part_bb instead. */
10118 se
= find_edge (iter_part_bb
, fin_bb
);
10119 re
= single_succ_edge (trip_update_bb
);
10120 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
10121 ene
= single_succ_edge (entry_bb
);
10123 psi
= gsi_start_phis (fin_bb
);
10124 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
10125 gsi_next (&psi
), ++i
)
10128 source_location locus
;
10131 t
= gimple_phi_result (phi
);
10132 gcc_assert (t
== redirect_edge_var_map_result (vm
));
10134 if (!single_pred_p (fin_bb
))
10135 t
= copy_ssa_name (t
, phi
);
10137 nphi
= create_phi_node (t
, iter_part_bb
);
10139 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
10140 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
10142 /* A special case -- fd->loop.v is not yet computed in
10143 iter_part_bb, we need to use vextra instead. */
10144 if (t
== fd
->loop
.v
)
10146 add_phi_arg (nphi
, t
, ene
, locus
);
10147 locus
= redirect_edge_var_map_location (vm
);
10148 tree back_arg
= redirect_edge_var_map_def (vm
);
10149 add_phi_arg (nphi
, back_arg
, re
, locus
);
10150 edge ce
= find_edge (cont_bb
, body_bb
);
10153 ce
= BRANCH_EDGE (cont_bb
);
10154 gcc_assert (single_succ (ce
->dest
) == body_bb
);
10155 ce
= single_succ_edge (ce
->dest
);
10157 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
10158 gcc_assert (inner_loop_phi
!= NULL
);
10159 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
10160 find_edge (seq_start_bb
, body_bb
), locus
);
10162 if (!single_pred_p (fin_bb
))
10163 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
10165 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
10166 redirect_edge_var_map_clear (re
);
10167 if (single_pred_p (fin_bb
))
10170 psi
= gsi_start_phis (fin_bb
);
10171 if (gsi_end_p (psi
))
10173 remove_phi_node (&psi
, false);
10176 /* Make phi node for trip. */
10177 phi
= create_phi_node (trip_main
, iter_part_bb
);
10178 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
10180 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
10185 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
10186 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
10187 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
10188 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10189 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10190 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
10191 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
10192 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10193 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10197 struct loop
*loop
= body_bb
->loop_father
;
10198 struct loop
*trip_loop
= alloc_loop ();
10199 trip_loop
->header
= iter_part_bb
;
10200 trip_loop
->latch
= trip_update_bb
;
10201 add_loop (trip_loop
, iter_part_bb
->loop_father
);
10203 if (loop
!= entry_bb
->loop_father
)
10205 gcc_assert (loop
->header
== body_bb
);
10206 gcc_assert (loop
->latch
== region
->cont
10207 || single_pred (loop
->latch
) == region
->cont
);
10208 trip_loop
->inner
= loop
;
10212 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10214 loop
= alloc_loop ();
10215 loop
->header
= body_bb
;
10216 if (collapse_bb
== NULL
)
10217 loop
->latch
= cont_bb
;
10218 add_loop (loop
, trip_loop
);
10223 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10225 for (V = N1; V cond N2; V += STEP) BODY;
10227 where COND is "<" or ">" or "!=", we generate pseudocode
10229 for (ind_var = low; ind_var < high; ind_var++)
10231 V = n1 + (ind_var * STEP)
10236 In the above pseudocode, low and high are function parameters of the
10237 child function. In the function below, we are inserting a temp.
10238 variable that will be making a call to two OMP functions that will not be
10239 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10240 with _Cilk_for). These functions are replaced with low and high
10241 by the function that handles taskreg. */
10245 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10247 bool broken_loop
= region
->cont
== NULL
;
10248 basic_block entry_bb
= region
->entry
;
10249 basic_block cont_bb
= region
->cont
;
10251 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10252 gcc_assert (broken_loop
10253 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10254 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10255 basic_block l1_bb
, l2_bb
;
10259 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10260 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10261 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10262 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10266 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10267 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10268 l2_bb
= single_succ (l1_bb
);
10270 basic_block exit_bb
= region
->exit
;
10271 basic_block l2_dom_bb
= NULL
;
10273 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10275 /* Below statements until the "tree high_val = ..." are pseudo statements
10276 used to pass information to be used by expand_omp_taskreg.
10277 low_val and high_val will be replaced by the __low and __high
10278 parameter from the child function.
10280 The call_exprs part is a place-holder, it is mainly used
10281 to distinctly identify to the top-level part that this is
10282 where we should put low and high (reasoning given in header
10286 = gimple_omp_parallel_child_fn (
10287 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10288 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10289 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10291 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10293 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10296 gcc_assert (low_val
&& high_val
);
10298 tree type
= TREE_TYPE (low_val
);
10299 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10300 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10302 /* Not needed in SSA form right now. */
10303 gcc_assert (!gimple_in_ssa_p (cfun
));
10304 if (l2_dom_bb
== NULL
)
10308 tree n2
= high_val
;
10310 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10312 /* Replace the GIMPLE_OMP_FOR statement. */
10313 gsi_replace (&gsi
, stmt
, true);
10317 /* Code to control the increment goes in the CONT_BB. */
10318 gsi
= gsi_last_bb (cont_bb
);
10319 stmt
= gsi_stmt (gsi
);
10320 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10321 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10322 build_one_cst (type
));
10324 /* Replace GIMPLE_OMP_CONTINUE. */
10325 gsi_replace (&gsi
, stmt
, true);
10328 /* Emit the condition in L1_BB. */
10329 gsi
= gsi_after_labels (l1_bb
);
10330 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10331 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10333 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10334 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10335 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10337 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10338 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10339 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10340 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10342 /* The condition is always '<' since the runtime will fill in the low
10343 and high values. */
10344 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10345 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10347 /* Remove GIMPLE_OMP_RETURN. */
10348 gsi
= gsi_last_bb (exit_bb
);
10349 gsi_remove (&gsi
, true);
10351 /* Connect the new blocks. */
10352 remove_edge (FALLTHRU_EDGE (entry_bb
));
10357 remove_edge (BRANCH_EDGE (entry_bb
));
10358 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10360 e
= BRANCH_EDGE (l1_bb
);
10361 ne
= FALLTHRU_EDGE (l1_bb
);
10362 e
->flags
= EDGE_TRUE_VALUE
;
10366 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10368 ne
= single_succ_edge (l1_bb
);
10369 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10372 ne
->flags
= EDGE_FALSE_VALUE
;
10373 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10374 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10376 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10377 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10378 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10382 struct loop
*loop
= alloc_loop ();
10383 loop
->header
= l1_bb
;
10384 loop
->latch
= cont_bb
;
10385 add_loop (loop
, l1_bb
->loop_father
);
10386 loop
->safelen
= INT_MAX
;
10389 /* Pick the correct library function based on the precision of the
10390 induction variable type. */
10391 tree lib_fun
= NULL_TREE
;
10392 if (TYPE_PRECISION (type
) == 32)
10393 lib_fun
= cilk_for_32_fndecl
;
10394 else if (TYPE_PRECISION (type
) == 64)
10395 lib_fun
= cilk_for_64_fndecl
;
10397 gcc_unreachable ();
10399 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10401 /* WS_ARGS contains the library function flavor to call:
10402 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10403 user-defined grain value. If the user does not define one, then zero
10404 is passed in by the parser. */
10405 vec_alloc (region
->ws_args
, 2);
10406 region
->ws_args
->quick_push (lib_fun
);
10407 region
->ws_args
->quick_push (fd
->chunk_size
);
10410 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10411 loop. Given parameters:
10413 for (V = N1; V cond N2; V += STEP) BODY;
10415 where COND is "<" or ">", we generate pseudocode
10423 if (V cond N2) goto L0; else goto L2;
10426 For collapsed loops, given parameters:
10428 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10429 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10430 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10433 we generate pseudocode
10439 count3 = (adj + N32 - N31) / STEP3;
10444 count2 = (adj + N22 - N21) / STEP2;
10449 count1 = (adj + N12 - N11) / STEP1;
10450 count = count1 * count2 * count3;
10460 V2 += (V3 cond3 N32) ? 0 : STEP2;
10461 V3 = (V3 cond3 N32) ? V3 : N31;
10462 V1 += (V2 cond2 N22) ? 0 : STEP1;
10463 V2 = (V2 cond2 N22) ? V2 : N21;
10465 if (V < count) goto L0; else goto L2;
10471 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10474 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10475 gimple_stmt_iterator gsi
;
10478 bool broken_loop
= region
->cont
== NULL
;
10480 tree
*counts
= NULL
;
10482 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10483 OMP_CLAUSE_SAFELEN
);
10484 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10485 OMP_CLAUSE__SIMDUID_
);
10488 type
= TREE_TYPE (fd
->loop
.v
);
10489 entry_bb
= region
->entry
;
10490 cont_bb
= region
->cont
;
10491 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10492 gcc_assert (broken_loop
10493 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10494 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10497 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10498 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10499 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10500 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10504 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10505 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10506 l2_bb
= single_succ (l1_bb
);
10508 exit_bb
= region
->exit
;
10511 gsi
= gsi_last_bb (entry_bb
);
10513 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10514 /* Not needed in SSA form right now. */
10515 gcc_assert (!gimple_in_ssa_p (cfun
));
10516 if (fd
->collapse
> 1)
10518 int first_zero_iter
= -1, dummy
= -1;
10519 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10521 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10522 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10523 zero_iter_bb
, first_zero_iter
,
10524 dummy_bb
, dummy
, l2_dom_bb
);
10526 if (l2_dom_bb
== NULL
)
10531 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10533 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10534 OMP_CLAUSE__LOOPTEMP_
);
10535 gcc_assert (innerc
);
10536 n1
= OMP_CLAUSE_DECL (innerc
);
10537 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10538 OMP_CLAUSE__LOOPTEMP_
);
10539 gcc_assert (innerc
);
10540 n2
= OMP_CLAUSE_DECL (innerc
);
10541 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10542 fold_convert (type
, n1
));
10543 if (fd
->collapse
> 1)
10546 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10552 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10553 fold_convert (type
, fd
->loop
.n1
));
10554 if (fd
->collapse
> 1)
10555 for (i
= 0; i
< fd
->collapse
; i
++)
10557 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10558 if (POINTER_TYPE_P (itype
))
10559 itype
= signed_type_for (itype
);
10560 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10561 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10565 /* Remove the GIMPLE_OMP_FOR statement. */
10566 gsi_remove (&gsi
, true);
10570 /* Code to control the increment goes in the CONT_BB. */
10571 gsi
= gsi_last_bb (cont_bb
);
10572 stmt
= gsi_stmt (gsi
);
10573 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10575 if (POINTER_TYPE_P (type
))
10576 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10578 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10579 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10581 if (fd
->collapse
> 1)
10583 i
= fd
->collapse
- 1;
10584 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10586 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10587 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10591 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10592 fd
->loops
[i
].step
);
10593 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10594 fd
->loops
[i
].v
, t
);
10596 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10598 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10600 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10601 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10602 if (POINTER_TYPE_P (itype2
))
10603 itype2
= signed_type_for (itype2
);
10604 t
= build3 (COND_EXPR
, itype2
,
10605 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10607 fold_convert (itype
, fd
->loops
[i
].n2
)),
10608 build_int_cst (itype2
, 0),
10609 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10610 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10611 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10613 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10614 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10616 t
= build3 (COND_EXPR
, itype
,
10617 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10619 fold_convert (itype
, fd
->loops
[i
].n2
)),
10621 fold_convert (itype
, fd
->loops
[i
].n1
));
10622 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10626 /* Remove GIMPLE_OMP_CONTINUE. */
10627 gsi_remove (&gsi
, true);
10630 /* Emit the condition in L1_BB. */
10631 gsi
= gsi_start_bb (l1_bb
);
10633 t
= fold_convert (type
, n2
);
10634 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10635 false, GSI_CONTINUE_LINKING
);
10636 tree v
= fd
->loop
.v
;
10637 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10638 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10639 false, GSI_CONTINUE_LINKING
);
10640 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10641 cond_stmt
= gimple_build_cond_empty (t
);
10642 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10643 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10645 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10648 gsi
= gsi_for_stmt (cond_stmt
);
10649 gimple_regimplify_operands (cond_stmt
, &gsi
);
10652 /* Remove GIMPLE_OMP_RETURN. */
10653 gsi
= gsi_last_bb (exit_bb
);
10654 gsi_remove (&gsi
, true);
10656 /* Connect the new blocks. */
10657 remove_edge (FALLTHRU_EDGE (entry_bb
));
10661 remove_edge (BRANCH_EDGE (entry_bb
));
10662 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10664 e
= BRANCH_EDGE (l1_bb
);
10665 ne
= FALLTHRU_EDGE (l1_bb
);
10666 e
->flags
= EDGE_TRUE_VALUE
;
10670 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10672 ne
= single_succ_edge (l1_bb
);
10673 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10676 ne
->flags
= EDGE_FALSE_VALUE
;
10677 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10678 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10680 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10681 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10682 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10686 struct loop
*loop
= alloc_loop ();
10687 loop
->header
= l1_bb
;
10688 loop
->latch
= cont_bb
;
10689 add_loop (loop
, l1_bb
->loop_father
);
10690 if (safelen
== NULL_TREE
)
10691 loop
->safelen
= INT_MAX
;
10694 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10695 if (TREE_CODE (safelen
) != INTEGER_CST
)
10697 else if (!tree_fits_uhwi_p (safelen
)
10698 || tree_to_uhwi (safelen
) > INT_MAX
)
10699 loop
->safelen
= INT_MAX
;
10701 loop
->safelen
= tree_to_uhwi (safelen
);
10702 if (loop
->safelen
== 1)
10707 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10708 cfun
->has_simduid_loops
= true;
10710 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10712 if ((flag_tree_loop_vectorize
10713 || (!global_options_set
.x_flag_tree_loop_vectorize
10714 && !global_options_set
.x_flag_tree_vectorize
))
10715 && flag_tree_loop_optimize
10716 && loop
->safelen
> 1)
10718 loop
->force_vectorize
= true;
10719 cfun
->has_force_vectorize_loops
= true;
10723 cfun
->has_simduid_loops
= true;
10726 /* Taskloop construct is represented after gimplification with
10727 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10728 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10729 which should just compute all the needed loop temporaries
10730 for GIMPLE_OMP_TASK. */
10733 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10734 struct omp_for_data
*fd
,
10735 gimple
*inner_stmt
)
10737 tree type
, bias
= NULL_TREE
;
10738 basic_block entry_bb
, cont_bb
, exit_bb
;
10739 gimple_stmt_iterator gsi
;
10740 gassign
*assign_stmt
;
10741 tree
*counts
= NULL
;
10744 gcc_assert (inner_stmt
);
10745 gcc_assert (region
->cont
);
10746 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10747 && gimple_omp_task_taskloop_p (inner_stmt
));
10748 type
= TREE_TYPE (fd
->loop
.v
);
10750 /* See if we need to bias by LLONG_MIN. */
10751 if (fd
->iter_type
== long_long_unsigned_type_node
10752 && TREE_CODE (type
) == INTEGER_TYPE
10753 && !TYPE_UNSIGNED (type
))
10757 if (fd
->loop
.cond_code
== LT_EXPR
)
10760 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10764 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10767 if (TREE_CODE (n1
) != INTEGER_CST
10768 || TREE_CODE (n2
) != INTEGER_CST
10769 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10770 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10773 entry_bb
= region
->entry
;
10774 cont_bb
= region
->cont
;
10775 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10776 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10777 exit_bb
= region
->exit
;
10779 gsi
= gsi_last_bb (entry_bb
);
10780 gimple
*for_stmt
= gsi_stmt (gsi
);
10781 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10782 if (fd
->collapse
> 1)
10784 int first_zero_iter
= -1, dummy
= -1;
10785 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10787 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10788 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10789 zero_iter_bb
, first_zero_iter
,
10790 dummy_bb
, dummy
, l2_dom_bb
);
10794 /* Some counts[i] vars might be uninitialized if
10795 some loop has zero iterations. But the body shouldn't
10796 be executed in that case, so just avoid uninit warnings. */
10797 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10798 if (SSA_VAR_P (counts
[i
]))
10799 TREE_NO_WARNING (counts
[i
]) = 1;
10801 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10802 entry_bb
= e
->dest
;
10803 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10804 gsi
= gsi_last_bb (entry_bb
);
10805 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10806 get_immediate_dominator (CDI_DOMINATORS
,
10814 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10815 && TYPE_PRECISION (TREE_TYPE (t0
))
10816 != TYPE_PRECISION (fd
->iter_type
))
10818 /* Avoid casting pointers to integer of a different size. */
10819 tree itype
= signed_type_for (type
);
10820 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10821 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10825 t1
= fold_convert (fd
->iter_type
, t1
);
10826 t0
= fold_convert (fd
->iter_type
, t0
);
10830 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10831 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10834 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10835 OMP_CLAUSE__LOOPTEMP_
);
10836 gcc_assert (innerc
);
10837 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10838 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10839 gcc_assert (innerc
);
10840 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10841 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10843 gcc_assert (innerc
);
10844 for (i
= 1; i
< fd
->collapse
; i
++)
10846 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10847 OMP_CLAUSE__LOOPTEMP_
);
10848 gcc_assert (innerc
);
10850 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10851 OMP_CLAUSE__LOOPTEMP_
);
10854 /* If needed (inner taskloop has lastprivate clause), propagate
10855 down the total number of iterations. */
10856 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10858 GSI_CONTINUE_LINKING
);
10859 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10860 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10864 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10865 GSI_CONTINUE_LINKING
);
10866 assign_stmt
= gimple_build_assign (startvar
, t0
);
10867 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10869 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10870 GSI_CONTINUE_LINKING
);
10871 assign_stmt
= gimple_build_assign (endvar
, t1
);
10872 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10873 if (fd
->collapse
> 1)
10874 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10876 /* Remove the GIMPLE_OMP_FOR statement. */
10877 gsi
= gsi_for_stmt (for_stmt
);
10878 gsi_remove (&gsi
, true);
10880 gsi
= gsi_last_bb (cont_bb
);
10881 gsi_remove (&gsi
, true);
10883 gsi
= gsi_last_bb (exit_bb
);
10884 gsi_remove (&gsi
, true);
10886 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10887 remove_edge (BRANCH_EDGE (entry_bb
));
10888 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10889 remove_edge (BRANCH_EDGE (cont_bb
));
10890 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10891 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10892 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10895 /* Taskloop construct is represented after gimplification with
10896 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10897 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10898 GOMP_taskloop{,_ull} function arranges for each task to be given just
10899 a single range of iterations. */
10902 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10903 struct omp_for_data
*fd
,
10904 gimple
*inner_stmt
)
10906 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10907 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10908 basic_block fin_bb
;
10909 gimple_stmt_iterator gsi
;
10911 bool broken_loop
= region
->cont
== NULL
;
10912 tree
*counts
= NULL
;
10915 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10916 if (POINTER_TYPE_P (type
))
10917 itype
= signed_type_for (type
);
10919 /* See if we need to bias by LLONG_MIN. */
10920 if (fd
->iter_type
== long_long_unsigned_type_node
10921 && TREE_CODE (type
) == INTEGER_TYPE
10922 && !TYPE_UNSIGNED (type
))
10926 if (fd
->loop
.cond_code
== LT_EXPR
)
10929 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10933 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10936 if (TREE_CODE (n1
) != INTEGER_CST
10937 || TREE_CODE (n2
) != INTEGER_CST
10938 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10939 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10942 entry_bb
= region
->entry
;
10943 cont_bb
= region
->cont
;
10944 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10945 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10946 gcc_assert (broken_loop
10947 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10948 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10951 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10952 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10954 exit_bb
= region
->exit
;
10956 /* Iteration space partitioning goes in ENTRY_BB. */
10957 gsi
= gsi_last_bb (entry_bb
);
10958 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10960 if (fd
->collapse
> 1)
10962 int first_zero_iter
= -1, dummy
= -1;
10963 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10965 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10966 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10967 fin_bb
, first_zero_iter
,
10968 dummy_bb
, dummy
, l2_dom_bb
);
10972 t
= integer_one_node
;
10974 step
= fd
->loop
.step
;
10975 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10976 OMP_CLAUSE__LOOPTEMP_
);
10977 gcc_assert (innerc
);
10978 n1
= OMP_CLAUSE_DECL (innerc
);
10979 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10980 gcc_assert (innerc
);
10981 n2
= OMP_CLAUSE_DECL (innerc
);
10984 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10985 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10987 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10988 true, NULL_TREE
, true, GSI_SAME_STMT
);
10989 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10990 true, NULL_TREE
, true, GSI_SAME_STMT
);
10991 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10992 true, NULL_TREE
, true, GSI_SAME_STMT
);
10994 tree startvar
= fd
->loop
.v
;
10995 tree endvar
= NULL_TREE
;
10997 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10999 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
11000 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
11001 gcc_assert (innerc
);
11002 startvar
= OMP_CLAUSE_DECL (innerc
);
11003 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
11004 OMP_CLAUSE__LOOPTEMP_
);
11005 gcc_assert (innerc
);
11006 endvar
= OMP_CLAUSE_DECL (innerc
);
11008 t
= fold_convert (TREE_TYPE (startvar
), n1
);
11009 t
= force_gimple_operand_gsi (&gsi
, t
,
11011 && TREE_ADDRESSABLE (startvar
),
11012 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
11013 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
11014 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11016 t
= fold_convert (TREE_TYPE (startvar
), n2
);
11017 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
11018 false, GSI_CONTINUE_LINKING
);
11021 assign_stmt
= gimple_build_assign (endvar
, e
);
11022 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11023 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
11024 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
11026 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
11027 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11029 if (fd
->collapse
> 1)
11030 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
11034 /* The code controlling the sequential loop replaces the
11035 GIMPLE_OMP_CONTINUE. */
11036 gsi
= gsi_last_bb (cont_bb
);
11037 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11038 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
11039 vmain
= gimple_omp_continue_control_use (cont_stmt
);
11040 vback
= gimple_omp_continue_control_def (cont_stmt
);
11042 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
11044 if (POINTER_TYPE_P (type
))
11045 t
= fold_build_pointer_plus (vmain
, step
);
11047 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
11048 t
= force_gimple_operand_gsi (&gsi
, t
,
11050 && TREE_ADDRESSABLE (vback
),
11051 NULL_TREE
, true, GSI_SAME_STMT
);
11052 assign_stmt
= gimple_build_assign (vback
, t
);
11053 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
11055 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
11056 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
11058 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
11061 /* Remove the GIMPLE_OMP_CONTINUE statement. */
11062 gsi_remove (&gsi
, true);
11064 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
11065 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
11068 /* Remove the GIMPLE_OMP_FOR statement. */
11069 gsi
= gsi_for_stmt (fd
->for_stmt
);
11070 gsi_remove (&gsi
, true);
11072 /* Remove the GIMPLE_OMP_RETURN statement. */
11073 gsi
= gsi_last_bb (exit_bb
);
11074 gsi_remove (&gsi
, true);
11076 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
11078 remove_edge (BRANCH_EDGE (entry_bb
));
11081 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
11082 region
->outer
->cont
= NULL
;
11085 /* Connect all the blocks. */
11088 ep
= find_edge (cont_bb
, body_bb
);
11089 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11094 else if (fd
->collapse
> 1)
11097 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
11100 ep
->flags
= EDGE_TRUE_VALUE
;
11101 find_edge (cont_bb
, fin_bb
)->flags
11102 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
11105 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
11106 recompute_dominator (CDI_DOMINATORS
, body_bb
));
11108 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
11109 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
11111 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
11113 struct loop
*loop
= alloc_loop ();
11114 loop
->header
= body_bb
;
11115 if (collapse_bb
== NULL
)
11116 loop
->latch
= cont_bb
;
11117 add_loop (loop
, body_bb
->loop_father
);
11121 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11122 partitioned loop. The lowering here is abstracted, in that the
11123 loop parameters are passed through internal functions, which are
11124 further lowered by oacc_device_lower, once we get to the target
11125 compiler. The loop is of the form:
11127 for (V = B; V LTGT E; V += S) {BODY}
11129 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11130 (constant 0 for no chunking) and we will have a GWV partitioning
11131 mask, specifying dimensions over which the loop is to be
11132 partitioned (see note below). We generate code that looks like:
11134 <entry_bb> [incoming FALL->body, BRANCH->exit]
11135 typedef signedintify (typeof (V)) T; // underlying signed integral type
11138 T DIR = LTGT == '<' ? +1 : -1;
11139 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11140 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11142 <head_bb> [created by splitting end of entry_bb]
11143 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11144 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11145 if (!(offset LTGT bound)) goto bottom_bb;
11147 <body_bb> [incoming]
11151 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11153 if (offset LTGT bound) goto body_bb; [*]
11155 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11157 if (chunk < chunk_max) goto head_bb;
11159 <exit_bb> [incoming]
11160 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11162 [*] Needed if V live at end of loop
11164 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11165 transition, and will be specified by a more general mechanism shortly.
11169 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
11171 tree v
= fd
->loop
.v
;
11172 enum tree_code cond_code
= fd
->loop
.cond_code
;
11173 enum tree_code plus_code
= PLUS_EXPR
;
11175 tree chunk_size
= integer_minus_one_node
;
11176 tree gwv
= integer_zero_node
;
11177 tree iter_type
= TREE_TYPE (v
);
11178 tree diff_type
= iter_type
;
11179 tree plus_type
= iter_type
;
11180 struct oacc_collapse
*counts
= NULL
;
11182 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
11183 == GF_OMP_FOR_KIND_OACC_LOOP
);
11184 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
11185 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
11187 if (POINTER_TYPE_P (iter_type
))
11189 plus_code
= POINTER_PLUS_EXPR
;
11190 plus_type
= sizetype
;
11192 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
11193 diff_type
= signed_type_for (diff_type
);
11195 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
11196 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
11197 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
11198 basic_block bottom_bb
= NULL
;
11200 /* entry_bb has two sucessors; the branch edge is to the exit
11201 block, fallthrough edge to body. */
11202 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
11203 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
11205 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11206 body_bb, or to a block whose only successor is the body_bb. Its
11207 fallthrough successor is the final block (same as the branch
11208 successor of the entry_bb). */
11211 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
11212 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
11214 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
11215 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
11218 gcc_assert (!gimple_in_ssa_p (cfun
));
11220 /* The exit block only has entry_bb and cont_bb as predecessors. */
11221 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
11224 tree chunk_max
= NULL_TREE
;
11225 tree bound
, offset
;
11226 tree step
= create_tmp_var (diff_type
, ".step");
11227 bool up
= cond_code
== LT_EXPR
;
11228 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
11229 bool chunking
= !gimple_in_ssa_p (cfun
);;
11232 /* SSA instances. */
11233 tree offset_incr
= NULL_TREE
;
11234 tree offset_init
= NULL_TREE
;
11236 gimple_stmt_iterator gsi
;
11242 edge split
, be
, fte
;
11244 /* Split the end of entry_bb to create head_bb. */
11245 split
= split_block (entry_bb
, last_stmt (entry_bb
));
11246 basic_block head_bb
= split
->dest
;
11247 entry_bb
= split
->src
;
11249 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11250 gsi
= gsi_last_bb (entry_bb
);
11251 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
11252 loc
= gimple_location (for_stmt
);
11254 if (gimple_in_ssa_p (cfun
))
11256 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11257 gcc_assert (integer_zerop (fd
->loop
.n1
));
11258 /* The SSA parallelizer does gang parallelism. */
11259 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11262 if (fd
->collapse
> 1)
11264 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11265 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11266 TREE_TYPE (fd
->loop
.n2
));
11268 if (SSA_VAR_P (fd
->loop
.n2
))
11270 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11271 true, GSI_SAME_STMT
);
11272 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11273 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11278 tree b
= fd
->loop
.n1
;
11279 tree e
= fd
->loop
.n2
;
11280 tree s
= fd
->loop
.step
;
11282 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11283 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11285 /* Convert the step, avoiding possible unsigned->signed overflow. */
11286 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11288 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11289 s
= fold_convert (diff_type
, s
);
11291 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11292 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11295 chunk_size
= integer_zero_node
;
11296 expr
= fold_convert (diff_type
, chunk_size
);
11297 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11298 NULL_TREE
, true, GSI_SAME_STMT
);
11299 /* Determine the range, avoiding possible unsigned->signed overflow. */
11300 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11301 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11302 fold_convert (plus_type
, negating
? b
: e
),
11303 fold_convert (plus_type
, negating
? e
: b
));
11304 expr
= fold_convert (diff_type
, expr
);
11306 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11307 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11308 NULL_TREE
, true, GSI_SAME_STMT
);
11310 chunk_no
= build_int_cst (diff_type
, 0);
11313 gcc_assert (!gimple_in_ssa_p (cfun
));
11316 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11317 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11319 ass
= gimple_build_assign (chunk_no
, expr
);
11320 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11322 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11323 build_int_cst (integer_type_node
,
11324 IFN_GOACC_LOOP_CHUNKS
),
11325 dir
, range
, s
, chunk_size
, gwv
);
11326 gimple_call_set_lhs (call
, chunk_max
);
11327 gimple_set_location (call
, loc
);
11328 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11331 chunk_size
= chunk_no
;
11333 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11334 build_int_cst (integer_type_node
,
11335 IFN_GOACC_LOOP_STEP
),
11336 dir
, range
, s
, chunk_size
, gwv
);
11337 gimple_call_set_lhs (call
, step
);
11338 gimple_set_location (call
, loc
);
11339 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11341 /* Remove the GIMPLE_OMP_FOR. */
11342 gsi_remove (&gsi
, true);
11344 /* Fixup edges from head_bb */
11345 be
= BRANCH_EDGE (head_bb
);
11346 fte
= FALLTHRU_EDGE (head_bb
);
11347 be
->flags
|= EDGE_FALSE_VALUE
;
11348 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11350 basic_block body_bb
= fte
->dest
;
11352 if (gimple_in_ssa_p (cfun
))
11354 gsi
= gsi_last_bb (cont_bb
);
11355 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11357 offset
= gimple_omp_continue_control_use (cont_stmt
);
11358 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11362 offset
= create_tmp_var (diff_type
, ".offset");
11363 offset_init
= offset_incr
= offset
;
11365 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11367 /* Loop offset & bound go into head_bb. */
11368 gsi
= gsi_start_bb (head_bb
);
11370 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11371 build_int_cst (integer_type_node
,
11372 IFN_GOACC_LOOP_OFFSET
),
11374 chunk_size
, gwv
, chunk_no
);
11375 gimple_call_set_lhs (call
, offset_init
);
11376 gimple_set_location (call
, loc
);
11377 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11379 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11380 build_int_cst (integer_type_node
,
11381 IFN_GOACC_LOOP_BOUND
),
11383 chunk_size
, gwv
, offset_init
);
11384 gimple_call_set_lhs (call
, bound
);
11385 gimple_set_location (call
, loc
);
11386 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11388 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11389 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11390 GSI_CONTINUE_LINKING
);
11392 /* V assignment goes into body_bb. */
11393 if (!gimple_in_ssa_p (cfun
))
11395 gsi
= gsi_start_bb (body_bb
);
11397 expr
= build2 (plus_code
, iter_type
, b
,
11398 fold_convert (plus_type
, offset
));
11399 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11400 true, GSI_SAME_STMT
);
11401 ass
= gimple_build_assign (v
, expr
);
11402 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11403 if (fd
->collapse
> 1)
11404 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11407 /* Loop increment goes into cont_bb. If this is not a loop, we
11408 will have spawned threads as if it was, and each one will
11409 execute one iteration. The specification is not explicit about
11410 whether such constructs are ill-formed or not, and they can
11411 occur, especially when noreturn routines are involved. */
11414 gsi
= gsi_last_bb (cont_bb
);
11415 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11416 loc
= gimple_location (cont_stmt
);
11418 /* Increment offset. */
11419 if (gimple_in_ssa_p (cfun
))
11420 expr
= build2 (plus_code
, iter_type
, offset
,
11421 fold_convert (plus_type
, step
));
11423 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11424 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11425 true, GSI_SAME_STMT
);
11426 ass
= gimple_build_assign (offset_incr
, expr
);
11427 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11428 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11429 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11431 /* Remove the GIMPLE_OMP_CONTINUE. */
11432 gsi_remove (&gsi
, true);
11434 /* Fixup edges from cont_bb */
11435 be
= BRANCH_EDGE (cont_bb
);
11436 fte
= FALLTHRU_EDGE (cont_bb
);
11437 be
->flags
|= EDGE_TRUE_VALUE
;
11438 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11442 /* Split the beginning of exit_bb to make bottom_bb. We
11443 need to insert a nop at the start, because splitting is
11444 after a stmt, not before. */
11445 gsi
= gsi_start_bb (exit_bb
);
11446 stmt
= gimple_build_nop ();
11447 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11448 split
= split_block (exit_bb
, stmt
);
11449 bottom_bb
= split
->src
;
11450 exit_bb
= split
->dest
;
11451 gsi
= gsi_last_bb (bottom_bb
);
11453 /* Chunk increment and test goes into bottom_bb. */
11454 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11455 build_int_cst (diff_type
, 1));
11456 ass
= gimple_build_assign (chunk_no
, expr
);
11457 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11459 /* Chunk test at end of bottom_bb. */
11460 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11461 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11462 GSI_CONTINUE_LINKING
);
11464 /* Fixup edges from bottom_bb. */
11465 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11466 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11470 gsi
= gsi_last_bb (exit_bb
);
11471 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11472 loc
= gimple_location (gsi_stmt (gsi
));
11474 if (!gimple_in_ssa_p (cfun
))
11476 /* Insert the final value of V, in case it is live. This is the
11477 value for the only thread that survives past the join. */
11478 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11479 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11480 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11481 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11482 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11483 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11484 true, GSI_SAME_STMT
);
11485 ass
= gimple_build_assign (v
, expr
);
11486 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11489 /* Remove the OMP_RETURN. */
11490 gsi_remove (&gsi
, true);
11494 /* We now have one or two nested loops. Update the loop
11496 struct loop
*parent
= entry_bb
->loop_father
;
11497 struct loop
*body
= body_bb
->loop_father
;
11501 struct loop
*chunk_loop
= alloc_loop ();
11502 chunk_loop
->header
= head_bb
;
11503 chunk_loop
->latch
= bottom_bb
;
11504 add_loop (chunk_loop
, parent
);
11505 parent
= chunk_loop
;
11507 else if (parent
!= body
)
11509 gcc_assert (body
->header
== body_bb
);
11510 gcc_assert (body
->latch
== cont_bb
11511 || single_pred (body
->latch
) == cont_bb
);
11517 struct loop
*body_loop
= alloc_loop ();
11518 body_loop
->header
= body_bb
;
11519 body_loop
->latch
= cont_bb
;
11520 add_loop (body_loop
, parent
);
11525 /* Expand the OMP loop defined by REGION. */
11528 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11530 struct omp_for_data fd
;
11531 struct omp_for_data_loop
*loops
;
11534 = (struct omp_for_data_loop
*)
11535 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11536 * sizeof (struct omp_for_data_loop
));
11537 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11539 region
->sched_kind
= fd
.sched_kind
;
11540 region
->sched_modifiers
= fd
.sched_modifiers
;
11542 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11543 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11544 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11547 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11548 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11549 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11552 /* If there isn't a continue then this is a degerate case where
11553 the introduction of abnormal edges during lowering will prevent
11554 original loops from being detected. Fix that up. */
11555 loops_state_set (LOOPS_NEED_FIXUP
);
11557 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11558 expand_omp_simd (region
, &fd
);
11559 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11560 expand_cilk_for (region
, &fd
);
11561 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11563 gcc_assert (!inner_stmt
);
11564 expand_oacc_for (region
, &fd
);
11566 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11568 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11569 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11571 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11573 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11574 && !fd
.have_ordered
)
11576 if (fd
.chunk_size
== NULL
)
11577 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11579 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11583 int fn_index
, start_ix
, next_ix
;
11585 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11586 == GF_OMP_FOR_KIND_FOR
);
11587 if (fd
.chunk_size
== NULL
11588 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11589 fd
.chunk_size
= integer_zero_node
;
11590 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11591 switch (fd
.sched_kind
)
11593 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11596 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11597 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11598 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11600 && !fd
.have_ordered
)
11602 fn_index
= 3 + fd
.sched_kind
;
11607 fn_index
= fd
.sched_kind
;
11611 fn_index
+= fd
.have_ordered
* 6;
11613 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11615 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11616 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11617 if (fd
.iter_type
== long_long_unsigned_type_node
)
11619 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11620 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11621 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11622 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11624 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11625 (enum built_in_function
) next_ix
, inner_stmt
);
11628 if (gimple_in_ssa_p (cfun
))
11629 update_ssa (TODO_update_ssa_only_virtuals
);
11633 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11635 v = GOMP_sections_start (n);
11652 v = GOMP_sections_next ();
11657 If this is a combined parallel sections, replace the call to
11658 GOMP_sections_start with call to GOMP_sections_next. */
11661 expand_omp_sections (struct omp_region
*region
)
11663 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11665 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11666 gimple_stmt_iterator si
, switch_si
;
11667 gomp_sections
*sections_stmt
;
11669 gomp_continue
*cont
;
11672 struct omp_region
*inner
;
11674 bool exit_reachable
= region
->cont
!= NULL
;
11676 gcc_assert (region
->exit
!= NULL
);
11677 entry_bb
= region
->entry
;
11678 l0_bb
= single_succ (entry_bb
);
11679 l1_bb
= region
->cont
;
11680 l2_bb
= region
->exit
;
11681 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11682 l2
= gimple_block_label (l2_bb
);
11685 /* This can happen if there are reductions. */
11686 len
= EDGE_COUNT (l0_bb
->succs
);
11687 gcc_assert (len
> 0);
11688 e
= EDGE_SUCC (l0_bb
, len
- 1);
11689 si
= gsi_last_bb (e
->dest
);
11692 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11693 l2
= gimple_block_label (e
->dest
);
11695 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11697 si
= gsi_last_bb (e
->dest
);
11699 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11701 l2
= gimple_block_label (e
->dest
);
11706 if (exit_reachable
)
11707 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11709 default_bb
= create_empty_bb (l0_bb
);
11711 /* We will build a switch() with enough cases for all the
11712 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11713 and a default case to abort if something goes wrong. */
11714 len
= EDGE_COUNT (l0_bb
->succs
);
11716 /* Use vec::quick_push on label_vec throughout, since we know the size
11718 auto_vec
<tree
> label_vec (len
);
11720 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11721 GIMPLE_OMP_SECTIONS statement. */
11722 si
= gsi_last_bb (entry_bb
);
11723 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11724 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11725 vin
= gimple_omp_sections_control (sections_stmt
);
11726 if (!is_combined_parallel (region
))
11728 /* If we are not inside a combined parallel+sections region,
11729 call GOMP_sections_start. */
11730 t
= build_int_cst (unsigned_type_node
, len
- 1);
11731 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11732 stmt
= gimple_build_call (u
, 1, t
);
11736 /* Otherwise, call GOMP_sections_next. */
11737 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11738 stmt
= gimple_build_call (u
, 0);
11740 gimple_call_set_lhs (stmt
, vin
);
11741 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11742 gsi_remove (&si
, true);
11744 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11746 switch_si
= gsi_last_bb (l0_bb
);
11747 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11748 if (exit_reachable
)
11750 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11751 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11752 vmain
= gimple_omp_continue_control_use (cont
);
11753 vnext
= gimple_omp_continue_control_def (cont
);
11761 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11762 label_vec
.quick_push (t
);
11765 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11766 for (inner
= region
->inner
, casei
= 1;
11768 inner
= inner
->next
, i
++, casei
++)
11770 basic_block s_entry_bb
, s_exit_bb
;
11772 /* Skip optional reduction region. */
11773 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11780 s_entry_bb
= inner
->entry
;
11781 s_exit_bb
= inner
->exit
;
11783 t
= gimple_block_label (s_entry_bb
);
11784 u
= build_int_cst (unsigned_type_node
, casei
);
11785 u
= build_case_label (u
, NULL
, t
);
11786 label_vec
.quick_push (u
);
11788 si
= gsi_last_bb (s_entry_bb
);
11789 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11790 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11791 gsi_remove (&si
, true);
11792 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11794 if (s_exit_bb
== NULL
)
11797 si
= gsi_last_bb (s_exit_bb
);
11798 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11799 gsi_remove (&si
, true);
11801 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11804 /* Error handling code goes in DEFAULT_BB. */
11805 t
= gimple_block_label (default_bb
);
11806 u
= build_case_label (NULL
, NULL
, t
);
11807 make_edge (l0_bb
, default_bb
, 0);
11808 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11810 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11811 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11812 gsi_remove (&switch_si
, true);
11814 si
= gsi_start_bb (default_bb
);
11815 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11816 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11818 if (exit_reachable
)
11822 /* Code to get the next section goes in L1_BB. */
11823 si
= gsi_last_bb (l1_bb
);
11824 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11826 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11827 stmt
= gimple_build_call (bfn_decl
, 0);
11828 gimple_call_set_lhs (stmt
, vnext
);
11829 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11830 gsi_remove (&si
, true);
11832 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11835 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11836 si
= gsi_last_bb (l2_bb
);
11837 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11838 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11839 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11840 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11842 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11843 stmt
= gimple_build_call (t
, 0);
11844 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11845 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11846 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11847 gsi_remove (&si
, true);
11849 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11853 /* Expand code for an OpenMP single directive. We've already expanded
11854 much of the code, here we simply place the GOMP_barrier call. */
11857 expand_omp_single (struct omp_region
*region
)
11859 basic_block entry_bb
, exit_bb
;
11860 gimple_stmt_iterator si
;
11862 entry_bb
= region
->entry
;
11863 exit_bb
= region
->exit
;
11865 si
= gsi_last_bb (entry_bb
);
11866 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11867 gsi_remove (&si
, true);
11868 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11870 si
= gsi_last_bb (exit_bb
);
11871 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11873 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11874 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11876 gsi_remove (&si
, true);
11877 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11881 /* Generic expansion for OpenMP synchronization directives: master,
11882 ordered and critical. All we need to do here is remove the entry
11883 and exit markers for REGION. */
11886 expand_omp_synch (struct omp_region
*region
)
11888 basic_block entry_bb
, exit_bb
;
11889 gimple_stmt_iterator si
;
11891 entry_bb
= region
->entry
;
11892 exit_bb
= region
->exit
;
11894 si
= gsi_last_bb (entry_bb
);
11895 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11896 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11897 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11898 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11899 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11900 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11901 gsi_remove (&si
, true);
11902 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11906 si
= gsi_last_bb (exit_bb
);
11907 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11908 gsi_remove (&si
, true);
11909 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11913 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11914 operation as a normal volatile load. */
11917 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11918 tree loaded_val
, int index
)
11920 enum built_in_function tmpbase
;
11921 gimple_stmt_iterator gsi
;
11922 basic_block store_bb
;
11925 tree decl
, call
, type
, itype
;
11927 gsi
= gsi_last_bb (load_bb
);
11928 stmt
= gsi_stmt (gsi
);
11929 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11930 loc
= gimple_location (stmt
);
11932 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11933 is smaller than word size, then expand_atomic_load assumes that the load
11934 is atomic. We could avoid the builtin entirely in this case. */
11936 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11937 decl
= builtin_decl_explicit (tmpbase
);
11938 if (decl
== NULL_TREE
)
11941 type
= TREE_TYPE (loaded_val
);
11942 itype
= TREE_TYPE (TREE_TYPE (decl
));
11944 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11945 build_int_cst (NULL
,
11946 gimple_omp_atomic_seq_cst_p (stmt
)
11948 : MEMMODEL_RELAXED
));
11949 if (!useless_type_conversion_p (type
, itype
))
11950 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11951 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11953 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11954 gsi_remove (&gsi
, true);
11956 store_bb
= single_succ (load_bb
);
11957 gsi
= gsi_last_bb (store_bb
);
11958 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11959 gsi_remove (&gsi
, true);
11961 if (gimple_in_ssa_p (cfun
))
11962 update_ssa (TODO_update_ssa_no_phi
);
11967 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11968 operation as a normal volatile store. */
11971 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11972 tree loaded_val
, tree stored_val
, int index
)
11974 enum built_in_function tmpbase
;
11975 gimple_stmt_iterator gsi
;
11976 basic_block store_bb
= single_succ (load_bb
);
11979 tree decl
, call
, type
, itype
;
11980 machine_mode imode
;
11983 gsi
= gsi_last_bb (load_bb
);
11984 stmt
= gsi_stmt (gsi
);
11985 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11987 /* If the load value is needed, then this isn't a store but an exchange. */
11988 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11990 gsi
= gsi_last_bb (store_bb
);
11991 stmt
= gsi_stmt (gsi
);
11992 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11993 loc
= gimple_location (stmt
);
11995 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11996 is smaller than word size, then expand_atomic_store assumes that the store
11997 is atomic. We could avoid the builtin entirely in this case. */
11999 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
12000 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
12001 decl
= builtin_decl_explicit (tmpbase
);
12002 if (decl
== NULL_TREE
)
12005 type
= TREE_TYPE (stored_val
);
12007 /* Dig out the type of the function's second argument. */
12008 itype
= TREE_TYPE (decl
);
12009 itype
= TYPE_ARG_TYPES (itype
);
12010 itype
= TREE_CHAIN (itype
);
12011 itype
= TREE_VALUE (itype
);
12012 imode
= TYPE_MODE (itype
);
12014 if (exchange
&& !can_atomic_exchange_p (imode
, true))
12017 if (!useless_type_conversion_p (itype
, type
))
12018 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
12019 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
12020 build_int_cst (NULL
,
12021 gimple_omp_atomic_seq_cst_p (stmt
)
12023 : MEMMODEL_RELAXED
));
12026 if (!useless_type_conversion_p (type
, itype
))
12027 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
12028 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
12031 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12032 gsi_remove (&gsi
, true);
12034 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
12035 gsi
= gsi_last_bb (load_bb
);
12036 gsi_remove (&gsi
, true);
12038 if (gimple_in_ssa_p (cfun
))
12039 update_ssa (TODO_update_ssa_no_phi
);
12044 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
12045 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
12046 size of the data type, and thus usable to find the index of the builtin
12047 decl. Returns false if the expression is not of the proper form. */
12050 expand_omp_atomic_fetch_op (basic_block load_bb
,
12051 tree addr
, tree loaded_val
,
12052 tree stored_val
, int index
)
12054 enum built_in_function oldbase
, newbase
, tmpbase
;
12055 tree decl
, itype
, call
;
12057 basic_block store_bb
= single_succ (load_bb
);
12058 gimple_stmt_iterator gsi
;
12061 enum tree_code code
;
12062 bool need_old
, need_new
;
12063 machine_mode imode
;
12066 /* We expect to find the following sequences:
12069 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12072 val = tmp OP something; (or: something OP tmp)
12073 GIMPLE_OMP_STORE (val)
12075 ???FIXME: Allow a more flexible sequence.
12076 Perhaps use data flow to pick the statements.
12080 gsi
= gsi_after_labels (store_bb
);
12081 stmt
= gsi_stmt (gsi
);
12082 loc
= gimple_location (stmt
);
12083 if (!is_gimple_assign (stmt
))
12086 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
12088 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
12089 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
12090 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
12091 gcc_checking_assert (!need_old
|| !need_new
);
12093 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
12096 /* Check for one of the supported fetch-op operations. */
12097 code
= gimple_assign_rhs_code (stmt
);
12101 case POINTER_PLUS_EXPR
:
12102 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
12103 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
12106 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
12107 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
12110 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
12111 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
12114 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
12115 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
12118 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
12119 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
12125 /* Make sure the expression is of the proper form. */
12126 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
12127 rhs
= gimple_assign_rhs2 (stmt
);
12128 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
12129 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
12130 rhs
= gimple_assign_rhs1 (stmt
);
12134 tmpbase
= ((enum built_in_function
)
12135 ((need_new
? newbase
: oldbase
) + index
+ 1));
12136 decl
= builtin_decl_explicit (tmpbase
);
12137 if (decl
== NULL_TREE
)
12139 itype
= TREE_TYPE (TREE_TYPE (decl
));
12140 imode
= TYPE_MODE (itype
);
12142 /* We could test all of the various optabs involved, but the fact of the
12143 matter is that (with the exception of i486 vs i586 and xadd) all targets
12144 that support any atomic operaton optab also implements compare-and-swap.
12145 Let optabs.c take care of expanding any compare-and-swap loop. */
12146 if (!can_compare_and_swap_p (imode
, true))
12149 gsi
= gsi_last_bb (load_bb
);
12150 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12152 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12153 It only requires that the operation happen atomically. Thus we can
12154 use the RELAXED memory model. */
12155 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
12156 fold_convert_loc (loc
, itype
, rhs
),
12157 build_int_cst (NULL
,
12158 seq_cst
? MEMMODEL_SEQ_CST
12159 : MEMMODEL_RELAXED
));
12161 if (need_old
|| need_new
)
12163 lhs
= need_old
? loaded_val
: stored_val
;
12164 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
12165 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
12168 call
= fold_convert_loc (loc
, void_type_node
, call
);
12169 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12170 gsi_remove (&gsi
, true);
12172 gsi
= gsi_last_bb (store_bb
);
12173 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
12174 gsi_remove (&gsi
, true);
12175 gsi
= gsi_last_bb (store_bb
);
12176 stmt
= gsi_stmt (gsi
);
12177 gsi_remove (&gsi
, true);
12179 if (gimple_in_ssa_p (cfun
))
12181 release_defs (stmt
);
12182 update_ssa (TODO_update_ssa_no_phi
);
12188 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12192 newval = rhs; // with oldval replacing *addr in rhs
12193 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12194 if (oldval != newval)
12197 INDEX is log2 of the size of the data type, and thus usable to find the
12198 index of the builtin decl. */
12201 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
12202 tree addr
, tree loaded_val
, tree stored_val
,
12205 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
12206 tree type
, itype
, cmpxchg
, iaddr
;
12207 gimple_stmt_iterator si
;
12208 basic_block loop_header
= single_succ (load_bb
);
12209 gimple
*phi
, *stmt
;
12211 enum built_in_function fncode
;
12213 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12214 order to use the RELAXED memory model effectively. */
12215 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12217 cmpxchg
= builtin_decl_explicit (fncode
);
12218 if (cmpxchg
== NULL_TREE
)
12220 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12221 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
12223 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
12226 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12227 si
= gsi_last_bb (load_bb
);
12228 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12230 /* For floating-point values, we'll need to view-convert them to integers
12231 so that we can perform the atomic compare and swap. Simplify the
12232 following code by always setting up the "i"ntegral variables. */
12233 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
12237 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
12240 = force_gimple_operand_gsi (&si
,
12241 fold_convert (TREE_TYPE (iaddr
), addr
),
12242 false, NULL_TREE
, true, GSI_SAME_STMT
);
12243 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
12244 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12245 loadedi
= create_tmp_var (itype
);
12246 if (gimple_in_ssa_p (cfun
))
12247 loadedi
= make_ssa_name (loadedi
);
12252 loadedi
= loaded_val
;
12255 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
12256 tree loaddecl
= builtin_decl_explicit (fncode
);
12259 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12260 build_call_expr (loaddecl
, 2, iaddr
,
12261 build_int_cst (NULL_TREE
,
12262 MEMMODEL_RELAXED
)));
12264 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12265 build_int_cst (TREE_TYPE (iaddr
), 0));
12268 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12271 /* Move the value to the LOADEDI temporary. */
12272 if (gimple_in_ssa_p (cfun
))
12274 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12275 phi
= create_phi_node (loadedi
, loop_header
);
12276 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12280 gsi_insert_before (&si
,
12281 gimple_build_assign (loadedi
, initial
),
12283 if (loadedi
!= loaded_val
)
12285 gimple_stmt_iterator gsi2
;
12288 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12289 gsi2
= gsi_start_bb (loop_header
);
12290 if (gimple_in_ssa_p (cfun
))
12293 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12294 true, GSI_SAME_STMT
);
12295 stmt
= gimple_build_assign (loaded_val
, x
);
12296 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12300 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12301 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12302 true, GSI_SAME_STMT
);
12305 gsi_remove (&si
, true);
12307 si
= gsi_last_bb (store_bb
);
12308 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12311 storedi
= stored_val
;
12314 force_gimple_operand_gsi (&si
,
12315 build1 (VIEW_CONVERT_EXPR
, itype
,
12316 stored_val
), true, NULL_TREE
, true,
12319 /* Build the compare&swap statement. */
12320 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12321 new_storedi
= force_gimple_operand_gsi (&si
,
12322 fold_convert (TREE_TYPE (loadedi
),
12325 true, GSI_SAME_STMT
);
12327 if (gimple_in_ssa_p (cfun
))
12328 old_vali
= loadedi
;
12331 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12332 stmt
= gimple_build_assign (old_vali
, loadedi
);
12333 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12335 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12336 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12339 /* Note that we always perform the comparison as an integer, even for
12340 floating point. This allows the atomic operation to properly
12341 succeed even with NaNs and -0.0. */
12342 stmt
= gimple_build_cond_empty
12343 (build2 (NE_EXPR
, boolean_type_node
,
12344 new_storedi
, old_vali
));
12345 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12348 e
= single_succ_edge (store_bb
);
12349 e
->flags
&= ~EDGE_FALLTHRU
;
12350 e
->flags
|= EDGE_FALSE_VALUE
;
12352 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12354 /* Copy the new value to loadedi (we already did that before the condition
12355 if we are not in SSA). */
12356 if (gimple_in_ssa_p (cfun
))
12358 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12359 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12362 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12363 gsi_remove (&si
, true);
12365 struct loop
*loop
= alloc_loop ();
12366 loop
->header
= loop_header
;
12367 loop
->latch
= store_bb
;
12368 add_loop (loop
, loop_header
->loop_father
);
12370 if (gimple_in_ssa_p (cfun
))
12371 update_ssa (TODO_update_ssa_no_phi
);
12376 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12378 GOMP_atomic_start ();
12380 GOMP_atomic_end ();
12382 The result is not globally atomic, but works so long as all parallel
12383 references are within #pragma omp atomic directives. According to
12384 responses received from omp@openmp.org, appears to be within spec.
12385 Which makes sense, since that's how several other compilers handle
12386 this situation as well.
12387 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12388 expanding. STORED_VAL is the operand of the matching
12389 GIMPLE_OMP_ATOMIC_STORE.
12392 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12393 loaded_val = *addr;
12396 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12397 *addr = stored_val;
12401 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12402 tree addr
, tree loaded_val
, tree stored_val
)
12404 gimple_stmt_iterator si
;
12408 si
= gsi_last_bb (load_bb
);
12409 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12411 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12412 t
= build_call_expr (t
, 0);
12413 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12415 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12416 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12417 gsi_remove (&si
, true);
12419 si
= gsi_last_bb (store_bb
);
12420 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12422 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12424 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12426 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12427 t
= build_call_expr (t
, 0);
12428 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12429 gsi_remove (&si
, true);
12431 if (gimple_in_ssa_p (cfun
))
12432 update_ssa (TODO_update_ssa_no_phi
);
12436 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12437 using expand_omp_atomic_fetch_op. If it failed, we try to
12438 call expand_omp_atomic_pipeline, and if it fails too, the
12439 ultimate fallback is wrapping the operation in a mutex
12440 (expand_omp_atomic_mutex). REGION is the atomic region built
12441 by build_omp_regions_1(). */
12444 expand_omp_atomic (struct omp_region
*region
)
12446 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12447 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12448 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12449 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12450 tree addr
= gimple_omp_atomic_load_rhs (load
);
12451 tree stored_val
= gimple_omp_atomic_store_val (store
);
12452 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12453 HOST_WIDE_INT index
;
12455 /* Make sure the type is one of the supported sizes. */
12456 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12457 index
= exact_log2 (index
);
12458 if (index
>= 0 && index
<= 4)
12460 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12462 /* __sync builtins require strict data alignment. */
12463 if (exact_log2 (align
) >= index
)
12466 if (loaded_val
== stored_val
12467 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12468 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12469 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12470 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12473 /* Atomic store. */
12474 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12475 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12476 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12477 && store_bb
== single_succ (load_bb
)
12478 && first_stmt (store_bb
) == store
12479 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12480 stored_val
, index
))
12483 /* When possible, use specialized atomic update functions. */
12484 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12485 && store_bb
== single_succ (load_bb
)
12486 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12487 loaded_val
, stored_val
, index
))
12490 /* If we don't have specialized __sync builtins, try and implement
12491 as a compare and swap loop. */
12492 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12493 loaded_val
, stored_val
, index
))
12498 /* The ultimate fallback is wrapping the operation in a mutex. */
12499 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12503 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12504 macro on gomp-constants.h. We do not check for overflow. */
12507 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12511 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12514 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12515 device
, build_int_cst (unsigned_type_node
,
12516 GOMP_LAUNCH_DEVICE_SHIFT
));
12517 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12522 /* Look for compute grid dimension clauses and convert to an attribute
12523 attached to FN. This permits the target-side code to (a) massage
12524 the dimensions, (b) emit that data and (c) optimize. Non-constant
12525 dimensions are pushed onto ARGS.
12527 The attribute value is a TREE_LIST. A set of dimensions is
12528 represented as a list of INTEGER_CST. Those that are runtime
12529 exprs are represented as an INTEGER_CST of zero.
12531 TOOO. Normally the attribute will just contain a single such list. If
12532 however it contains a list of lists, this will represent the use of
12533 device_type. Each member of the outer list is an assoc list of
12534 dimensions, keyed by the device type. The first entry will be the
12535 default. Well, that's the plan. */
12537 #define OACC_FN_ATTRIB "oacc function"
12539 /* Replace any existing oacc fn attribute with updated dimensions. */
12542 replace_oacc_fn_attrib (tree fn
, tree dims
)
12544 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12545 tree attribs
= DECL_ATTRIBUTES (fn
);
12547 /* If we happen to be present as the first attrib, drop it. */
12548 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12549 attribs
= TREE_CHAIN (attribs
);
12550 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12553 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12554 function attribute. Push any that are non-constant onto the ARGS
12555 list, along with an appropriate GOMP_LAUNCH_DIM tag. IS_KERNEL is
12556 true, if these are for a kernels region offload function. */
12559 set_oacc_fn_attrib (tree fn
, tree clauses
, bool is_kernel
, vec
<tree
> *args
)
12561 /* Must match GOMP_DIM ordering. */
12562 static const omp_clause_code ids
[]
12563 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12564 OMP_CLAUSE_VECTOR_LENGTH
};
12566 tree dims
[GOMP_DIM_MAX
];
12567 tree attr
= NULL_TREE
;
12568 unsigned non_const
= 0;
12570 for (ix
= GOMP_DIM_MAX
; ix
--;)
12572 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12573 tree dim
= NULL_TREE
;
12576 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12578 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12580 dim
= integer_zero_node
;
12581 non_const
|= GOMP_DIM_MASK (ix
);
12583 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12584 /* Note kernelness with TREE_PUBLIC. */
12586 TREE_PUBLIC (attr
) = 1;
12589 replace_oacc_fn_attrib (fn
, attr
);
12593 /* Push a dynamic argument set. */
12594 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12595 NULL_TREE
, non_const
));
12596 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12597 if (non_const
& GOMP_DIM_MASK (ix
))
12598 args
->safe_push (dims
[ix
]);
12602 /* Process the routine's dimension clauess to generate an attribute
12603 value. Issue diagnostics as appropriate. We default to SEQ
12604 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12605 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12606 can have a loop partitioned on it. non-zero indicates
12607 yes, zero indicates no. By construction once a non-zero has been
12608 reached, further inner dimensions must also be non-zero. We set
12609 TREE_VALUE to zero for the dimensions that may be partitioned and
12610 1 for the other ones -- if a loop is (erroneously) spawned at
12611 an outer level, we don't want to try and partition it. */
12614 build_oacc_routine_dims (tree clauses
)
12616 /* Must match GOMP_DIM ordering. */
12617 static const omp_clause_code ids
[] =
12618 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12622 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12623 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12624 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12627 error_at (OMP_CLAUSE_LOCATION (clauses
),
12628 "multiple loop axes specified for routine");
12633 /* Default to SEQ. */
12635 level
= GOMP_DIM_MAX
;
12637 tree dims
= NULL_TREE
;
12639 for (ix
= GOMP_DIM_MAX
; ix
--;)
12640 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12641 build_int_cst (integer_type_node
, ix
< level
), dims
);
12646 /* Retrieve the oacc function attrib and return it. Non-oacc
12647 functions will return NULL. */
12650 get_oacc_fn_attrib (tree fn
)
12652 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12655 /* Return true if this oacc fn attrib is for a kernels offload
12656 region. We use the TREE_PUBLIC flag of each dimension -- only
12657 need to check the first one. */
12660 oacc_fn_attrib_kernels_p (tree attr
)
12662 return TREE_PUBLIC (TREE_VALUE (attr
));
12665 /* Return level at which oacc routine may spawn a partitioned loop, or
12666 -1 if it is not a routine (i.e. is an offload fn). */
12669 oacc_fn_attrib_level (tree attr
)
12671 tree pos
= TREE_VALUE (attr
);
12673 if (!TREE_PURPOSE (pos
))
12677 for (ix
= 0; ix
!= GOMP_DIM_MAX
;
12678 ix
++, pos
= TREE_CHAIN (pos
))
12679 if (!integer_zerop (TREE_PURPOSE (pos
)))
12685 /* Extract an oacc execution dimension from FN. FN must be an
12686 offloaded function or routine that has already had its execution
12687 dimensions lowered to the target-specific values. */
12690 get_oacc_fn_dim_size (tree fn
, int axis
)
12692 tree attrs
= get_oacc_fn_attrib (fn
);
12694 gcc_assert (axis
< GOMP_DIM_MAX
);
12696 tree dims
= TREE_VALUE (attrs
);
12698 dims
= TREE_CHAIN (dims
);
12700 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12705 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12706 IFN_GOACC_DIM_SIZE call. */
12709 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12711 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12712 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12713 tree arg
= gimple_call_arg (stmt
, 0);
12714 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12716 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12720 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12724 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
12725 basic_block region_exit
)
12727 struct loop
*outer
= region_entry
->loop_father
;
12728 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
12730 /* Don't parallelize the kernels region if it contains more than one outer
12732 unsigned int nr_outer_loops
= 0;
12733 struct loop
*single_outer
= NULL
;
12734 for (struct loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
12736 gcc_assert (loop_outer (loop
) == outer
);
12738 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
12741 if (region_exit
!= NULL
12742 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
12746 single_outer
= loop
;
12748 if (nr_outer_loops
!= 1)
12751 for (struct loop
*loop
= single_outer
->inner
; loop
!= NULL
; loop
= loop
->inner
)
12755 /* Mark the loops in the region. */
12756 for (struct loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
12757 loop
->in_oacc_kernels_region
= true;
12760 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12762 struct GTY(()) grid_launch_attributes_trees
12764 tree kernel_dim_array_type
;
12765 tree kernel_lattrs_dimnum_decl
;
12766 tree kernel_lattrs_grid_decl
;
12767 tree kernel_lattrs_group_decl
;
12768 tree kernel_launch_attributes_type
;
12771 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
12773 /* Create types used to pass kernel launch attributes to target. */
12776 grid_create_kernel_launch_attr_types (void)
12778 if (grid_attr_trees
)
12780 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
12782 tree dim_arr_index_type
12783 = build_index_type (build_int_cst (integer_type_node
, 2));
12784 grid_attr_trees
->kernel_dim_array_type
12785 = build_array_type (uint32_type_node
, dim_arr_index_type
);
12787 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
12788 grid_attr_trees
->kernel_lattrs_dimnum_decl
12789 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
12791 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
12793 grid_attr_trees
->kernel_lattrs_grid_decl
12794 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
12795 grid_attr_trees
->kernel_dim_array_type
);
12796 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
12797 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
12798 grid_attr_trees
->kernel_lattrs_group_decl
12799 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
12800 grid_attr_trees
->kernel_dim_array_type
);
12801 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
12802 = grid_attr_trees
->kernel_lattrs_grid_decl
;
12803 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
12804 "__gomp_kernel_launch_attributes",
12805 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
12808 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12809 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12810 of type uint32_type_node. */
12813 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
12814 tree fld_decl
, int index
, tree value
)
12816 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
12817 build3 (COMPONENT_REF
,
12818 grid_attr_trees
->kernel_dim_array_type
,
12819 range_var
, fld_decl
, NULL_TREE
),
12820 build_int_cst (integer_type_node
, index
),
12821 NULL_TREE
, NULL_TREE
);
12822 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
12825 /* Return a tree representation of a pointer to a structure with grid and
12826 work-group size information. Statements filling that information will be
12827 inserted before GSI, TGT_STMT is the target statement which has the
12828 necessary information in it. */
12831 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
12832 gomp_target
*tgt_stmt
)
12834 grid_create_kernel_launch_attr_types ();
12835 tree u32_one
= build_one_cst (uint32_type_node
);
12836 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
12837 "__kernel_launch_attrs");
12839 unsigned max_dim
= 0;
12840 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
12842 clause
= OMP_CLAUSE_CHAIN (clause
))
12844 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
12847 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
12848 max_dim
= MAX (dim
, max_dim
);
12850 grid_insert_store_range_dim (gsi
, lattrs
,
12851 grid_attr_trees
->kernel_lattrs_grid_decl
,
12852 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
12853 grid_insert_store_range_dim (gsi
, lattrs
,
12854 grid_attr_trees
->kernel_lattrs_group_decl
,
12855 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
12858 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
12859 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
12860 /* At this moment we cannot gridify a loop with a collapse clause. */
12861 /* TODO: Adjust when we support bigger collapse. */
12862 gcc_assert (max_dim
== 0);
12863 gsi_insert_before (gsi
, gimple_build_assign (dimref
, u32_one
), GSI_SAME_STMT
);
12864 TREE_ADDRESSABLE (lattrs
) = 1;
12865 return build_fold_addr_expr (lattrs
);
12868 /* Build target argument identifier from the DEVICE identifier, value
12869 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12872 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
12874 tree t
= build_int_cst (integer_type_node
, device
);
12875 if (subseqent_param
)
12876 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12877 build_int_cst (integer_type_node
,
12878 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
12879 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12880 build_int_cst (integer_type_node
, id
));
12884 /* Like above but return it in type that can be directly stored as an element
12885 of the argument array. */
12888 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
12890 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
12891 return fold_convert (ptr_type_node
, t
);
12894 /* Return a target argument consisting of DEVICE identifier, value identifier
12895 ID, and the actual VALUE. */
12898 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
12901 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
12902 fold_convert (integer_type_node
, value
),
12903 build_int_cst (unsigned_type_node
,
12904 GOMP_TARGET_ARG_VALUE_SHIFT
));
12905 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12906 get_target_argument_identifier_1 (device
, false, id
));
12907 t
= fold_convert (ptr_type_node
, t
);
12908 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
12911 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12912 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12913 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12917 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
12918 int id
, tree value
, vec
<tree
> *args
)
12920 if (tree_fits_shwi_p (value
)
12921 && tree_to_shwi (value
) > -(1 << 15)
12922 && tree_to_shwi (value
) < (1 << 15))
12923 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
12926 args
->quick_push (get_target_argument_identifier (device
, true, id
));
12927 value
= fold_convert (ptr_type_node
, value
);
12928 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
12930 args
->quick_push (value
);
12934 /* Create an array of arguments that is then passed to GOMP_target. */
12937 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
12939 auto_vec
<tree
, 6> args
;
12940 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
12941 tree t
, c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12943 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
12945 t
= integer_minus_one_node
;
12946 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12947 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
12949 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12951 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
12953 t
= integer_minus_one_node
;
12954 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12955 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
12958 /* Add HSA-specific grid sizes, if available. */
12959 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
12960 OMP_CLAUSE__GRIDDIM_
))
12962 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true,
12963 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
);
12964 args
.quick_push (t
);
12965 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
12968 /* Produce more, perhaps device specific, arguments here. */
12970 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
12971 args
.length () + 1),
12972 ".omp_target_args");
12973 for (unsigned i
= 0; i
< args
.length (); i
++)
12975 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12976 build_int_cst (integer_type_node
, i
),
12977 NULL_TREE
, NULL_TREE
);
12978 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
12981 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12982 build_int_cst (integer_type_node
, args
.length ()),
12983 NULL_TREE
, NULL_TREE
);
12984 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
12986 TREE_ADDRESSABLE (argarray
) = 1;
12987 return build_fold_addr_expr (argarray
);
12990 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12993 expand_omp_target (struct omp_region
*region
)
12995 basic_block entry_bb
, exit_bb
, new_bb
;
12996 struct function
*child_cfun
;
12997 tree child_fn
, block
, t
;
12998 gimple_stmt_iterator gsi
;
12999 gomp_target
*entry_stmt
;
13002 bool offloaded
, data_region
;
13004 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
13005 new_bb
= region
->entry
;
13007 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
13008 switch (gimple_omp_target_kind (entry_stmt
))
13010 case GF_OMP_TARGET_KIND_REGION
:
13011 case GF_OMP_TARGET_KIND_UPDATE
:
13012 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13013 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13014 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13015 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13016 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13017 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13018 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13019 data_region
= false;
13021 case GF_OMP_TARGET_KIND_DATA
:
13022 case GF_OMP_TARGET_KIND_OACC_DATA
:
13023 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13024 data_region
= true;
13027 gcc_unreachable ();
13030 child_fn
= NULL_TREE
;
13034 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
13035 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
13038 /* Supported by expand_omp_taskreg, but not here. */
13039 if (child_cfun
!= NULL
)
13040 gcc_checking_assert (!child_cfun
->cfg
);
13041 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
13043 entry_bb
= region
->entry
;
13044 exit_bb
= region
->exit
;
13046 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
13047 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
13051 unsigned srcidx
, dstidx
, num
;
13053 /* If the offloading region needs data sent from the parent
13054 function, then the very first statement (except possible
13055 tree profile counter updates) of the offloading body
13056 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
13057 &.OMP_DATA_O is passed as an argument to the child function,
13058 we need to replace it with the argument as seen by the child
13061 In most cases, this will end up being the identity assignment
13062 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
13063 a function call that has been inlined, the original PARM_DECL
13064 .OMP_DATA_I may have been converted into a different local
13065 variable. In which case, we need to keep the assignment. */
13066 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
13069 basic_block entry_succ_bb
= single_succ (entry_bb
);
13070 gimple_stmt_iterator gsi
;
13072 gimple
*tgtcopy_stmt
= NULL
;
13073 tree sender
= TREE_VEC_ELT (data_arg
, 0);
13075 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
13077 gcc_assert (!gsi_end_p (gsi
));
13078 stmt
= gsi_stmt (gsi
);
13079 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
13082 if (gimple_num_ops (stmt
) == 2)
13084 tree arg
= gimple_assign_rhs1 (stmt
);
13086 /* We're ignoring the subcode because we're
13087 effectively doing a STRIP_NOPS. */
13089 if (TREE_CODE (arg
) == ADDR_EXPR
13090 && TREE_OPERAND (arg
, 0) == sender
)
13092 tgtcopy_stmt
= stmt
;
13098 gcc_assert (tgtcopy_stmt
!= NULL
);
13099 arg
= DECL_ARGUMENTS (child_fn
);
13101 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
13102 gsi_remove (&gsi
, true);
13105 /* Declare local variables needed in CHILD_CFUN. */
13106 block
= DECL_INITIAL (child_fn
);
13107 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
13108 /* The gimplifier could record temporaries in the offloading block
13109 rather than in containing function's local_decls chain,
13110 which would mean cgraph missed finalizing them. Do it now. */
13111 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
13112 if (TREE_CODE (t
) == VAR_DECL
13114 && !DECL_EXTERNAL (t
))
13115 varpool_node::finalize_decl (t
);
13116 DECL_SAVED_TREE (child_fn
) = NULL
;
13117 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13118 gimple_set_body (child_fn
, NULL
);
13119 TREE_USED (block
) = 1;
13121 /* Reset DECL_CONTEXT on function arguments. */
13122 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13123 DECL_CONTEXT (t
) = child_fn
;
13125 /* Split ENTRY_BB at GIMPLE_*,
13126 so that it can be moved to the child function. */
13127 gsi
= gsi_last_bb (entry_bb
);
13128 stmt
= gsi_stmt (gsi
);
13130 && gimple_code (stmt
) == gimple_code (entry_stmt
));
13131 e
= split_block (entry_bb
, stmt
);
13132 gsi_remove (&gsi
, true);
13133 entry_bb
= e
->dest
;
13134 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
13136 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13139 gsi
= gsi_last_bb (exit_bb
);
13140 gcc_assert (!gsi_end_p (gsi
)
13141 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13142 stmt
= gimple_build_return (NULL
);
13143 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
13144 gsi_remove (&gsi
, true);
13147 /* Move the offloading region into CHILD_CFUN. */
13149 block
= gimple_block (entry_stmt
);
13151 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
13153 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
13154 /* When the OMP expansion process cannot guarantee an up-to-date
13155 loop tree arrange for the child function to fixup loops. */
13156 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13157 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
13159 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13160 num
= vec_safe_length (child_cfun
->local_decls
);
13161 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
13163 t
= (*child_cfun
->local_decls
)[srcidx
];
13164 if (DECL_CONTEXT (t
) == cfun
->decl
)
13166 if (srcidx
!= dstidx
)
13167 (*child_cfun
->local_decls
)[dstidx
] = t
;
13171 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
13173 /* Inform the callgraph about the new function. */
13174 child_cfun
->curr_properties
= cfun
->curr_properties
;
13175 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
13176 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
13177 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
13178 node
->parallelized_function
= 1;
13179 cgraph_node::add_new_function (child_fn
, true);
13181 /* Add the new function to the offload table. */
13182 if (ENABLE_OFFLOADING
)
13183 vec_safe_push (offload_funcs
, child_fn
);
13185 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
13186 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
13188 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13189 fixed in a following pass. */
13190 push_cfun (child_cfun
);
13192 assign_assembler_name_if_neeeded (child_fn
);
13193 cgraph_edge::rebuild_edges ();
13195 /* Some EH regions might become dead, see PR34608. If
13196 pass_cleanup_cfg isn't the first pass to happen with the
13197 new child, these dead EH edges might cause problems.
13198 Clean them up now. */
13199 if (flag_exceptions
)
13202 bool changed
= false;
13204 FOR_EACH_BB_FN (bb
, cfun
)
13205 changed
|= gimple_purge_dead_eh_edges (bb
);
13207 cleanup_tree_cfg ();
13209 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13210 verify_loop_structure ();
13213 if (dump_file
&& !gimple_in_ssa_p (cfun
))
13215 omp_any_child_fn_dumped
= true;
13216 dump_function_header (dump_file
, child_fn
, dump_flags
);
13217 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
13221 /* Emit a library call to launch the offloading region, or do data
13223 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
13224 enum built_in_function start_ix
;
13225 location_t clause_loc
;
13226 unsigned int flags_i
= 0;
13227 bool oacc_kernels_p
= false;
13229 switch (gimple_omp_target_kind (entry_stmt
))
13231 case GF_OMP_TARGET_KIND_REGION
:
13232 start_ix
= BUILT_IN_GOMP_TARGET
;
13234 case GF_OMP_TARGET_KIND_DATA
:
13235 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
13237 case GF_OMP_TARGET_KIND_UPDATE
:
13238 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
13240 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13241 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13243 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13244 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13245 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
13247 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13248 oacc_kernels_p
= true;
13250 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13251 start_ix
= BUILT_IN_GOACC_PARALLEL
;
13253 case GF_OMP_TARGET_KIND_OACC_DATA
:
13254 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13255 start_ix
= BUILT_IN_GOACC_DATA_START
;
13257 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13258 start_ix
= BUILT_IN_GOACC_UPDATE
;
13260 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13261 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
13263 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13264 start_ix
= BUILT_IN_GOACC_DECLARE
;
13267 gcc_unreachable ();
13270 clauses
= gimple_omp_target_clauses (entry_stmt
);
13272 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13273 library choose) and there is no conditional. */
13275 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
13277 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
13279 cond
= OMP_CLAUSE_IF_EXPR (c
);
13281 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
13284 /* Even if we pass it to all library function calls, it is currently only
13285 defined/used for the OpenMP target ones. */
13286 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
13287 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
13288 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
13289 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
13291 device
= OMP_CLAUSE_DEVICE_ID (c
);
13292 clause_loc
= OMP_CLAUSE_LOCATION (c
);
13295 clause_loc
= gimple_location (entry_stmt
);
13297 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
13299 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
13301 /* Ensure 'device' is of the correct type. */
13302 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
13304 /* If we found the clause 'if (cond)', build
13305 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13308 cond
= gimple_boolify (cond
);
13310 basic_block cond_bb
, then_bb
, else_bb
;
13314 tmp_var
= create_tmp_var (TREE_TYPE (device
));
13316 e
= split_block_after_labels (new_bb
);
13319 gsi
= gsi_last_bb (new_bb
);
13321 e
= split_block (new_bb
, gsi_stmt (gsi
));
13327 then_bb
= create_empty_bb (cond_bb
);
13328 else_bb
= create_empty_bb (then_bb
);
13329 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
13330 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
13332 stmt
= gimple_build_cond_empty (cond
);
13333 gsi
= gsi_last_bb (cond_bb
);
13334 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13336 gsi
= gsi_start_bb (then_bb
);
13337 stmt
= gimple_build_assign (tmp_var
, device
);
13338 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13340 gsi
= gsi_start_bb (else_bb
);
13341 stmt
= gimple_build_assign (tmp_var
,
13342 build_int_cst (integer_type_node
,
13343 GOMP_DEVICE_HOST_FALLBACK
));
13344 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13346 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
13347 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
13348 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
13349 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
13350 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
13351 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
13354 gsi
= gsi_last_bb (new_bb
);
13358 gsi
= gsi_last_bb (new_bb
);
13359 device
= force_gimple_operand_gsi (&gsi
, device
, true, NULL_TREE
,
13360 true, GSI_SAME_STMT
);
13363 t
= gimple_omp_target_data_arg (entry_stmt
);
13366 t1
= size_zero_node
;
13367 t2
= build_zero_cst (ptr_type_node
);
13373 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
13374 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
13375 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
13376 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
13377 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
13381 bool tagging
= false;
13382 /* The maximum number used by any start_ix, without varargs. */
13383 auto_vec
<tree
, 11> args
;
13384 args
.quick_push (device
);
13386 args
.quick_push (build_fold_addr_expr (child_fn
));
13387 args
.quick_push (t1
);
13388 args
.quick_push (t2
);
13389 args
.quick_push (t3
);
13390 args
.quick_push (t4
);
13393 case BUILT_IN_GOACC_DATA_START
:
13394 case BUILT_IN_GOACC_DECLARE
:
13395 case BUILT_IN_GOMP_TARGET_DATA
:
13397 case BUILT_IN_GOMP_TARGET
:
13398 case BUILT_IN_GOMP_TARGET_UPDATE
:
13399 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
13400 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
13401 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
13403 depend
= OMP_CLAUSE_DECL (c
);
13405 depend
= build_int_cst (ptr_type_node
, 0);
13406 args
.quick_push (depend
);
13407 if (start_ix
== BUILT_IN_GOMP_TARGET
)
13408 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
13410 case BUILT_IN_GOACC_PARALLEL
:
13412 set_oacc_fn_attrib (child_fn
, clauses
, oacc_kernels_p
, &args
);
13416 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
13417 case BUILT_IN_GOACC_UPDATE
:
13419 tree t_async
= NULL_TREE
;
13421 /* If present, use the value specified by the respective
13422 clause, making sure that is of the correct type. */
13423 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
13425 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13427 OMP_CLAUSE_ASYNC_EXPR (c
));
13429 /* Default values for t_async. */
13430 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
13432 build_int_cst (integer_type_node
,
13434 if (tagging
&& t_async
)
13436 unsigned HOST_WIDE_INT i_async
= GOMP_LAUNCH_OP_MAX
;
13438 if (TREE_CODE (t_async
) == INTEGER_CST
)
13440 /* See if we can pack the async arg in to the tag's
13442 i_async
= TREE_INT_CST_LOW (t_async
);
13443 if (i_async
< GOMP_LAUNCH_OP_MAX
)
13444 t_async
= NULL_TREE
;
13446 i_async
= GOMP_LAUNCH_OP_MAX
;
13448 args
.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC
, NULL_TREE
,
13452 args
.safe_push (t_async
);
13454 /* Save the argument index, and ... */
13455 unsigned t_wait_idx
= args
.length ();
13456 unsigned num_waits
= 0;
13457 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
13459 /* ... push a placeholder. */
13460 args
.safe_push (integer_zero_node
);
13462 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
13463 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
13465 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13467 OMP_CLAUSE_WAIT_EXPR (c
)));
13471 if (!tagging
|| num_waits
)
13475 /* Now that we know the number, update the placeholder. */
13477 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
13479 len
= build_int_cst (integer_type_node
, num_waits
);
13480 len
= fold_convert_loc (gimple_location (entry_stmt
),
13481 unsigned_type_node
, len
);
13482 args
[t_wait_idx
] = len
;
13487 gcc_unreachable ();
13490 /* Push terminal marker - zero. */
13491 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
13493 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
13494 gimple_set_location (g
, gimple_location (entry_stmt
));
13495 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13498 g
= gsi_stmt (gsi
);
13499 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
13500 gsi_remove (&gsi
, true);
13502 if (data_region
&& region
->exit
)
13504 gsi
= gsi_last_bb (region
->exit
);
13505 g
= gsi_stmt (gsi
);
13506 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
13507 gsi_remove (&gsi
, true);
13511 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13512 variable derived from the thread number. */
13515 grid_expand_omp_for_loop (struct omp_region
*kfor
)
13519 gimple_stmt_iterator gsi
;
13521 struct omp_for_data fd
;
13523 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
13524 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
13525 == GF_OMP_FOR_KIND_GRID_LOOP
);
13526 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
13528 gcc_assert (gimple_omp_for_collapse (for_stmt
) == 1);
13529 gcc_assert (kfor
->cont
);
13530 extract_omp_for_data (for_stmt
, &fd
, NULL
);
13532 itype
= type
= TREE_TYPE (fd
.loop
.v
);
13533 if (POINTER_TYPE_P (type
))
13534 itype
= signed_type_for (type
);
13536 gsi
= gsi_start_bb (body_bb
);
13539 step
= fd
.loop
.step
;
13540 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
13541 true, NULL_TREE
, true, GSI_SAME_STMT
);
13542 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
13543 true, NULL_TREE
, true, GSI_SAME_STMT
);
13544 threadid
= build_call_expr (builtin_decl_explicit
13545 (BUILT_IN_OMP_GET_THREAD_NUM
), 0);
13546 threadid
= fold_convert (itype
, threadid
);
13547 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
13548 true, GSI_SAME_STMT
);
13550 tree startvar
= fd
.loop
.v
;
13551 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
13552 if (POINTER_TYPE_P (type
))
13553 t
= fold_build_pointer_plus (n1
, t
);
13555 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
13556 t
= fold_convert (type
, t
);
13557 t
= force_gimple_operand_gsi (&gsi
, t
,
13559 && TREE_ADDRESSABLE (startvar
),
13560 NULL_TREE
, true, GSI_SAME_STMT
);
13561 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
13562 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
13564 /* Remove the omp for statement */
13565 gsi
= gsi_last_bb (kfor
->entry
);
13566 gsi_remove (&gsi
, true);
13568 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13569 gsi
= gsi_last_bb (kfor
->cont
);
13570 gcc_assert (!gsi_end_p (gsi
)
13571 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
13572 gsi_remove (&gsi
, true);
13574 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13575 gsi
= gsi_last_bb (kfor
->exit
);
13576 gcc_assert (!gsi_end_p (gsi
)
13577 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13578 gsi_remove (&gsi
, true);
13580 /* Fixup the much simpler CFG. */
13581 remove_edge (find_edge (kfor
->cont
, body_bb
));
13583 if (kfor
->cont
!= body_bb
)
13584 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
13585 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
13588 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13591 struct grid_arg_decl_map
13597 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13598 pertaining to kernel function. */
13601 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
13603 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13604 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
13607 if (t
== adm
->old_arg
)
13608 *tp
= adm
->new_arg
;
13609 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
13613 static void expand_omp (struct omp_region
*region
);
13615 /* If TARGET region contains a kernel body for loop, remove its region from the
13616 TARGET and expand it in GPGPU kernel fashion. */
13619 grid_expand_target_grid_body (struct omp_region
*target
)
13621 if (!hsa_gen_requested_p ())
13624 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
13625 struct omp_region
**pp
;
13627 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
13628 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
13631 struct omp_region
*gpukernel
= *pp
;
13633 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
13636 /* HSA cannot handle OACC stuff. */
13637 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
13639 gcc_checking_assert (orig_child_fndecl
);
13640 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13641 OMP_CLAUSE__GRIDDIM_
));
13642 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
13644 hsa_register_kernel (n
);
13648 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13649 OMP_CLAUSE__GRIDDIM_
));
13650 tree inside_block
= gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
13651 *pp
= gpukernel
->next
;
13652 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
13653 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
13656 struct omp_region
*kfor
= *pp
;
13658 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor
)->entry
))
13659 == GF_OMP_FOR_KIND_GRID_LOOP
);
13662 expand_omp (kfor
->inner
);
13663 if (gpukernel
->inner
)
13664 expand_omp (gpukernel
->inner
);
13666 tree kern_fndecl
= copy_node (orig_child_fndecl
);
13667 DECL_NAME (kern_fndecl
) = clone_function_name (kern_fndecl
, "kernel");
13668 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
13669 tree tgtblock
= gimple_block (tgt_stmt
);
13670 tree fniniblock
= make_node (BLOCK
);
13671 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = tgtblock
;
13672 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
13673 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
13674 BLOCK_SUPERCONTEXT (fniniblock
) = kern_fndecl
;
13675 DECL_INITIAL (kern_fndecl
) = fniniblock
;
13676 push_struct_function (kern_fndecl
);
13677 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
13678 init_tree_ssa (cfun
);
13681 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
13682 gcc_assert (!DECL_CHAIN (old_parm_decl
));
13683 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
13684 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
13685 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
13686 gcc_assert (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (kern_fndecl
))));
13687 DECL_RESULT (kern_fndecl
) = copy_node (DECL_RESULT (kern_fndecl
));
13688 DECL_CONTEXT (DECL_RESULT (kern_fndecl
)) = kern_fndecl
;
13689 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
13690 kern_cfun
->curr_properties
= cfun
->curr_properties
;
13692 remove_edge (BRANCH_EDGE (kfor
->entry
));
13693 grid_expand_omp_for_loop (kfor
);
13695 /* Remove the omp for statement */
13696 gimple_stmt_iterator gsi
= gsi_last_bb (gpukernel
->entry
);
13697 gsi_remove (&gsi
, true);
13698 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13700 gsi
= gsi_last_bb (gpukernel
->exit
);
13701 gcc_assert (!gsi_end_p (gsi
)
13702 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13703 gimple
*ret_stmt
= gimple_build_return (NULL
);
13704 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
13705 gsi_remove (&gsi
, true);
13707 /* Statements in the first BB in the target construct have been produced by
13708 target lowering and must be copied inside the GPUKERNEL, with the two
13709 exceptions of the first OMP statement and the OMP_DATA assignment
13711 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
13712 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
13713 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
13714 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
13715 !gsi_end_p (tsi
); gsi_next (&tsi
))
13717 gimple
*stmt
= gsi_stmt (tsi
);
13718 if (is_gimple_omp (stmt
))
13721 && is_gimple_assign (stmt
)
13722 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
13723 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
13725 gimple
*copy
= gimple_copy (stmt
);
13726 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
13727 gimple_set_block (copy
, fniniblock
);
13730 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
13731 gpukernel
->exit
, inside_block
);
13733 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
13734 kcn
->mark_force_output ();
13735 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
13737 hsa_register_kernel (kcn
, orig_child
);
13739 cgraph_node::add_new_function (kern_fndecl
, true);
13740 push_cfun (kern_cfun
);
13741 cgraph_edge::rebuild_edges ();
13743 /* Re-map any mention of the PARM_DECL of the original function to the
13744 PARM_DECL of the new one.
13746 TODO: It would be great if lowering produced references into the GPU
13747 kernel decl straight away and we did not have to do this. */
13748 struct grid_arg_decl_map adm
;
13749 adm
.old_arg
= old_parm_decl
;
13750 adm
.new_arg
= new_parm_decl
;
13752 FOR_EACH_BB_FN (bb
, kern_cfun
)
13754 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13756 gimple
*stmt
= gsi_stmt (gsi
);
13757 struct walk_stmt_info wi
;
13758 memset (&wi
, 0, sizeof (wi
));
13760 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
13768 /* Expand the parallel region tree rooted at REGION. Expansion
13769 proceeds in depth-first order. Innermost regions are expanded
13770 first. This way, parallel regions that require a new function to
13771 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13772 internal dependencies in their body. */
13775 expand_omp (struct omp_region
*region
)
13777 omp_any_child_fn_dumped
= false;
13780 location_t saved_location
;
13781 gimple
*inner_stmt
= NULL
;
13783 /* First, determine whether this is a combined parallel+workshare
13785 if (region
->type
== GIMPLE_OMP_PARALLEL
)
13786 determine_parallel_type (region
);
13787 else if (region
->type
== GIMPLE_OMP_TARGET
)
13788 grid_expand_target_grid_body (region
);
13790 if (region
->type
== GIMPLE_OMP_FOR
13791 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
13792 inner_stmt
= last_stmt (region
->inner
->entry
);
13795 expand_omp (region
->inner
);
13797 saved_location
= input_location
;
13798 if (gimple_has_location (last_stmt (region
->entry
)))
13799 input_location
= gimple_location (last_stmt (region
->entry
));
13801 switch (region
->type
)
13803 case GIMPLE_OMP_PARALLEL
:
13804 case GIMPLE_OMP_TASK
:
13805 expand_omp_taskreg (region
);
13808 case GIMPLE_OMP_FOR
:
13809 expand_omp_for (region
, inner_stmt
);
13812 case GIMPLE_OMP_SECTIONS
:
13813 expand_omp_sections (region
);
13816 case GIMPLE_OMP_SECTION
:
13817 /* Individual omp sections are handled together with their
13818 parent GIMPLE_OMP_SECTIONS region. */
13821 case GIMPLE_OMP_SINGLE
:
13822 expand_omp_single (region
);
13825 case GIMPLE_OMP_ORDERED
:
13827 gomp_ordered
*ord_stmt
13828 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13829 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13830 OMP_CLAUSE_DEPEND
))
13832 /* We'll expand these when expanding corresponding
13833 worksharing region with ordered(n) clause. */
13834 gcc_assert (region
->outer
13835 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13836 region
->ord_stmt
= ord_stmt
;
13841 case GIMPLE_OMP_MASTER
:
13842 case GIMPLE_OMP_TASKGROUP
:
13843 case GIMPLE_OMP_CRITICAL
:
13844 case GIMPLE_OMP_TEAMS
:
13845 expand_omp_synch (region
);
13848 case GIMPLE_OMP_ATOMIC_LOAD
:
13849 expand_omp_atomic (region
);
13852 case GIMPLE_OMP_TARGET
:
13853 expand_omp_target (region
);
13857 gcc_unreachable ();
13860 input_location
= saved_location
;
13861 region
= region
->next
;
13863 if (omp_any_child_fn_dumped
)
13866 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13867 omp_any_child_fn_dumped
= false;
13872 /* Helper for build_omp_regions. Scan the dominator tree starting at
13873 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13874 true, the function ends once a single tree is built (otherwise, whole
13875 forest of OMP constructs may be built). */
13878 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13881 gimple_stmt_iterator gsi
;
13885 gsi
= gsi_last_bb (bb
);
13886 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13888 struct omp_region
*region
;
13889 enum gimple_code code
;
13891 stmt
= gsi_stmt (gsi
);
13892 code
= gimple_code (stmt
);
13893 if (code
== GIMPLE_OMP_RETURN
)
13895 /* STMT is the return point out of region PARENT. Mark it
13896 as the exit point and make PARENT the immediately
13897 enclosing region. */
13898 gcc_assert (parent
);
13901 parent
= parent
->outer
;
13903 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13905 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13906 GIMPLE_OMP_RETURN, but matches with
13907 GIMPLE_OMP_ATOMIC_LOAD. */
13908 gcc_assert (parent
);
13909 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13912 parent
= parent
->outer
;
13914 else if (code
== GIMPLE_OMP_CONTINUE
)
13916 gcc_assert (parent
);
13919 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13921 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13922 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13926 region
= new_omp_region (bb
, code
, parent
);
13928 if (code
== GIMPLE_OMP_TARGET
)
13930 switch (gimple_omp_target_kind (stmt
))
13932 case GF_OMP_TARGET_KIND_REGION
:
13933 case GF_OMP_TARGET_KIND_DATA
:
13934 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13935 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13936 case GF_OMP_TARGET_KIND_OACC_DATA
:
13937 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13939 case GF_OMP_TARGET_KIND_UPDATE
:
13940 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13941 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13942 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13943 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13944 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13945 /* ..., other than for those stand-alone directives... */
13949 gcc_unreachable ();
13952 else if (code
== GIMPLE_OMP_ORDERED
13953 && find_omp_clause (gimple_omp_ordered_clauses
13954 (as_a
<gomp_ordered
*> (stmt
)),
13955 OMP_CLAUSE_DEPEND
))
13956 /* #pragma omp ordered depend is also just a stand-alone
13959 /* ..., this directive becomes the parent for a new region. */
13965 if (single_tree
&& !parent
)
13968 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13970 son
= next_dom_son (CDI_DOMINATORS
, son
))
13971 build_omp_regions_1 (son
, parent
, single_tree
);
13974 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13975 root_omp_region. */
13978 build_omp_regions_root (basic_block root
)
13980 gcc_assert (root_omp_region
== NULL
);
13981 build_omp_regions_1 (root
, NULL
, true);
13982 gcc_assert (root_omp_region
!= NULL
);
13985 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13988 omp_expand_local (basic_block head
)
13990 build_omp_regions_root (head
);
13991 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13993 fprintf (dump_file
, "\nOMP region tree\n\n");
13994 dump_omp_region (dump_file
, root_omp_region
, 0);
13995 fprintf (dump_file
, "\n");
13998 remove_exit_barriers (root_omp_region
);
13999 expand_omp (root_omp_region
);
14001 free_omp_regions ();
14004 /* Scan the CFG and build a tree of OMP regions. Return the root of
14005 the OMP region tree. */
14008 build_omp_regions (void)
14010 gcc_assert (root_omp_region
== NULL
);
14011 calculate_dominance_info (CDI_DOMINATORS
);
14012 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
14015 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
14017 static unsigned int
14018 execute_expand_omp (void)
14020 build_omp_regions ();
14022 if (!root_omp_region
)
14027 fprintf (dump_file
, "\nOMP region tree\n\n");
14028 dump_omp_region (dump_file
, root_omp_region
, 0);
14029 fprintf (dump_file
, "\n");
14032 remove_exit_barriers (root_omp_region
);
14034 expand_omp (root_omp_region
);
14036 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
14037 verify_loop_structure ();
14038 cleanup_tree_cfg ();
14040 free_omp_regions ();
14045 /* OMP expansion -- the default pass, run before creation of SSA form. */
14049 const pass_data pass_data_expand_omp
=
14051 GIMPLE_PASS
, /* type */
14052 "ompexp", /* name */
14053 OPTGROUP_NONE
, /* optinfo_flags */
14054 TV_NONE
, /* tv_id */
14055 PROP_gimple_any
, /* properties_required */
14056 PROP_gimple_eomp
, /* properties_provided */
14057 0, /* properties_destroyed */
14058 0, /* todo_flags_start */
14059 0, /* todo_flags_finish */
14062 class pass_expand_omp
: public gimple_opt_pass
14065 pass_expand_omp (gcc::context
*ctxt
)
14066 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
14069 /* opt_pass methods: */
14070 virtual unsigned int execute (function
*)
14072 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
14073 || flag_openmp_simd
!= 0)
14074 && !seen_error ());
14076 /* This pass always runs, to provide PROP_gimple_eomp.
14077 But often, there is nothing to do. */
14081 return execute_expand_omp ();
14084 }; // class pass_expand_omp
14086 } // anon namespace
14089 make_pass_expand_omp (gcc::context
*ctxt
)
14091 return new pass_expand_omp (ctxt
);
14096 const pass_data pass_data_expand_omp_ssa
=
14098 GIMPLE_PASS
, /* type */
14099 "ompexpssa", /* name */
14100 OPTGROUP_NONE
, /* optinfo_flags */
14101 TV_NONE
, /* tv_id */
14102 PROP_cfg
| PROP_ssa
, /* properties_required */
14103 PROP_gimple_eomp
, /* properties_provided */
14104 0, /* properties_destroyed */
14105 0, /* todo_flags_start */
14106 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
14109 class pass_expand_omp_ssa
: public gimple_opt_pass
14112 pass_expand_omp_ssa (gcc::context
*ctxt
)
14113 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
14116 /* opt_pass methods: */
14117 virtual bool gate (function
*fun
)
14119 return !(fun
->curr_properties
& PROP_gimple_eomp
);
14121 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
14122 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
14124 }; // class pass_expand_omp_ssa
14126 } // anon namespace
14129 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
14131 return new pass_expand_omp_ssa (ctxt
);
14134 /* Routines to lower OMP directives into OMP-GIMPLE. */
14136 /* If ctx is a worksharing context inside of a cancellable parallel
14137 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14138 and conditional branch to parallel's cancel_label to handle
14139 cancellation in the implicit barrier. */
14142 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
14144 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
14145 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
14146 if (gimple_omp_return_nowait_p (omp_return
))
14149 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
14150 && ctx
->outer
->cancellable
)
14152 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
14153 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
14154 tree lhs
= create_tmp_var (c_bool_type
);
14155 gimple_omp_return_set_lhs (omp_return
, lhs
);
14156 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
14157 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
14158 fold_convert (c_bool_type
,
14159 boolean_false_node
),
14160 ctx
->outer
->cancel_label
, fallthru_label
);
14161 gimple_seq_add_stmt (body
, g
);
14162 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
14166 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14167 CTX is the enclosing OMP context for the current statement. */
14170 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14172 tree block
, control
;
14173 gimple_stmt_iterator tgsi
;
14174 gomp_sections
*stmt
;
14176 gbind
*new_stmt
, *bind
;
14177 gimple_seq ilist
, dlist
, olist
, new_body
;
14179 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
14181 push_gimplify_context ();
14185 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
14186 &ilist
, &dlist
, ctx
, NULL
);
14188 new_body
= gimple_omp_body (stmt
);
14189 gimple_omp_set_body (stmt
, NULL
);
14190 tgsi
= gsi_start (new_body
);
14191 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
14196 sec_start
= gsi_stmt (tgsi
);
14197 sctx
= maybe_lookup_ctx (sec_start
);
14200 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
14201 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
14202 GSI_CONTINUE_LINKING
);
14203 gimple_omp_set_body (sec_start
, NULL
);
14205 if (gsi_one_before_end_p (tgsi
))
14207 gimple_seq l
= NULL
;
14208 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
14210 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
14211 gimple_omp_section_set_last (sec_start
);
14214 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
14215 GSI_CONTINUE_LINKING
);
14218 block
= make_node (BLOCK
);
14219 bind
= gimple_build_bind (NULL
, new_body
, block
);
14222 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
14224 block
= make_node (BLOCK
);
14225 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14226 gsi_replace (gsi_p
, new_stmt
, true);
14228 pop_gimplify_context (new_stmt
);
14229 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14230 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14231 if (BLOCK_VARS (block
))
14232 TREE_USED (block
) = 1;
14235 gimple_seq_add_seq (&new_body
, ilist
);
14236 gimple_seq_add_stmt (&new_body
, stmt
);
14237 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
14238 gimple_seq_add_stmt (&new_body
, bind
);
14240 control
= create_tmp_var (unsigned_type_node
, ".section");
14241 t
= gimple_build_omp_continue (control
, control
);
14242 gimple_omp_sections_set_control (stmt
, control
);
14243 gimple_seq_add_stmt (&new_body
, t
);
14245 gimple_seq_add_seq (&new_body
, olist
);
14246 if (ctx
->cancellable
)
14247 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14248 gimple_seq_add_seq (&new_body
, dlist
);
14250 new_body
= maybe_catch_exception (new_body
);
14252 t
= gimple_build_omp_return
14253 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
14254 OMP_CLAUSE_NOWAIT
));
14255 gimple_seq_add_stmt (&new_body
, t
);
14256 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
14258 gimple_bind_set_body (new_stmt
, new_body
);
14262 /* A subroutine of lower_omp_single. Expand the simple form of
14263 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14265 if (GOMP_single_start ())
14267 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14269 FIXME. It may be better to delay expanding the logic of this until
14270 pass_expand_omp. The expanded logic may make the job more difficult
14271 to a synchronization analysis pass. */
14274 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
14276 location_t loc
= gimple_location (single_stmt
);
14277 tree tlabel
= create_artificial_label (loc
);
14278 tree flabel
= create_artificial_label (loc
);
14279 gimple
*call
, *cond
;
14282 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
14283 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
14284 call
= gimple_build_call (decl
, 0);
14285 gimple_call_set_lhs (call
, lhs
);
14286 gimple_seq_add_stmt (pre_p
, call
);
14288 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
14289 fold_convert_loc (loc
, TREE_TYPE (lhs
),
14290 boolean_true_node
),
14292 gimple_seq_add_stmt (pre_p
, cond
);
14293 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
14294 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14295 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
14299 /* A subroutine of lower_omp_single. Expand the simple form of
14300 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14302 #pragma omp single copyprivate (a, b, c)
14304 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14307 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14313 GOMP_single_copy_end (©out);
14324 FIXME. It may be better to delay expanding the logic of this until
14325 pass_expand_omp. The expanded logic may make the job more difficult
14326 to a synchronization analysis pass. */
14329 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
14332 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
14333 gimple_seq copyin_seq
;
14334 location_t loc
= gimple_location (single_stmt
);
14336 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
14338 ptr_type
= build_pointer_type (ctx
->record_type
);
14339 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
14341 l0
= create_artificial_label (loc
);
14342 l1
= create_artificial_label (loc
);
14343 l2
= create_artificial_label (loc
);
14345 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
14346 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
14347 t
= fold_convert_loc (loc
, ptr_type
, t
);
14348 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
14350 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
14351 build_int_cst (ptr_type
, 0));
14352 t
= build3 (COND_EXPR
, void_type_node
, t
,
14353 build_and_jump (&l0
), build_and_jump (&l1
));
14354 gimplify_and_add (t
, pre_p
);
14356 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
14358 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14361 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
14364 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14365 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
14366 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
14367 gimplify_and_add (t
, pre_p
);
14369 t
= build_and_jump (&l2
);
14370 gimplify_and_add (t
, pre_p
);
14372 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
14374 gimple_seq_add_seq (pre_p
, copyin_seq
);
14376 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
14380 /* Expand code for an OpenMP single directive. */
14383 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14387 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
14389 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
14391 push_gimplify_context ();
14393 block
= make_node (BLOCK
);
14394 bind
= gimple_build_bind (NULL
, NULL
, block
);
14395 gsi_replace (gsi_p
, bind
, true);
14398 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
14399 &bind_body
, &dlist
, ctx
, NULL
);
14400 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
14402 gimple_seq_add_stmt (&bind_body
, single_stmt
);
14404 if (ctx
->record_type
)
14405 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
14407 lower_omp_single_simple (single_stmt
, &bind_body
);
14409 gimple_omp_set_body (single_stmt
, NULL
);
14411 gimple_seq_add_seq (&bind_body
, dlist
);
14413 bind_body
= maybe_catch_exception (bind_body
);
14415 t
= gimple_build_omp_return
14416 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
14417 OMP_CLAUSE_NOWAIT
));
14418 gimple_seq_add_stmt (&bind_body_tail
, t
);
14419 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
14420 if (ctx
->record_type
)
14422 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
14423 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14424 TREE_THIS_VOLATILE (clobber
) = 1;
14425 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
14426 clobber
), GSI_SAME_STMT
);
14428 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
14429 gimple_bind_set_body (bind
, bind_body
);
14431 pop_gimplify_context (bind
);
14433 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14434 BLOCK_VARS (block
) = ctx
->block_vars
;
14435 if (BLOCK_VARS (block
))
14436 TREE_USED (block
) = 1;
14440 /* Expand code for an OpenMP master directive. */
14443 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14445 tree block
, lab
= NULL
, x
, bfn_decl
;
14446 gimple
*stmt
= gsi_stmt (*gsi_p
);
14448 location_t loc
= gimple_location (stmt
);
14451 push_gimplify_context ();
14453 block
= make_node (BLOCK
);
14454 bind
= gimple_build_bind (NULL
, NULL
, block
);
14455 gsi_replace (gsi_p
, bind
, true);
14456 gimple_bind_add_stmt (bind
, stmt
);
14458 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
14459 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
14460 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
14461 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
14463 gimplify_and_add (x
, &tseq
);
14464 gimple_bind_add_seq (bind
, tseq
);
14466 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14467 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14468 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14469 gimple_omp_set_body (stmt
, NULL
);
14471 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
14473 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14475 pop_gimplify_context (bind
);
14477 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14478 BLOCK_VARS (block
) = ctx
->block_vars
;
14482 /* Expand code for an OpenMP taskgroup directive. */
14485 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14487 gimple
*stmt
= gsi_stmt (*gsi_p
);
14490 tree block
= make_node (BLOCK
);
14492 bind
= gimple_build_bind (NULL
, NULL
, block
);
14493 gsi_replace (gsi_p
, bind
, true);
14494 gimple_bind_add_stmt (bind
, stmt
);
14496 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
14498 gimple_bind_add_stmt (bind
, x
);
14500 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14501 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14502 gimple_omp_set_body (stmt
, NULL
);
14504 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14506 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14507 BLOCK_VARS (block
) = ctx
->block_vars
;
14511 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14514 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
14517 struct omp_for_data fd
;
14518 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
14521 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
14522 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
14523 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
14527 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14528 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
14529 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14530 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
14532 /* Merge depend clauses from multiple adjacent
14533 #pragma omp ordered depend(sink:...) constructs
14534 into one #pragma omp ordered depend(sink:...), so that
14535 we can optimize them together. */
14536 gimple_stmt_iterator gsi
= *gsi_p
;
14538 while (!gsi_end_p (gsi
))
14540 gimple
*stmt
= gsi_stmt (gsi
);
14541 if (is_gimple_debug (stmt
)
14542 || gimple_code (stmt
) == GIMPLE_NOP
)
14547 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
14549 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
14550 c
= gimple_omp_ordered_clauses (ord_stmt2
);
14552 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
14553 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14556 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
14558 gsi_remove (&gsi
, true);
14562 /* Canonicalize sink dependence clauses into one folded clause if
14565 The basic algorithm is to create a sink vector whose first
14566 element is the GCD of all the first elements, and whose remaining
14567 elements are the minimum of the subsequent columns.
14569 We ignore dependence vectors whose first element is zero because
14570 such dependencies are known to be executed by the same thread.
14572 We take into account the direction of the loop, so a minimum
14573 becomes a maximum if the loop is iterating forwards. We also
14574 ignore sink clauses where the loop direction is unknown, or where
14575 the offsets are clearly invalid because they are not a multiple
14576 of the loop increment.
14580 #pragma omp for ordered(2)
14581 for (i=0; i < N; ++i)
14582 for (j=0; j < M; ++j)
14584 #pragma omp ordered \
14585 depend(sink:i-8,j-2) \
14586 depend(sink:i,j-1) \ // Completely ignored because i+0.
14587 depend(sink:i-4,j-3) \
14588 depend(sink:i-6,j-4)
14589 #pragma omp ordered depend(source)
14594 depend(sink:-gcd(8,4,6),-min(2,3,4))
14599 /* FIXME: Computing GCD's where the first element is zero is
14600 non-trivial in the presence of collapsed loops. Do this later. */
14601 if (fd
.collapse
> 1)
14604 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
14605 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
14606 tree folded_dep
= NULL_TREE
;
14607 /* TRUE if the first dimension's offset is negative. */
14608 bool neg_offset_p
= false;
14610 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14612 while ((c
= *list_p
) != NULL
)
14614 bool remove
= false;
14616 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
14617 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14618 goto next_ordered_clause
;
14621 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
14622 vec
&& TREE_CODE (vec
) == TREE_LIST
;
14623 vec
= TREE_CHAIN (vec
), ++i
)
14625 gcc_assert (i
< len
);
14627 /* extract_omp_for_data has canonicalized the condition. */
14628 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
14629 || fd
.loops
[i
].cond_code
== GT_EXPR
);
14630 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
14631 bool maybe_lexically_later
= true;
14633 /* While the committee makes up its mind, bail if we have any
14634 non-constant steps. */
14635 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
14636 goto lower_omp_ordered_ret
;
14638 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
14639 if (POINTER_TYPE_P (itype
))
14641 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
14642 TYPE_PRECISION (itype
),
14643 TYPE_SIGN (itype
));
14645 /* Ignore invalid offsets that are not multiples of the step. */
14646 if (!wi::multiple_of_p
14647 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
14650 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
14651 "ignoring sink clause with offset that is not "
14652 "a multiple of the loop step");
14654 goto next_ordered_clause
;
14657 /* Calculate the first dimension. The first dimension of
14658 the folded dependency vector is the GCD of the first
14659 elements, while ignoring any first elements whose offset
14663 /* Ignore dependence vectors whose first dimension is 0. */
14667 goto next_ordered_clause
;
14671 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
14673 error_at (OMP_CLAUSE_LOCATION (c
),
14674 "first offset must be in opposite direction "
14675 "of loop iterations");
14676 goto lower_omp_ordered_ret
;
14680 neg_offset_p
= forward
;
14681 /* Initialize the first time around. */
14682 if (folded_dep
== NULL_TREE
)
14685 folded_deps
[0] = offset
;
14688 folded_deps
[0] = wi::gcd (folded_deps
[0],
14692 /* Calculate minimum for the remaining dimensions. */
14695 folded_deps
[len
+ i
- 1] = offset
;
14696 if (folded_dep
== c
)
14697 folded_deps
[i
] = offset
;
14698 else if (maybe_lexically_later
14699 && !wi::eq_p (folded_deps
[i
], offset
))
14701 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
14705 for (j
= 1; j
<= i
; j
++)
14706 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
14709 maybe_lexically_later
= false;
14713 gcc_assert (i
== len
);
14717 next_ordered_clause
:
14719 *list_p
= OMP_CLAUSE_CHAIN (c
);
14721 list_p
= &OMP_CLAUSE_CHAIN (c
);
14727 folded_deps
[0] = -folded_deps
[0];
14729 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
14730 if (POINTER_TYPE_P (itype
))
14733 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14734 = wide_int_to_tree (itype
, folded_deps
[0]);
14735 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14736 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14739 lower_omp_ordered_ret
:
14741 /* Ordered without clauses is #pragma omp threads, while we want
14742 a nop instead if we remove all clauses. */
14743 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14744 gsi_replace (gsi_p
, gimple_build_nop (), true);
14748 /* Expand code for an OpenMP ordered directive. */
14751 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14754 gimple
*stmt
= gsi_stmt (*gsi_p
);
14755 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14758 bool simd
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14760 bool threads
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14761 OMP_CLAUSE_THREADS
);
14763 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14764 OMP_CLAUSE_DEPEND
))
14766 /* FIXME: This is needs to be moved to the expansion to verify various
14767 conditions only testable on cfg with dominators computed, and also
14768 all the depend clauses to be merged still might need to be available
14769 for the runtime checks. */
14771 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14775 push_gimplify_context ();
14777 block
= make_node (BLOCK
);
14778 bind
= gimple_build_bind (NULL
, NULL
, block
);
14779 gsi_replace (gsi_p
, bind
, true);
14780 gimple_bind_add_stmt (bind
, stmt
);
14784 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
14785 build_int_cst (NULL_TREE
, threads
));
14786 cfun
->has_simduid_loops
= true;
14789 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14791 gimple_bind_add_stmt (bind
, x
);
14793 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14794 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14795 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14796 gimple_omp_set_body (stmt
, NULL
);
14799 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
14800 build_int_cst (NULL_TREE
, threads
));
14802 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14804 gimple_bind_add_stmt (bind
, x
);
14806 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14808 pop_gimplify_context (bind
);
14810 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14811 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14815 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14816 substitution of a couple of function calls. But in the NAMED case,
14817 requires that languages coordinate a symbol name. It is therefore
14818 best put here in common code. */
14820 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14823 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14826 tree name
, lock
, unlock
;
14827 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14829 location_t loc
= gimple_location (stmt
);
14832 name
= gimple_omp_critical_name (stmt
);
14837 if (!critical_name_mutexes
)
14838 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14840 tree
*n
= critical_name_mutexes
->get (name
);
14845 decl
= create_tmp_var_raw (ptr_type_node
);
14847 new_str
= ACONCAT ((".gomp_critical_user_",
14848 IDENTIFIER_POINTER (name
), NULL
));
14849 DECL_NAME (decl
) = get_identifier (new_str
);
14850 TREE_PUBLIC (decl
) = 1;
14851 TREE_STATIC (decl
) = 1;
14852 DECL_COMMON (decl
) = 1;
14853 DECL_ARTIFICIAL (decl
) = 1;
14854 DECL_IGNORED_P (decl
) = 1;
14856 varpool_node::finalize_decl (decl
);
14858 critical_name_mutexes
->put (name
, decl
);
14863 /* If '#pragma omp critical' is inside offloaded region or
14864 inside function marked as offloadable, the symbol must be
14865 marked as offloadable too. */
14867 if (cgraph_node::get (current_function_decl
)->offloadable
)
14868 varpool_node::get_create (decl
)->offloadable
= 1;
14870 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14871 if (is_gimple_omp_offloaded (octx
->stmt
))
14873 varpool_node::get_create (decl
)->offloadable
= 1;
14877 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14878 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14880 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14881 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14882 build_fold_addr_expr_loc (loc
, decl
));
14886 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14887 lock
= build_call_expr_loc (loc
, lock
, 0);
14889 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14890 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14893 push_gimplify_context ();
14895 block
= make_node (BLOCK
);
14896 bind
= gimple_build_bind (NULL
, NULL
, block
);
14897 gsi_replace (gsi_p
, bind
, true);
14898 gimple_bind_add_stmt (bind
, stmt
);
14900 tbody
= gimple_bind_body (bind
);
14901 gimplify_and_add (lock
, &tbody
);
14902 gimple_bind_set_body (bind
, tbody
);
14904 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14905 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14906 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14907 gimple_omp_set_body (stmt
, NULL
);
14909 tbody
= gimple_bind_body (bind
);
14910 gimplify_and_add (unlock
, &tbody
);
14911 gimple_bind_set_body (bind
, tbody
);
14913 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14915 pop_gimplify_context (bind
);
14916 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14917 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14921 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14922 for a lastprivate clause. Given a loop control predicate of (V
14923 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14924 is appended to *DLIST, iterator initialization is appended to
14928 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14929 gimple_seq
*dlist
, struct omp_context
*ctx
)
14931 tree clauses
, cond
, vinit
;
14932 enum tree_code cond_code
;
14935 cond_code
= fd
->loop
.cond_code
;
14936 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14938 /* When possible, use a strict equality expression. This can let VRP
14939 type optimizations deduce the value and remove a copy. */
14940 if (tree_fits_shwi_p (fd
->loop
.step
))
14942 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14943 if (step
== 1 || step
== -1)
14944 cond_code
= EQ_EXPR
;
14947 tree n2
= fd
->loop
.n2
;
14948 if (fd
->collapse
> 1
14949 && TREE_CODE (n2
) != INTEGER_CST
14950 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14952 struct omp_context
*taskreg_ctx
= NULL
;
14953 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14955 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14956 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14957 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14959 if (gimple_omp_for_combined_into_p (gfor
))
14961 gcc_assert (ctx
->outer
->outer
14962 && is_parallel_ctx (ctx
->outer
->outer
));
14963 taskreg_ctx
= ctx
->outer
->outer
;
14967 struct omp_for_data outer_fd
;
14968 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14969 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14972 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14973 taskreg_ctx
= ctx
->outer
->outer
;
14975 else if (is_taskreg_ctx (ctx
->outer
))
14976 taskreg_ctx
= ctx
->outer
;
14981 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14982 OMP_CLAUSE__LOOPTEMP_
);
14983 gcc_assert (innerc
);
14984 for (i
= 0; i
< fd
->collapse
; i
++)
14986 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14987 OMP_CLAUSE__LOOPTEMP_
);
14988 gcc_assert (innerc
);
14990 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14991 OMP_CLAUSE__LOOPTEMP_
);
14993 n2
= fold_convert (TREE_TYPE (n2
),
14994 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14998 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
15000 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
15002 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
15003 if (!gimple_seq_empty_p (stmts
))
15005 gimple_seq_add_seq (&stmts
, *dlist
);
15008 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
15009 vinit
= fd
->loop
.n1
;
15010 if (cond_code
== EQ_EXPR
15011 && tree_fits_shwi_p (fd
->loop
.n2
)
15012 && ! integer_zerop (fd
->loop
.n2
))
15013 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
15015 vinit
= unshare_expr (vinit
);
15017 /* Initialize the iterator variable, so that threads that don't execute
15018 any iterations don't execute the lastprivate clauses by accident. */
15019 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
15024 /* Lower code for an OMP loop directive. */
15027 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15029 tree
*rhs_p
, block
;
15030 struct omp_for_data fd
, *fdp
= NULL
;
15031 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
15033 gimple_seq omp_for_body
, body
, dlist
;
15034 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
15037 push_gimplify_context ();
15039 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
15041 block
= make_node (BLOCK
);
15042 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
15043 /* Replace at gsi right away, so that 'stmt' is no member
15044 of a sequence anymore as we're going to add to a different
15046 gsi_replace (gsi_p
, new_stmt
, true);
15048 /* Move declaration of temporaries in the loop body before we make
15050 omp_for_body
= gimple_omp_body (stmt
);
15051 if (!gimple_seq_empty_p (omp_for_body
)
15052 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
15055 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
15056 tree vars
= gimple_bind_vars (inner_bind
);
15057 gimple_bind_append_vars (new_stmt
, vars
);
15058 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
15059 keep them on the inner_bind and it's block. */
15060 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
15061 if (gimple_bind_block (inner_bind
))
15062 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
15065 if (gimple_omp_for_combined_into_p (stmt
))
15067 extract_omp_for_data (stmt
, &fd
, NULL
);
15070 /* We need two temporaries with fd.loop.v type (istart/iend)
15071 and then (fd.collapse - 1) temporaries with the same
15072 type for count2 ... countN-1 vars if not constant. */
15074 tree type
= fd
.iter_type
;
15075 if (fd
.collapse
> 1
15076 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
15077 count
+= fd
.collapse
- 1;
15079 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
15080 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
15081 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
15082 tree clauses
= *pc
;
15085 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
15086 OMP_CLAUSE__LOOPTEMP_
);
15087 for (i
= 0; i
< count
; i
++)
15092 gcc_assert (outerc
);
15093 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
15094 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
15095 OMP_CLAUSE__LOOPTEMP_
);
15099 temp
= create_tmp_var (type
);
15100 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
15102 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
15103 OMP_CLAUSE_DECL (*pc
) = temp
;
15104 pc
= &OMP_CLAUSE_CHAIN (*pc
);
15109 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15112 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
15114 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
15116 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
15118 /* Lower the header expressions. At this point, we can assume that
15119 the header is of the form:
15121 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15123 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15124 using the .omp_data_s mapping, if needed. */
15125 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
15127 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
15128 if (!is_gimple_min_invariant (*rhs_p
))
15129 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15131 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
15132 if (!is_gimple_min_invariant (*rhs_p
))
15133 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15135 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
15136 if (!is_gimple_min_invariant (*rhs_p
))
15137 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15140 /* Once lowered, extract the bounds and clauses. */
15141 extract_omp_for_data (stmt
, &fd
, NULL
);
15143 if (is_gimple_omp_oacc (ctx
->stmt
)
15144 && !ctx_in_oacc_kernels_region (ctx
))
15145 lower_oacc_head_tail (gimple_location (stmt
),
15146 gimple_omp_for_clauses (stmt
),
15147 &oacc_head
, &oacc_tail
, ctx
);
15149 /* Add OpenACC partitioning and reduction markers just before the loop */
15151 gimple_seq_add_seq (&body
, oacc_head
);
15153 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
15155 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
15156 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15157 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
15158 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
15160 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
15161 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
15162 OMP_CLAUSE_LINEAR_STEP (c
)
15163 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
15167 if (!gimple_omp_for_grid_phony (stmt
))
15168 gimple_seq_add_stmt (&body
, stmt
);
15169 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
15171 if (!gimple_omp_for_grid_phony (stmt
))
15172 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
15175 /* After the loop, add exit clauses. */
15176 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
15178 if (ctx
->cancellable
)
15179 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
15181 gimple_seq_add_seq (&body
, dlist
);
15183 body
= maybe_catch_exception (body
);
15185 if (!gimple_omp_for_grid_phony (stmt
))
15187 /* Region exit marker goes at the end of the loop body. */
15188 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
15189 maybe_add_implicit_barrier_cancel (ctx
, &body
);
15192 /* Add OpenACC joining and reduction markers just after the loop. */
15194 gimple_seq_add_seq (&body
, oacc_tail
);
15196 pop_gimplify_context (new_stmt
);
15198 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
15199 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
15200 if (BLOCK_VARS (block
))
15201 TREE_USED (block
) = 1;
15203 gimple_bind_set_body (new_stmt
, body
);
15204 gimple_omp_set_body (stmt
, NULL
);
15205 gimple_omp_for_set_pre_body (stmt
, NULL
);
15208 /* Callback for walk_stmts. Check if the current statement only contains
15209 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15212 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
15213 bool *handled_ops_p
,
15214 struct walk_stmt_info
*wi
)
15216 int *info
= (int *) wi
->info
;
15217 gimple
*stmt
= gsi_stmt (*gsi_p
);
15219 *handled_ops_p
= true;
15220 switch (gimple_code (stmt
))
15224 case GIMPLE_OMP_FOR
:
15225 case GIMPLE_OMP_SECTIONS
:
15226 *info
= *info
== 0 ? 1 : -1;
15235 struct omp_taskcopy_context
15237 /* This field must be at the beginning, as we do "inheritance": Some
15238 callback functions for tree-inline.c (e.g., omp_copy_decl)
15239 receive a copy_body_data pointer that is up-casted to an
15240 omp_context pointer. */
15246 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
15248 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
15250 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
15251 return create_tmp_var (TREE_TYPE (var
));
15257 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
15259 tree name
, new_fields
= NULL
, type
, f
;
15261 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
15262 name
= DECL_NAME (TYPE_NAME (orig_type
));
15263 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
15264 TYPE_DECL
, name
, type
);
15265 TYPE_NAME (type
) = name
;
15267 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
15269 tree new_f
= copy_node (f
);
15270 DECL_CONTEXT (new_f
) = type
;
15271 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
15272 TREE_CHAIN (new_f
) = new_fields
;
15273 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15274 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15275 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
15277 new_fields
= new_f
;
15278 tcctx
->cb
.decl_map
->put (f
, new_f
);
15280 TYPE_FIELDS (type
) = nreverse (new_fields
);
15281 layout_type (type
);
15285 /* Create task copyfn. */
15288 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
15290 struct function
*child_cfun
;
15291 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
15292 tree record_type
, srecord_type
, bind
, list
;
15293 bool record_needs_remap
= false, srecord_needs_remap
= false;
15295 struct omp_taskcopy_context tcctx
;
15296 location_t loc
= gimple_location (task_stmt
);
15298 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
15299 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
15300 gcc_assert (child_cfun
->cfg
== NULL
);
15301 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
15303 /* Reset DECL_CONTEXT on function arguments. */
15304 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
15305 DECL_CONTEXT (t
) = child_fn
;
15307 /* Populate the function. */
15308 push_gimplify_context ();
15309 push_cfun (child_cfun
);
15311 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
15312 TREE_SIDE_EFFECTS (bind
) = 1;
15314 DECL_SAVED_TREE (child_fn
) = bind
;
15315 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
15317 /* Remap src and dst argument types if needed. */
15318 record_type
= ctx
->record_type
;
15319 srecord_type
= ctx
->srecord_type
;
15320 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
15321 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15323 record_needs_remap
= true;
15326 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
15327 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15329 srecord_needs_remap
= true;
15333 if (record_needs_remap
|| srecord_needs_remap
)
15335 memset (&tcctx
, '\0', sizeof (tcctx
));
15336 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
15337 tcctx
.cb
.dst_fn
= child_fn
;
15338 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
15339 gcc_checking_assert (tcctx
.cb
.src_node
);
15340 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
15341 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
15342 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
15343 tcctx
.cb
.eh_lp_nr
= 0;
15344 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
15345 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
15348 if (record_needs_remap
)
15349 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
15350 if (srecord_needs_remap
)
15351 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
15354 tcctx
.cb
.decl_map
= NULL
;
15356 arg
= DECL_ARGUMENTS (child_fn
);
15357 TREE_TYPE (arg
) = build_pointer_type (record_type
);
15358 sarg
= DECL_CHAIN (arg
);
15359 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
15361 /* First pass: initialize temporaries used in record_type and srecord_type
15362 sizes and field offsets. */
15363 if (tcctx
.cb
.decl_map
)
15364 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15365 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15369 decl
= OMP_CLAUSE_DECL (c
);
15370 p
= tcctx
.cb
.decl_map
->get (decl
);
15373 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15374 sf
= (tree
) n
->value
;
15375 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15376 src
= build_simple_mem_ref_loc (loc
, sarg
);
15377 src
= omp_build_component_ref (src
, sf
);
15378 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
15379 append_to_statement_list (t
, &list
);
15382 /* Second pass: copy shared var pointers and copy construct non-VLA
15383 firstprivate vars. */
15384 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15385 switch (OMP_CLAUSE_CODE (c
))
15387 splay_tree_key key
;
15388 case OMP_CLAUSE_SHARED
:
15389 decl
= OMP_CLAUSE_DECL (c
);
15390 key
= (splay_tree_key
) decl
;
15391 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
15392 key
= (splay_tree_key
) &DECL_UID (decl
);
15393 n
= splay_tree_lookup (ctx
->field_map
, key
);
15396 f
= (tree
) n
->value
;
15397 if (tcctx
.cb
.decl_map
)
15398 f
= *tcctx
.cb
.decl_map
->get (f
);
15399 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
15400 sf
= (tree
) n
->value
;
15401 if (tcctx
.cb
.decl_map
)
15402 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15403 src
= build_simple_mem_ref_loc (loc
, sarg
);
15404 src
= omp_build_component_ref (src
, sf
);
15405 dst
= build_simple_mem_ref_loc (loc
, arg
);
15406 dst
= omp_build_component_ref (dst
, f
);
15407 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15408 append_to_statement_list (t
, &list
);
15410 case OMP_CLAUSE_FIRSTPRIVATE
:
15411 decl
= OMP_CLAUSE_DECL (c
);
15412 if (is_variable_sized (decl
))
15414 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15417 f
= (tree
) n
->value
;
15418 if (tcctx
.cb
.decl_map
)
15419 f
= *tcctx
.cb
.decl_map
->get (f
);
15420 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15423 sf
= (tree
) n
->value
;
15424 if (tcctx
.cb
.decl_map
)
15425 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15426 src
= build_simple_mem_ref_loc (loc
, sarg
);
15427 src
= omp_build_component_ref (src
, sf
);
15428 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
15429 src
= build_simple_mem_ref_loc (loc
, src
);
15433 dst
= build_simple_mem_ref_loc (loc
, arg
);
15434 dst
= omp_build_component_ref (dst
, f
);
15435 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15436 append_to_statement_list (t
, &list
);
15438 case OMP_CLAUSE_PRIVATE
:
15439 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
15441 decl
= OMP_CLAUSE_DECL (c
);
15442 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15443 f
= (tree
) n
->value
;
15444 if (tcctx
.cb
.decl_map
)
15445 f
= *tcctx
.cb
.decl_map
->get (f
);
15446 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15449 sf
= (tree
) n
->value
;
15450 if (tcctx
.cb
.decl_map
)
15451 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15452 src
= build_simple_mem_ref_loc (loc
, sarg
);
15453 src
= omp_build_component_ref (src
, sf
);
15454 if (use_pointer_for_field (decl
, NULL
))
15455 src
= build_simple_mem_ref_loc (loc
, src
);
15459 dst
= build_simple_mem_ref_loc (loc
, arg
);
15460 dst
= omp_build_component_ref (dst
, f
);
15461 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15462 append_to_statement_list (t
, &list
);
15468 /* Last pass: handle VLA firstprivates. */
15469 if (tcctx
.cb
.decl_map
)
15470 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15471 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15475 decl
= OMP_CLAUSE_DECL (c
);
15476 if (!is_variable_sized (decl
))
15478 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15481 f
= (tree
) n
->value
;
15482 f
= *tcctx
.cb
.decl_map
->get (f
);
15483 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
15484 ind
= DECL_VALUE_EXPR (decl
);
15485 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
15486 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
15487 n
= splay_tree_lookup (ctx
->sfield_map
,
15488 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15489 sf
= (tree
) n
->value
;
15490 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15491 src
= build_simple_mem_ref_loc (loc
, sarg
);
15492 src
= omp_build_component_ref (src
, sf
);
15493 src
= build_simple_mem_ref_loc (loc
, src
);
15494 dst
= build_simple_mem_ref_loc (loc
, arg
);
15495 dst
= omp_build_component_ref (dst
, f
);
15496 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15497 append_to_statement_list (t
, &list
);
15498 n
= splay_tree_lookup (ctx
->field_map
,
15499 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15500 df
= (tree
) n
->value
;
15501 df
= *tcctx
.cb
.decl_map
->get (df
);
15502 ptr
= build_simple_mem_ref_loc (loc
, arg
);
15503 ptr
= omp_build_component_ref (ptr
, df
);
15504 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
15505 build_fold_addr_expr_loc (loc
, dst
));
15506 append_to_statement_list (t
, &list
);
15509 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
15510 append_to_statement_list (t
, &list
);
15512 if (tcctx
.cb
.decl_map
)
15513 delete tcctx
.cb
.decl_map
;
15514 pop_gimplify_context (NULL
);
15515 BIND_EXPR_BODY (bind
) = list
;
15520 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
15524 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
15526 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
15527 gcc_assert (clauses
);
15528 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15529 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
15530 switch (OMP_CLAUSE_DEPEND_KIND (c
))
15532 case OMP_CLAUSE_DEPEND_IN
:
15535 case OMP_CLAUSE_DEPEND_OUT
:
15536 case OMP_CLAUSE_DEPEND_INOUT
:
15539 case OMP_CLAUSE_DEPEND_SOURCE
:
15540 case OMP_CLAUSE_DEPEND_SINK
:
15543 gcc_unreachable ();
15545 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
15546 tree array
= create_tmp_var (type
);
15547 TREE_ADDRESSABLE (array
) = 1;
15548 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
15550 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
15551 gimple_seq_add_stmt (iseq
, g
);
15552 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
15554 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
15555 gimple_seq_add_stmt (iseq
, g
);
15556 for (i
= 0; i
< 2; i
++)
15558 if ((i
? n_in
: n_out
) == 0)
15560 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15561 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
15562 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
15564 tree t
= OMP_CLAUSE_DECL (c
);
15565 t
= fold_convert (ptr_type_node
, t
);
15566 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
15567 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
15568 NULL_TREE
, NULL_TREE
);
15569 g
= gimple_build_assign (r
, t
);
15570 gimple_seq_add_stmt (iseq
, g
);
15573 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
15574 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
15575 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
15577 tree clobber
= build_constructor (type
, NULL
);
15578 TREE_THIS_VOLATILE (clobber
) = 1;
15579 g
= gimple_build_assign (array
, clobber
);
15580 gimple_seq_add_stmt (oseq
, g
);
15583 /* Lower the OpenMP parallel or task directive in the current statement
15584 in GSI_P. CTX holds context information for the directive. */
15587 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15591 gimple
*stmt
= gsi_stmt (*gsi_p
);
15592 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
15593 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
15594 location_t loc
= gimple_location (stmt
);
15596 clauses
= gimple_omp_taskreg_clauses (stmt
);
15598 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
15599 par_body
= gimple_bind_body (par_bind
);
15600 child_fn
= ctx
->cb
.dst_fn
;
15601 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15602 && !gimple_omp_parallel_combined_p (stmt
))
15604 struct walk_stmt_info wi
;
15607 memset (&wi
, 0, sizeof (wi
));
15609 wi
.val_only
= true;
15610 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
15612 gimple_omp_parallel_set_combined_p (stmt
, true);
15614 gimple_seq dep_ilist
= NULL
;
15615 gimple_seq dep_olist
= NULL
;
15616 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
15617 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15619 push_gimplify_context ();
15620 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15621 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
15622 &dep_ilist
, &dep_olist
);
15625 if (ctx
->srecord_type
)
15626 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
15628 push_gimplify_context ();
15633 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15634 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
15635 if (phony_construct
&& ctx
->record_type
)
15637 gcc_checking_assert (!ctx
->receiver_decl
);
15638 ctx
->receiver_decl
= create_tmp_var
15639 (build_reference_type (ctx
->record_type
), ".omp_rec");
15641 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
15642 lower_omp (&par_body
, ctx
);
15643 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
15644 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
15646 /* Declare all the variables created by mapping and the variables
15647 declared in the scope of the parallel body. */
15648 record_vars_into (ctx
->block_vars
, child_fn
);
15649 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
15651 if (ctx
->record_type
)
15654 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
15655 : ctx
->record_type
, ".omp_data_o");
15656 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15657 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15658 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
15663 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
15664 lower_send_shared_vars (&ilist
, &olist
, ctx
);
15666 if (ctx
->record_type
)
15668 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
15669 TREE_THIS_VOLATILE (clobber
) = 1;
15670 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15674 /* Once all the expansions are done, sequence all the different
15675 fragments inside gimple_omp_body. */
15679 if (ctx
->record_type
)
15681 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15682 /* fixup_child_record_type might have changed receiver_decl's type. */
15683 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15684 gimple_seq_add_stmt (&new_body
,
15685 gimple_build_assign (ctx
->receiver_decl
, t
));
15688 gimple_seq_add_seq (&new_body
, par_ilist
);
15689 gimple_seq_add_seq (&new_body
, par_body
);
15690 gimple_seq_add_seq (&new_body
, par_rlist
);
15691 if (ctx
->cancellable
)
15692 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
15693 gimple_seq_add_seq (&new_body
, par_olist
);
15694 new_body
= maybe_catch_exception (new_body
);
15695 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
15696 gimple_seq_add_stmt (&new_body
,
15697 gimple_build_omp_continue (integer_zero_node
,
15698 integer_zero_node
));
15699 if (!phony_construct
)
15701 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15702 gimple_omp_set_body (stmt
, new_body
);
15705 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
15706 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15707 gimple_bind_add_seq (bind
, ilist
);
15708 if (!phony_construct
)
15709 gimple_bind_add_stmt (bind
, stmt
);
15711 gimple_bind_add_seq (bind
, new_body
);
15712 gimple_bind_add_seq (bind
, olist
);
15714 pop_gimplify_context (NULL
);
15718 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15719 gimple_bind_add_stmt (dep_bind
, bind
);
15720 gimple_bind_add_seq (dep_bind
, dep_olist
);
15721 pop_gimplify_context (dep_bind
);
15725 /* Lower the GIMPLE_OMP_TARGET in the current statement
15726 in GSI_P. CTX holds context information for the directive. */
15729 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15732 tree child_fn
, t
, c
;
15733 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
15734 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
15735 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
15736 location_t loc
= gimple_location (stmt
);
15737 bool offloaded
, data_region
;
15738 unsigned int map_cnt
= 0;
15740 offloaded
= is_gimple_omp_offloaded (stmt
);
15741 switch (gimple_omp_target_kind (stmt
))
15743 case GF_OMP_TARGET_KIND_REGION
:
15744 case GF_OMP_TARGET_KIND_UPDATE
:
15745 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15746 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15747 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15748 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15749 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15750 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15751 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
15752 data_region
= false;
15754 case GF_OMP_TARGET_KIND_DATA
:
15755 case GF_OMP_TARGET_KIND_OACC_DATA
:
15756 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
15757 data_region
= true;
15760 gcc_unreachable ();
15763 clauses
= gimple_omp_target_clauses (stmt
);
15765 gimple_seq dep_ilist
= NULL
;
15766 gimple_seq dep_olist
= NULL
;
15767 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15769 push_gimplify_context ();
15770 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15771 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15772 &dep_ilist
, &dep_olist
);
15779 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15780 tgt_body
= gimple_bind_body (tgt_bind
);
15782 else if (data_region
)
15783 tgt_body
= gimple_omp_body (stmt
);
15784 child_fn
= ctx
->cb
.dst_fn
;
15786 push_gimplify_context ();
15789 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15790 switch (OMP_CLAUSE_CODE (c
))
15796 case OMP_CLAUSE_MAP
:
15798 /* First check what we're prepared to handle in the following. */
15799 switch (OMP_CLAUSE_MAP_KIND (c
))
15801 case GOMP_MAP_ALLOC
:
15803 case GOMP_MAP_FROM
:
15804 case GOMP_MAP_TOFROM
:
15805 case GOMP_MAP_POINTER
:
15806 case GOMP_MAP_TO_PSET
:
15807 case GOMP_MAP_DELETE
:
15808 case GOMP_MAP_RELEASE
:
15809 case GOMP_MAP_ALWAYS_TO
:
15810 case GOMP_MAP_ALWAYS_FROM
:
15811 case GOMP_MAP_ALWAYS_TOFROM
:
15812 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15813 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
15814 case GOMP_MAP_STRUCT
:
15815 case GOMP_MAP_ALWAYS_POINTER
:
15817 case GOMP_MAP_FORCE_ALLOC
:
15818 case GOMP_MAP_FORCE_TO
:
15819 case GOMP_MAP_FORCE_FROM
:
15820 case GOMP_MAP_FORCE_TOFROM
:
15821 case GOMP_MAP_FORCE_PRESENT
:
15822 case GOMP_MAP_FORCE_DEVICEPTR
:
15823 case GOMP_MAP_DEVICE_RESIDENT
:
15824 case GOMP_MAP_LINK
:
15825 gcc_assert (is_gimple_omp_oacc (stmt
));
15828 gcc_unreachable ();
15832 case OMP_CLAUSE_TO
:
15833 case OMP_CLAUSE_FROM
:
15835 var
= OMP_CLAUSE_DECL (c
);
15838 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15839 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15840 && (OMP_CLAUSE_MAP_KIND (c
)
15841 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15846 if (DECL_SIZE (var
)
15847 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15849 tree var2
= DECL_VALUE_EXPR (var
);
15850 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15851 var2
= TREE_OPERAND (var2
, 0);
15852 gcc_assert (DECL_P (var2
));
15857 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15858 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15859 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15861 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15863 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15864 && varpool_node::get_create (var
)->offloadable
)
15867 tree type
= build_pointer_type (TREE_TYPE (var
));
15868 tree new_var
= lookup_decl (var
, ctx
);
15869 x
= create_tmp_var_raw (type
, get_name (new_var
));
15870 gimple_add_tmp_var (x
);
15871 x
= build_simple_mem_ref (x
);
15872 SET_DECL_VALUE_EXPR (new_var
, x
);
15873 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15878 if (!maybe_lookup_field (var
, ctx
))
15881 /* Don't remap oacc parallel reduction variables, because the
15882 intermediate result must be local to each gang. */
15883 if (offloaded
&& !(OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15884 && OMP_CLAUSE_MAP_IN_REDUCTION (c
)))
15886 x
= build_receiver_ref (var
, true, ctx
);
15887 tree new_var
= lookup_decl (var
, ctx
);
15889 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15890 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15891 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15892 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15893 x
= build_simple_mem_ref (x
);
15894 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15896 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15897 if (is_reference (new_var
))
15899 /* Create a local object to hold the instance
15901 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15902 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15903 tree inst
= create_tmp_var (type
, id
);
15904 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15905 x
= build_fold_addr_expr (inst
);
15907 gimplify_assign (new_var
, x
, &fplist
);
15909 else if (DECL_P (new_var
))
15911 SET_DECL_VALUE_EXPR (new_var
, x
);
15912 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15915 gcc_unreachable ();
15920 case OMP_CLAUSE_FIRSTPRIVATE
:
15921 if (is_oacc_parallel (ctx
))
15922 goto oacc_firstprivate
;
15924 var
= OMP_CLAUSE_DECL (c
);
15925 if (!is_reference (var
)
15926 && !is_gimple_reg_type (TREE_TYPE (var
)))
15928 tree new_var
= lookup_decl (var
, ctx
);
15929 if (is_variable_sized (var
))
15931 tree pvar
= DECL_VALUE_EXPR (var
);
15932 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15933 pvar
= TREE_OPERAND (pvar
, 0);
15934 gcc_assert (DECL_P (pvar
));
15935 tree new_pvar
= lookup_decl (pvar
, ctx
);
15936 x
= build_fold_indirect_ref (new_pvar
);
15937 TREE_THIS_NOTRAP (x
) = 1;
15940 x
= build_receiver_ref (var
, true, ctx
);
15941 SET_DECL_VALUE_EXPR (new_var
, x
);
15942 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15946 case OMP_CLAUSE_PRIVATE
:
15947 if (is_gimple_omp_oacc (ctx
->stmt
))
15949 var
= OMP_CLAUSE_DECL (c
);
15950 if (is_variable_sized (var
))
15952 tree new_var
= lookup_decl (var
, ctx
);
15953 tree pvar
= DECL_VALUE_EXPR (var
);
15954 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15955 pvar
= TREE_OPERAND (pvar
, 0);
15956 gcc_assert (DECL_P (pvar
));
15957 tree new_pvar
= lookup_decl (pvar
, ctx
);
15958 x
= build_fold_indirect_ref (new_pvar
);
15959 TREE_THIS_NOTRAP (x
) = 1;
15960 SET_DECL_VALUE_EXPR (new_var
, x
);
15961 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15965 case OMP_CLAUSE_USE_DEVICE_PTR
:
15966 case OMP_CLAUSE_IS_DEVICE_PTR
:
15967 var
= OMP_CLAUSE_DECL (c
);
15969 if (is_variable_sized (var
))
15971 tree new_var
= lookup_decl (var
, ctx
);
15972 tree pvar
= DECL_VALUE_EXPR (var
);
15973 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15974 pvar
= TREE_OPERAND (pvar
, 0);
15975 gcc_assert (DECL_P (pvar
));
15976 tree new_pvar
= lookup_decl (pvar
, ctx
);
15977 x
= build_fold_indirect_ref (new_pvar
);
15978 TREE_THIS_NOTRAP (x
) = 1;
15979 SET_DECL_VALUE_EXPR (new_var
, x
);
15980 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15982 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15984 tree new_var
= lookup_decl (var
, ctx
);
15985 tree type
= build_pointer_type (TREE_TYPE (var
));
15986 x
= create_tmp_var_raw (type
, get_name (new_var
));
15987 gimple_add_tmp_var (x
);
15988 x
= build_simple_mem_ref (x
);
15989 SET_DECL_VALUE_EXPR (new_var
, x
);
15990 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15994 tree new_var
= lookup_decl (var
, ctx
);
15995 x
= create_tmp_var_raw (TREE_TYPE (new_var
), get_name (new_var
));
15996 gimple_add_tmp_var (x
);
15997 SET_DECL_VALUE_EXPR (new_var
, x
);
15998 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
16005 target_nesting_level
++;
16006 lower_omp (&tgt_body
, ctx
);
16007 target_nesting_level
--;
16009 else if (data_region
)
16010 lower_omp (&tgt_body
, ctx
);
16014 /* Declare all the variables created by mapping and the variables
16015 declared in the scope of the target body. */
16016 record_vars_into (ctx
->block_vars
, child_fn
);
16017 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
16022 if (ctx
->record_type
)
16025 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
16026 DECL_NAMELESS (ctx
->sender_decl
) = 1;
16027 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
16028 t
= make_tree_vec (3);
16029 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
16030 TREE_VEC_ELT (t
, 1)
16031 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
16032 ".omp_data_sizes");
16033 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
16034 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
16035 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
16036 tree tkind_type
= short_unsigned_type_node
;
16037 int talign_shift
= 8;
16038 TREE_VEC_ELT (t
, 2)
16039 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
16040 ".omp_data_kinds");
16041 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
16042 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
16043 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
16044 gimple_omp_target_set_data_arg (stmt
, t
);
16046 vec
<constructor_elt
, va_gc
> *vsize
;
16047 vec
<constructor_elt
, va_gc
> *vkind
;
16048 vec_alloc (vsize
, map_cnt
);
16049 vec_alloc (vkind
, map_cnt
);
16050 unsigned int map_idx
= 0;
16052 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16053 switch (OMP_CLAUSE_CODE (c
))
16055 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
16056 unsigned int talign
;
16061 case OMP_CLAUSE_MAP
:
16062 case OMP_CLAUSE_TO
:
16063 case OMP_CLAUSE_FROM
:
16064 oacc_firstprivate_map
:
16066 ovar
= OMP_CLAUSE_DECL (c
);
16067 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16068 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16069 || (OMP_CLAUSE_MAP_KIND (c
)
16070 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16072 if (!DECL_P (ovar
))
16074 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16075 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
16077 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
16078 == get_base_address (ovar
));
16079 nc
= OMP_CLAUSE_CHAIN (c
);
16080 ovar
= OMP_CLAUSE_DECL (nc
);
16084 tree x
= build_sender_ref (ovar
, ctx
);
16086 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
16087 gimplify_assign (x
, v
, &ilist
);
16093 if (DECL_SIZE (ovar
)
16094 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
16096 tree ovar2
= DECL_VALUE_EXPR (ovar
);
16097 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
16098 ovar2
= TREE_OPERAND (ovar2
, 0);
16099 gcc_assert (DECL_P (ovar2
));
16102 if (!maybe_lookup_field (ovar
, ctx
))
16106 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
16107 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
16108 talign
= DECL_ALIGN_UNIT (ovar
);
16111 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16112 x
= build_sender_ref (ovar
, ctx
);
16114 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16115 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
16116 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
16117 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
16119 gcc_assert (offloaded
);
16121 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
16122 mark_addressable (avar
);
16123 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
16124 talign
= DECL_ALIGN_UNIT (avar
);
16125 avar
= build_fold_addr_expr (avar
);
16126 gimplify_assign (x
, avar
, &ilist
);
16128 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16130 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
16131 if (!is_reference (var
))
16133 if (is_gimple_reg (var
)
16134 && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16135 TREE_NO_WARNING (var
) = 1;
16136 var
= build_fold_addr_expr (var
);
16139 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16140 gimplify_assign (x
, var
, &ilist
);
16142 else if (is_gimple_reg (var
))
16144 gcc_assert (offloaded
);
16145 tree avar
= create_tmp_var (TREE_TYPE (var
));
16146 mark_addressable (avar
);
16147 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
16148 if (GOMP_MAP_COPY_TO_P (map_kind
)
16149 || map_kind
== GOMP_MAP_POINTER
16150 || map_kind
== GOMP_MAP_TO_PSET
16151 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16153 /* If we need to initialize a temporary
16154 with VAR because it is not addressable, and
16155 the variable hasn't been initialized yet, then
16156 we'll get a warning for the store to avar.
16157 Don't warn in that case, the mapping might
16159 TREE_NO_WARNING (var
) = 1;
16160 gimplify_assign (avar
, var
, &ilist
);
16162 avar
= build_fold_addr_expr (avar
);
16163 gimplify_assign (x
, avar
, &ilist
);
16164 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
16165 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16166 && !TYPE_READONLY (TREE_TYPE (var
)))
16168 x
= unshare_expr (x
);
16169 x
= build_simple_mem_ref (x
);
16170 gimplify_assign (var
, x
, &olist
);
16175 var
= build_fold_addr_expr (var
);
16176 gimplify_assign (x
, var
, &ilist
);
16180 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16182 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16183 s
= TREE_TYPE (ovar
);
16184 if (TREE_CODE (s
) == REFERENCE_TYPE
)
16186 s
= TYPE_SIZE_UNIT (s
);
16189 s
= OMP_CLAUSE_SIZE (c
);
16190 if (s
== NULL_TREE
)
16191 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16192 s
= fold_convert (size_type_node
, s
);
16193 purpose
= size_int (map_idx
++);
16194 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16195 if (TREE_CODE (s
) != INTEGER_CST
)
16196 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16198 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
16199 switch (OMP_CLAUSE_CODE (c
))
16201 case OMP_CLAUSE_MAP
:
16202 tkind
= OMP_CLAUSE_MAP_KIND (c
);
16203 tkind_zero
= tkind
;
16204 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
16207 case GOMP_MAP_ALLOC
:
16209 case GOMP_MAP_FROM
:
16210 case GOMP_MAP_TOFROM
:
16211 case GOMP_MAP_ALWAYS_TO
:
16212 case GOMP_MAP_ALWAYS_FROM
:
16213 case GOMP_MAP_ALWAYS_TOFROM
:
16214 case GOMP_MAP_RELEASE
:
16215 case GOMP_MAP_FORCE_TO
:
16216 case GOMP_MAP_FORCE_FROM
:
16217 case GOMP_MAP_FORCE_TOFROM
:
16218 case GOMP_MAP_FORCE_PRESENT
:
16219 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
16221 case GOMP_MAP_DELETE
:
16222 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
16226 if (tkind_zero
!= tkind
)
16228 if (integer_zerop (s
))
16229 tkind
= tkind_zero
;
16230 else if (integer_nonzerop (s
))
16231 tkind_zero
= tkind
;
16234 case OMP_CLAUSE_FIRSTPRIVATE
:
16235 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16236 tkind
= GOMP_MAP_TO
;
16237 tkind_zero
= tkind
;
16239 case OMP_CLAUSE_TO
:
16240 tkind
= GOMP_MAP_TO
;
16241 tkind_zero
= tkind
;
16243 case OMP_CLAUSE_FROM
:
16244 tkind
= GOMP_MAP_FROM
;
16245 tkind_zero
= tkind
;
16248 gcc_unreachable ();
16250 gcc_checking_assert (tkind
16251 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16252 gcc_checking_assert (tkind_zero
16253 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16254 talign
= ceil_log2 (talign
);
16255 tkind
|= talign
<< talign_shift
;
16256 tkind_zero
|= talign
<< talign_shift
;
16257 gcc_checking_assert (tkind
16258 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16259 gcc_checking_assert (tkind_zero
16260 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16261 if (tkind
== tkind_zero
)
16262 x
= build_int_cstu (tkind_type
, tkind
);
16265 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
16266 x
= build3 (COND_EXPR
, tkind_type
,
16267 fold_build2 (EQ_EXPR
, boolean_type_node
,
16268 unshare_expr (s
), size_zero_node
),
16269 build_int_cstu (tkind_type
, tkind_zero
),
16270 build_int_cstu (tkind_type
, tkind
));
16272 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
16277 case OMP_CLAUSE_FIRSTPRIVATE
:
16278 if (is_oacc_parallel (ctx
))
16279 goto oacc_firstprivate_map
;
16280 ovar
= OMP_CLAUSE_DECL (c
);
16281 if (is_reference (ovar
))
16282 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16284 talign
= DECL_ALIGN_UNIT (ovar
);
16285 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16286 x
= build_sender_ref (ovar
, ctx
);
16287 tkind
= GOMP_MAP_FIRSTPRIVATE
;
16288 type
= TREE_TYPE (ovar
);
16289 if (is_reference (ovar
))
16290 type
= TREE_TYPE (type
);
16291 if ((INTEGRAL_TYPE_P (type
)
16292 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16293 || TREE_CODE (type
) == POINTER_TYPE
)
16295 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16297 if (is_reference (var
))
16298 t
= build_simple_mem_ref (var
);
16299 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16300 TREE_NO_WARNING (var
) = 1;
16301 if (TREE_CODE (type
) != POINTER_TYPE
)
16302 t
= fold_convert (pointer_sized_int_node
, t
);
16303 t
= fold_convert (TREE_TYPE (x
), t
);
16304 gimplify_assign (x
, t
, &ilist
);
16306 else if (is_reference (var
))
16307 gimplify_assign (x
, var
, &ilist
);
16308 else if (is_gimple_reg (var
))
16310 tree avar
= create_tmp_var (TREE_TYPE (var
));
16311 mark_addressable (avar
);
16312 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16313 TREE_NO_WARNING (var
) = 1;
16314 gimplify_assign (avar
, var
, &ilist
);
16315 avar
= build_fold_addr_expr (avar
);
16316 gimplify_assign (x
, avar
, &ilist
);
16320 var
= build_fold_addr_expr (var
);
16321 gimplify_assign (x
, var
, &ilist
);
16323 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
16325 else if (is_reference (var
))
16326 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16328 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16329 s
= fold_convert (size_type_node
, s
);
16330 purpose
= size_int (map_idx
++);
16331 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16332 if (TREE_CODE (s
) != INTEGER_CST
)
16333 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16335 gcc_checking_assert (tkind
16336 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16337 talign
= ceil_log2 (talign
);
16338 tkind
|= talign
<< talign_shift
;
16339 gcc_checking_assert (tkind
16340 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16341 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16342 build_int_cstu (tkind_type
, tkind
));
16345 case OMP_CLAUSE_USE_DEVICE_PTR
:
16346 case OMP_CLAUSE_IS_DEVICE_PTR
:
16347 ovar
= OMP_CLAUSE_DECL (c
);
16348 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16349 x
= build_sender_ref (ovar
, ctx
);
16350 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16351 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
16353 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16354 type
= TREE_TYPE (ovar
);
16355 if (TREE_CODE (type
) == ARRAY_TYPE
)
16356 var
= build_fold_addr_expr (var
);
16359 if (is_reference (ovar
))
16361 type
= TREE_TYPE (type
);
16362 if (TREE_CODE (type
) != ARRAY_TYPE
)
16363 var
= build_simple_mem_ref (var
);
16364 var
= fold_convert (TREE_TYPE (x
), var
);
16367 gimplify_assign (x
, var
, &ilist
);
16369 purpose
= size_int (map_idx
++);
16370 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16371 gcc_checking_assert (tkind
16372 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16373 gcc_checking_assert (tkind
16374 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16375 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16376 build_int_cstu (tkind_type
, tkind
));
16380 gcc_assert (map_idx
== map_cnt
);
16382 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
16383 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
16384 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
16385 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
16386 for (int i
= 1; i
<= 2; i
++)
16387 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
16389 gimple_seq initlist
= NULL
;
16390 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
16391 TREE_VEC_ELT (t
, i
)),
16392 &initlist
, true, NULL_TREE
);
16393 gimple_seq_add_seq (&ilist
, initlist
);
16395 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
16397 TREE_THIS_VOLATILE (clobber
) = 1;
16398 gimple_seq_add_stmt (&olist
,
16399 gimple_build_assign (TREE_VEC_ELT (t
, i
),
16403 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
16404 TREE_THIS_VOLATILE (clobber
) = 1;
16405 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
16409 /* Once all the expansions are done, sequence all the different
16410 fragments inside gimple_omp_body. */
16415 && ctx
->record_type
)
16417 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
16418 /* fixup_child_record_type might have changed receiver_decl's type. */
16419 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
16420 gimple_seq_add_stmt (&new_body
,
16421 gimple_build_assign (ctx
->receiver_decl
, t
));
16423 gimple_seq_add_seq (&new_body
, fplist
);
16425 if (offloaded
|| data_region
)
16427 tree prev
= NULL_TREE
;
16428 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16429 switch (OMP_CLAUSE_CODE (c
))
16434 case OMP_CLAUSE_FIRSTPRIVATE
:
16435 if (is_gimple_omp_oacc (ctx
->stmt
))
16437 var
= OMP_CLAUSE_DECL (c
);
16438 if (is_reference (var
)
16439 || is_gimple_reg_type (TREE_TYPE (var
)))
16441 tree new_var
= lookup_decl (var
, ctx
);
16443 type
= TREE_TYPE (var
);
16444 if (is_reference (var
))
16445 type
= TREE_TYPE (type
);
16446 if ((INTEGRAL_TYPE_P (type
)
16447 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16448 || TREE_CODE (type
) == POINTER_TYPE
)
16450 x
= build_receiver_ref (var
, false, ctx
);
16451 if (TREE_CODE (type
) != POINTER_TYPE
)
16452 x
= fold_convert (pointer_sized_int_node
, x
);
16453 x
= fold_convert (type
, x
);
16454 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16456 if (is_reference (var
))
16458 tree v
= create_tmp_var_raw (type
, get_name (var
));
16459 gimple_add_tmp_var (v
);
16460 TREE_ADDRESSABLE (v
) = 1;
16461 gimple_seq_add_stmt (&new_body
,
16462 gimple_build_assign (v
, x
));
16463 x
= build_fold_addr_expr (v
);
16465 gimple_seq_add_stmt (&new_body
,
16466 gimple_build_assign (new_var
, x
));
16470 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
16471 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16473 gimple_seq_add_stmt (&new_body
,
16474 gimple_build_assign (new_var
, x
));
16477 else if (is_variable_sized (var
))
16479 tree pvar
= DECL_VALUE_EXPR (var
);
16480 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16481 pvar
= TREE_OPERAND (pvar
, 0);
16482 gcc_assert (DECL_P (pvar
));
16483 tree new_var
= lookup_decl (pvar
, ctx
);
16484 x
= build_receiver_ref (var
, false, ctx
);
16485 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16486 gimple_seq_add_stmt (&new_body
,
16487 gimple_build_assign (new_var
, x
));
16490 case OMP_CLAUSE_PRIVATE
:
16491 if (is_gimple_omp_oacc (ctx
->stmt
))
16493 var
= OMP_CLAUSE_DECL (c
);
16494 if (is_reference (var
))
16496 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16497 tree new_var
= lookup_decl (var
, ctx
);
16498 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16499 if (TREE_CONSTANT (x
))
16501 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
16503 gimple_add_tmp_var (x
);
16504 TREE_ADDRESSABLE (x
) = 1;
16505 x
= build_fold_addr_expr_loc (clause_loc
, x
);
16510 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16511 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16512 gimple_seq_add_stmt (&new_body
,
16513 gimple_build_assign (new_var
, x
));
16516 case OMP_CLAUSE_USE_DEVICE_PTR
:
16517 case OMP_CLAUSE_IS_DEVICE_PTR
:
16518 var
= OMP_CLAUSE_DECL (c
);
16519 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16520 x
= build_sender_ref (var
, ctx
);
16522 x
= build_receiver_ref (var
, false, ctx
);
16523 if (is_variable_sized (var
))
16525 tree pvar
= DECL_VALUE_EXPR (var
);
16526 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16527 pvar
= TREE_OPERAND (pvar
, 0);
16528 gcc_assert (DECL_P (pvar
));
16529 tree new_var
= lookup_decl (pvar
, ctx
);
16530 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16531 gimple_seq_add_stmt (&new_body
,
16532 gimple_build_assign (new_var
, x
));
16534 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16536 tree new_var
= lookup_decl (var
, ctx
);
16537 new_var
= DECL_VALUE_EXPR (new_var
);
16538 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
16539 new_var
= TREE_OPERAND (new_var
, 0);
16540 gcc_assert (DECL_P (new_var
));
16541 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16542 gimple_seq_add_stmt (&new_body
,
16543 gimple_build_assign (new_var
, x
));
16547 tree type
= TREE_TYPE (var
);
16548 tree new_var
= lookup_decl (var
, ctx
);
16549 if (is_reference (var
))
16551 type
= TREE_TYPE (type
);
16552 if (TREE_CODE (type
) != ARRAY_TYPE
)
16554 tree v
= create_tmp_var_raw (type
, get_name (var
));
16555 gimple_add_tmp_var (v
);
16556 TREE_ADDRESSABLE (v
) = 1;
16557 x
= fold_convert (type
, x
);
16558 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16560 gimple_seq_add_stmt (&new_body
,
16561 gimple_build_assign (v
, x
));
16562 x
= build_fold_addr_expr (v
);
16565 new_var
= DECL_VALUE_EXPR (new_var
);
16566 x
= fold_convert (TREE_TYPE (new_var
), x
);
16567 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16568 gimple_seq_add_stmt (&new_body
,
16569 gimple_build_assign (new_var
, x
));
16573 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16574 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16575 are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
16576 or references to VLAs. */
16577 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16578 switch (OMP_CLAUSE_CODE (c
))
16583 case OMP_CLAUSE_MAP
:
16584 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16585 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16587 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16588 HOST_WIDE_INT offset
= 0;
16590 var
= OMP_CLAUSE_DECL (c
);
16592 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
16593 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
16595 && varpool_node::get_create (var
)->offloadable
)
16597 if (TREE_CODE (var
) == INDIRECT_REF
16598 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
16599 var
= TREE_OPERAND (var
, 0);
16600 if (TREE_CODE (var
) == COMPONENT_REF
)
16602 var
= get_addr_base_and_unit_offset (var
, &offset
);
16603 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
16605 else if (DECL_SIZE (var
)
16606 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
16608 tree var2
= DECL_VALUE_EXPR (var
);
16609 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
16610 var2
= TREE_OPERAND (var2
, 0);
16611 gcc_assert (DECL_P (var2
));
16614 tree new_var
= lookup_decl (var
, ctx
), x
;
16615 tree type
= TREE_TYPE (new_var
);
16617 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
16618 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
16621 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
16623 new_var
= build2 (MEM_REF
, type
,
16624 build_fold_addr_expr (new_var
),
16625 build_int_cst (build_pointer_type (type
),
16628 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
16630 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
16631 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
16632 new_var
= build2 (MEM_REF
, type
,
16633 build_fold_addr_expr (new_var
),
16634 build_int_cst (build_pointer_type (type
),
16638 is_ref
= is_reference (var
);
16639 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16641 bool ref_to_array
= false;
16644 type
= TREE_TYPE (type
);
16645 if (TREE_CODE (type
) == ARRAY_TYPE
)
16647 type
= build_pointer_type (type
);
16648 ref_to_array
= true;
16651 else if (TREE_CODE (type
) == ARRAY_TYPE
)
16653 tree decl2
= DECL_VALUE_EXPR (new_var
);
16654 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
16655 decl2
= TREE_OPERAND (decl2
, 0);
16656 gcc_assert (DECL_P (decl2
));
16658 type
= TREE_TYPE (new_var
);
16660 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
16661 x
= fold_convert_loc (clause_loc
, type
, x
);
16662 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
16664 tree bias
= OMP_CLAUSE_SIZE (c
);
16666 bias
= lookup_decl (bias
, ctx
);
16667 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
16668 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
16670 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
16671 TREE_TYPE (x
), x
, bias
);
16674 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16675 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16676 if (is_ref
&& !ref_to_array
)
16678 tree t
= create_tmp_var_raw (type
, get_name (var
));
16679 gimple_add_tmp_var (t
);
16680 TREE_ADDRESSABLE (t
) = 1;
16681 gimple_seq_add_stmt (&new_body
,
16682 gimple_build_assign (t
, x
));
16683 x
= build_fold_addr_expr_loc (clause_loc
, t
);
16685 gimple_seq_add_stmt (&new_body
,
16686 gimple_build_assign (new_var
, x
));
16689 else if (OMP_CLAUSE_CHAIN (c
)
16690 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
16692 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16693 == GOMP_MAP_FIRSTPRIVATE_POINTER
16694 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16695 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16698 case OMP_CLAUSE_PRIVATE
:
16699 var
= OMP_CLAUSE_DECL (c
);
16700 if (is_variable_sized (var
))
16702 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16703 tree new_var
= lookup_decl (var
, ctx
);
16704 tree pvar
= DECL_VALUE_EXPR (var
);
16705 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16706 pvar
= TREE_OPERAND (pvar
, 0);
16707 gcc_assert (DECL_P (pvar
));
16708 tree new_pvar
= lookup_decl (pvar
, ctx
);
16709 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16710 tree al
= size_int (DECL_ALIGN (var
));
16711 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
16712 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16713 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
16714 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16715 gimple_seq_add_stmt (&new_body
,
16716 gimple_build_assign (new_pvar
, x
));
16718 else if (is_reference (var
) && !is_gimple_omp_oacc (ctx
->stmt
))
16720 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16721 tree new_var
= lookup_decl (var
, ctx
);
16722 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16723 if (TREE_CONSTANT (x
))
16728 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16729 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
16730 tree al
= size_int (TYPE_ALIGN (rtype
));
16731 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16734 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16735 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16736 gimple_seq_add_stmt (&new_body
,
16737 gimple_build_assign (new_var
, x
));
16742 gimple_seq fork_seq
= NULL
;
16743 gimple_seq join_seq
= NULL
;
16745 if (is_oacc_parallel (ctx
))
16747 /* If there are reductions on the offloaded region itself, treat
16748 them as a dummy GANG loop. */
16749 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
16751 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
16752 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
16755 gimple_seq_add_seq (&new_body
, fork_seq
);
16756 gimple_seq_add_seq (&new_body
, tgt_body
);
16757 gimple_seq_add_seq (&new_body
, join_seq
);
16760 new_body
= maybe_catch_exception (new_body
);
16762 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
16763 gimple_omp_set_body (stmt
, new_body
);
16766 bind
= gimple_build_bind (NULL
, NULL
,
16767 tgt_bind
? gimple_bind_block (tgt_bind
)
16769 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
16770 gimple_bind_add_seq (bind
, ilist
);
16771 gimple_bind_add_stmt (bind
, stmt
);
16772 gimple_bind_add_seq (bind
, olist
);
16774 pop_gimplify_context (NULL
);
16778 gimple_bind_add_seq (dep_bind
, dep_ilist
);
16779 gimple_bind_add_stmt (dep_bind
, bind
);
16780 gimple_bind_add_seq (dep_bind
, dep_olist
);
16781 pop_gimplify_context (dep_bind
);
16785 /* Expand code for an OpenMP teams directive. */
16788 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16790 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
16791 push_gimplify_context ();
16793 tree block
= make_node (BLOCK
);
16794 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
16795 gsi_replace (gsi_p
, bind
, true);
16796 gimple_seq bind_body
= NULL
;
16797 gimple_seq dlist
= NULL
;
16798 gimple_seq olist
= NULL
;
16800 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16801 OMP_CLAUSE_NUM_TEAMS
);
16802 if (num_teams
== NULL_TREE
)
16803 num_teams
= build_int_cst (unsigned_type_node
, 0);
16806 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16807 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16808 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16810 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16811 OMP_CLAUSE_THREAD_LIMIT
);
16812 if (thread_limit
== NULL_TREE
)
16813 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16816 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16817 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16818 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16822 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16823 &bind_body
, &dlist
, ctx
, NULL
);
16824 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16825 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16826 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16828 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16829 location_t loc
= gimple_location (teams_stmt
);
16830 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16831 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16832 gimple_set_location (call
, loc
);
16833 gimple_seq_add_stmt (&bind_body
, call
);
16836 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16837 gimple_omp_set_body (teams_stmt
, NULL
);
16838 gimple_seq_add_seq (&bind_body
, olist
);
16839 gimple_seq_add_seq (&bind_body
, dlist
);
16840 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16841 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16842 gimple_bind_set_body (bind
, bind_body
);
16844 pop_gimplify_context (bind
);
16846 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16847 BLOCK_VARS (block
) = ctx
->block_vars
;
16848 if (BLOCK_VARS (block
))
16849 TREE_USED (block
) = 1;
16852 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16855 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16857 gimple
*stmt
= gsi_stmt (*gsi_p
);
16858 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
16859 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
16860 gimple_build_omp_return (false));
16864 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16865 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16866 of OMP context, but with task_shared_vars set. */
16869 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16874 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16875 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16878 if (task_shared_vars
16880 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16883 /* If a global variable has been privatized, TREE_CONSTANT on
16884 ADDR_EXPR might be wrong. */
16885 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16886 recompute_tree_invariant_for_addr_expr (t
);
16888 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16892 /* Data to be communicated between lower_omp_regimplify_operands and
16893 lower_omp_regimplify_operands_p. */
16895 struct lower_omp_regimplify_operands_data
16901 /* Helper function for lower_omp_regimplify_operands. Find
16902 omp_member_access_dummy_var vars and adjust temporarily their
16903 DECL_VALUE_EXPRs if needed. */
16906 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16909 tree t
= omp_member_access_dummy_var (*tp
);
16912 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16913 lower_omp_regimplify_operands_data
*ldata
16914 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16915 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16918 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16919 ldata
->decls
->safe_push (*tp
);
16920 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16921 SET_DECL_VALUE_EXPR (*tp
, v
);
16924 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16928 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16929 of omp_member_access_dummy_var vars during regimplification. */
16932 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16933 gimple_stmt_iterator
*gsi_p
)
16935 auto_vec
<tree
, 10> decls
;
16938 struct walk_stmt_info wi
;
16939 memset (&wi
, '\0', sizeof (wi
));
16940 struct lower_omp_regimplify_operands_data data
;
16942 data
.decls
= &decls
;
16944 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16946 gimple_regimplify_operands (stmt
, gsi_p
);
16947 while (!decls
.is_empty ())
16949 tree t
= decls
.pop ();
16950 tree v
= decls
.pop ();
16951 SET_DECL_VALUE_EXPR (t
, v
);
16956 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16958 gimple
*stmt
= gsi_stmt (*gsi_p
);
16959 struct walk_stmt_info wi
;
16962 if (gimple_has_location (stmt
))
16963 input_location
= gimple_location (stmt
);
16965 if (task_shared_vars
)
16966 memset (&wi
, '\0', sizeof (wi
));
16968 /* If we have issued syntax errors, avoid doing any heavy lifting.
16969 Just replace the OMP directives with a NOP to avoid
16970 confusing RTL expansion. */
16971 if (seen_error () && is_gimple_omp (stmt
))
16973 gsi_replace (gsi_p
, gimple_build_nop (), true);
16977 switch (gimple_code (stmt
))
16981 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16982 if ((ctx
|| task_shared_vars
)
16983 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16984 lower_omp_regimplify_p
,
16985 ctx
? NULL
: &wi
, NULL
)
16986 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16987 lower_omp_regimplify_p
,
16988 ctx
? NULL
: &wi
, NULL
)))
16989 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16993 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16995 case GIMPLE_EH_FILTER
:
16996 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16999 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
17000 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
17002 case GIMPLE_TRANSACTION
:
17003 lower_omp (gimple_transaction_body_ptr (
17004 as_a
<gtransaction
*> (stmt
)),
17008 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
17010 case GIMPLE_OMP_PARALLEL
:
17011 case GIMPLE_OMP_TASK
:
17012 ctx
= maybe_lookup_ctx (stmt
);
17014 if (ctx
->cancellable
)
17015 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17016 lower_omp_taskreg (gsi_p
, ctx
);
17018 case GIMPLE_OMP_FOR
:
17019 ctx
= maybe_lookup_ctx (stmt
);
17021 if (ctx
->cancellable
)
17022 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17023 lower_omp_for (gsi_p
, ctx
);
17025 case GIMPLE_OMP_SECTIONS
:
17026 ctx
= maybe_lookup_ctx (stmt
);
17028 if (ctx
->cancellable
)
17029 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17030 lower_omp_sections (gsi_p
, ctx
);
17032 case GIMPLE_OMP_SINGLE
:
17033 ctx
= maybe_lookup_ctx (stmt
);
17035 lower_omp_single (gsi_p
, ctx
);
17037 case GIMPLE_OMP_MASTER
:
17038 ctx
= maybe_lookup_ctx (stmt
);
17040 lower_omp_master (gsi_p
, ctx
);
17042 case GIMPLE_OMP_TASKGROUP
:
17043 ctx
= maybe_lookup_ctx (stmt
);
17045 lower_omp_taskgroup (gsi_p
, ctx
);
17047 case GIMPLE_OMP_ORDERED
:
17048 ctx
= maybe_lookup_ctx (stmt
);
17050 lower_omp_ordered (gsi_p
, ctx
);
17052 case GIMPLE_OMP_CRITICAL
:
17053 ctx
= maybe_lookup_ctx (stmt
);
17055 lower_omp_critical (gsi_p
, ctx
);
17057 case GIMPLE_OMP_ATOMIC_LOAD
:
17058 if ((ctx
|| task_shared_vars
)
17059 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
17060 as_a
<gomp_atomic_load
*> (stmt
)),
17061 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
17062 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17064 case GIMPLE_OMP_TARGET
:
17065 ctx
= maybe_lookup_ctx (stmt
);
17067 lower_omp_target (gsi_p
, ctx
);
17069 case GIMPLE_OMP_TEAMS
:
17070 ctx
= maybe_lookup_ctx (stmt
);
17072 lower_omp_teams (gsi_p
, ctx
);
17074 case GIMPLE_OMP_GRID_BODY
:
17075 ctx
= maybe_lookup_ctx (stmt
);
17077 lower_omp_grid_body (gsi_p
, ctx
);
17081 call_stmt
= as_a
<gcall
*> (stmt
);
17082 fndecl
= gimple_call_fndecl (call_stmt
);
17084 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
17085 switch (DECL_FUNCTION_CODE (fndecl
))
17087 case BUILT_IN_GOMP_BARRIER
:
17091 case BUILT_IN_GOMP_CANCEL
:
17092 case BUILT_IN_GOMP_CANCELLATION_POINT
:
17095 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
17096 cctx
= cctx
->outer
;
17097 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
17098 if (!cctx
->cancellable
)
17100 if (DECL_FUNCTION_CODE (fndecl
)
17101 == BUILT_IN_GOMP_CANCELLATION_POINT
)
17103 stmt
= gimple_build_nop ();
17104 gsi_replace (gsi_p
, stmt
, false);
17108 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
17110 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
17111 gimple_call_set_fndecl (call_stmt
, fndecl
);
17112 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
17115 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
17116 gimple_call_set_lhs (call_stmt
, lhs
);
17117 tree fallthru_label
;
17118 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
17120 g
= gimple_build_label (fallthru_label
);
17121 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17122 g
= gimple_build_cond (NE_EXPR
, lhs
,
17123 fold_convert (TREE_TYPE (lhs
),
17124 boolean_false_node
),
17125 cctx
->cancel_label
, fallthru_label
);
17126 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17133 if ((ctx
|| task_shared_vars
)
17134 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
17137 /* Just remove clobbers, this should happen only if we have
17138 "privatized" local addressable variables in SIMD regions,
17139 the clobber isn't needed in that case and gimplifying address
17140 of the ARRAY_REF into a pointer and creating MEM_REF based
17141 clobber would create worse code than we get with the clobber
17143 if (gimple_clobber_p (stmt
))
17145 gsi_replace (gsi_p
, gimple_build_nop (), true);
17148 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17155 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
17157 location_t saved_location
= input_location
;
17158 gimple_stmt_iterator gsi
;
17159 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17160 lower_omp_1 (&gsi
, ctx
);
17161 /* During gimplification, we haven't folded statments inside offloading
17162 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17163 if (target_nesting_level
|| taskreg_nesting_level
)
17164 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17166 input_location
= saved_location
;
17169 /* Returen true if STMT is an assignment of a register-type into a local
17173 grid_reg_assignment_to_local_var_p (gimple
*stmt
)
17175 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
17178 tree lhs
= gimple_assign_lhs (assign
);
17179 if (TREE_CODE (lhs
) != VAR_DECL
17180 || !is_gimple_reg_type (TREE_TYPE (lhs
))
17181 || is_global_var (lhs
))
17186 /* Return true if all statements in SEQ are assignments to local register-type
17190 grid_seq_only_contains_local_assignments (gimple_seq seq
)
17195 gimple_stmt_iterator gsi
;
17196 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17197 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi
)))
17202 /* Scan statements in SEQ and call itself recursively on any bind. If during
17203 whole search only assignments to register-type local variables and one
17204 single OMP statement is encountered, return true, otherwise return false.
17205 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17206 are used for dumping a note about a failure. */
17209 grid_find_single_omp_among_assignments_1 (gimple_seq seq
, location_t target_loc
,
17210 const char *name
, gimple
**ret
)
17212 gimple_stmt_iterator gsi
;
17213 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17215 gimple
*stmt
= gsi_stmt (gsi
);
17217 if (grid_reg_assignment_to_local_var_p (stmt
))
17219 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17221 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind
),
17222 target_loc
, name
, ret
))
17225 else if (is_gimple_omp (stmt
))
17229 if (dump_enabled_p ())
17230 dump_printf_loc (MSG_NOTE
, target_loc
,
17231 "Will not turn target construct into a simple "
17232 "GPGPU kernel because %s construct contains "
17233 "multiple OpenMP constructs\n", name
);
17240 if (dump_enabled_p ())
17241 dump_printf_loc (MSG_NOTE
, target_loc
,
17242 "Will not turn target construct into a simple "
17243 "GPGPU kernel because %s construct contains "
17244 "a complex statement\n", name
);
17251 /* Scan statements in SEQ and make sure that it and any binds in it contain
17252 only assignments to local register-type variables and one OMP construct. If
17253 so, return that construct, otherwise return NULL. If dumping is enabled and
17254 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17258 grid_find_single_omp_among_assignments (gimple_seq seq
, location_t target_loc
,
17263 if (dump_enabled_p ())
17264 dump_printf_loc (MSG_NOTE
, target_loc
,
17265 "Will not turn target construct into a simple "
17266 "GPGPU kernel because %s construct has empty "
17272 gimple
*ret
= NULL
;
17273 if (grid_find_single_omp_among_assignments_1 (seq
, target_loc
, name
, &ret
))
17275 if (!ret
&& dump_enabled_p ())
17276 dump_printf_loc (MSG_NOTE
, target_loc
,
17277 "Will not turn target construct into a simple "
17278 "GPGPU kernel because %s construct does not contain"
17279 "any other OpenMP construct\n", name
);
17286 /* Walker function looking for statements there is no point gridifying (and for
17287 noreturn function calls which we cannot do). Return non-NULL if such a
17288 function is found. */
17291 grid_find_ungridifiable_statement (gimple_stmt_iterator
*gsi
,
17292 bool *handled_ops_p
,
17293 struct walk_stmt_info
*wi
)
17295 *handled_ops_p
= false;
17296 gimple
*stmt
= gsi_stmt (*gsi
);
17297 switch (gimple_code (stmt
))
17300 if (gimple_call_noreturn_p (as_a
<gcall
*> (stmt
)))
17302 *handled_ops_p
= true;
17304 return error_mark_node
;
17308 /* We may reduce the following list if we find a way to implement the
17309 clauses, but now there is no point trying further. */
17310 case GIMPLE_OMP_CRITICAL
:
17311 case GIMPLE_OMP_TASKGROUP
:
17312 case GIMPLE_OMP_TASK
:
17313 case GIMPLE_OMP_SECTION
:
17314 case GIMPLE_OMP_SECTIONS
:
17315 case GIMPLE_OMP_SECTIONS_SWITCH
:
17316 case GIMPLE_OMP_TARGET
:
17317 case GIMPLE_OMP_ORDERED
:
17318 *handled_ops_p
= true;
17320 return error_mark_node
;
17322 case GIMPLE_OMP_FOR
:
17323 if ((gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
17324 && gimple_omp_for_combined_into_p (stmt
))
17326 *handled_ops_p
= true;
17328 return error_mark_node
;
17339 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17340 kernel, return true, otherwise return false. In the case of success, also
17341 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17345 grid_target_follows_gridifiable_pattern (gomp_target
*target
, tree
*group_size_p
)
17347 if (gimple_omp_target_kind (target
) != GF_OMP_TARGET_KIND_REGION
)
17350 location_t tloc
= gimple_location (target
);
17352 = grid_find_single_omp_among_assignments (gimple_omp_body (target
),
17356 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17357 tree group_size
= NULL
;
17360 dump_printf_loc (MSG_NOTE
, tloc
,
17361 "Will not turn target construct into a simple "
17362 "GPGPU kernel because it does not have a sole teams "
17363 "construct in it.\n");
17367 tree clauses
= gimple_omp_teams_clauses (teams
);
17370 switch (OMP_CLAUSE_CODE (clauses
))
17372 case OMP_CLAUSE_NUM_TEAMS
:
17373 if (dump_enabled_p ())
17374 dump_printf_loc (MSG_NOTE
, tloc
,
17375 "Will not turn target construct into a "
17376 "gridified GPGPU kernel because we cannot "
17377 "handle num_teams clause of teams "
17381 case OMP_CLAUSE_REDUCTION
:
17382 if (dump_enabled_p ())
17383 dump_printf_loc (MSG_NOTE
, tloc
,
17384 "Will not turn target construct into a "
17385 "gridified GPGPU kernel because a reduction "
17386 "clause is present\n ");
17389 case OMP_CLAUSE_LASTPRIVATE
:
17390 if (dump_enabled_p ())
17391 dump_printf_loc (MSG_NOTE
, tloc
,
17392 "Will not turn target construct into a "
17393 "gridified GPGPU kernel because a lastprivate "
17394 "clause is present\n ");
17397 case OMP_CLAUSE_THREAD_LIMIT
:
17398 group_size
= OMP_CLAUSE_OPERAND (clauses
, 0);
17404 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17407 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (teams
), tloc
,
17411 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17414 dump_printf_loc (MSG_NOTE
, tloc
,
17415 "Will not turn target construct into a simple "
17416 "GPGPU kernel because the teams construct does not have "
17417 "a sole distribute construct in it.\n");
17421 gcc_assert (gimple_omp_for_kind (dist
) == GF_OMP_FOR_KIND_DISTRIBUTE
);
17422 if (!gimple_omp_for_combined_p (dist
))
17424 if (dump_enabled_p ())
17425 dump_printf_loc (MSG_NOTE
, tloc
,
17426 "Will not turn target construct into a gridified GPGPU "
17427 "kernel because we cannot handle a standalone "
17428 "distribute construct\n ");
17431 if (dist
->collapse
> 1)
17433 if (dump_enabled_p ())
17434 dump_printf_loc (MSG_NOTE
, tloc
,
17435 "Will not turn target construct into a gridified GPGPU "
17436 "kernel because the distribute construct contains "
17437 "collapse clause\n");
17440 struct omp_for_data fd
;
17441 extract_omp_for_data (dist
, &fd
, NULL
);
17444 if (group_size
&& !operand_equal_p (group_size
, fd
.chunk_size
, 0))
17446 if (dump_enabled_p ())
17447 dump_printf_loc (MSG_NOTE
, tloc
,
17448 "Will not turn target construct into a "
17449 "gridified GPGPU kernel because the teams "
17450 "thread limit is different from distribute "
17451 "schedule chunk\n");
17454 group_size
= fd
.chunk_size
;
17456 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (dist
), tloc
,
17458 gomp_parallel
*par
;
17459 if (!stmt
|| !(par
= dyn_cast
<gomp_parallel
*> (stmt
)))
17462 clauses
= gimple_omp_parallel_clauses (par
);
17465 switch (OMP_CLAUSE_CODE (clauses
))
17467 case OMP_CLAUSE_NUM_THREADS
:
17468 if (dump_enabled_p ())
17469 dump_printf_loc (MSG_NOTE
, tloc
,
17470 "Will not turn target construct into a gridified"
17471 "GPGPU kernel because there is a num_threads "
17472 "clause of the parallel construct\n");
17475 case OMP_CLAUSE_REDUCTION
:
17476 if (dump_enabled_p ())
17477 dump_printf_loc (MSG_NOTE
, tloc
,
17478 "Will not turn target construct into a "
17479 "gridified GPGPU kernel because a reduction "
17480 "clause is present\n ");
17483 case OMP_CLAUSE_LASTPRIVATE
:
17484 if (dump_enabled_p ())
17485 dump_printf_loc (MSG_NOTE
, tloc
,
17486 "Will not turn target construct into a "
17487 "gridified GPGPU kernel because a lastprivate "
17488 "clause is present\n ");
17494 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17497 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (par
), tloc
,
17500 if (!stmt
|| !(gfor
= dyn_cast
<gomp_for
*> (stmt
)))
17503 if (gimple_omp_for_kind (gfor
) != GF_OMP_FOR_KIND_FOR
)
17505 if (dump_enabled_p ())
17506 dump_printf_loc (MSG_NOTE
, tloc
,
17507 "Will not turn target construct into a gridified GPGPU "
17508 "kernel because the inner loop is not a simple for "
17512 if (gfor
->collapse
> 1)
17514 if (dump_enabled_p ())
17515 dump_printf_loc (MSG_NOTE
, tloc
,
17516 "Will not turn target construct into a gridified GPGPU "
17517 "kernel because the inner loop contains collapse "
17522 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor
)))
17524 if (dump_enabled_p ())
17525 dump_printf_loc (MSG_NOTE
, tloc
,
17526 "Will not turn target construct into a gridified GPGPU "
17527 "kernel because the inner loop pre_body contains"
17528 "a complex instruction\n");
17532 clauses
= gimple_omp_for_clauses (gfor
);
17535 switch (OMP_CLAUSE_CODE (clauses
))
17537 case OMP_CLAUSE_SCHEDULE
:
17538 if (OMP_CLAUSE_SCHEDULE_KIND (clauses
) != OMP_CLAUSE_SCHEDULE_AUTO
)
17540 if (dump_enabled_p ())
17541 dump_printf_loc (MSG_NOTE
, tloc
,
17542 "Will not turn target construct into a "
17543 "gridified GPGPU kernel because the inner "
17544 "loop has a non-automatic scheduling clause\n");
17549 case OMP_CLAUSE_REDUCTION
:
17550 if (dump_enabled_p ())
17551 dump_printf_loc (MSG_NOTE
, tloc
,
17552 "Will not turn target construct into a "
17553 "gridified GPGPU kernel because a reduction "
17554 "clause is present\n ");
17557 case OMP_CLAUSE_LASTPRIVATE
:
17558 if (dump_enabled_p ())
17559 dump_printf_loc (MSG_NOTE
, tloc
,
17560 "Will not turn target construct into a "
17561 "gridified GPGPU kernel because a lastprivate "
17562 "clause is present\n ");
17568 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17571 struct walk_stmt_info wi
;
17572 memset (&wi
, 0, sizeof (wi
));
17573 if (walk_gimple_seq (gimple_omp_body (gfor
),
17574 grid_find_ungridifiable_statement
,
17577 gimple
*bad
= (gimple
*) wi
.info
;
17578 if (dump_enabled_p ())
17580 if (is_gimple_call (bad
))
17581 dump_printf_loc (MSG_NOTE
, tloc
,
17582 "Will not turn target construct into a gridified "
17583 " GPGPU kernel because the inner loop contains "
17584 "call to a noreturn function\n");
17585 if (gimple_code (bad
) == GIMPLE_OMP_FOR
)
17586 dump_printf_loc (MSG_NOTE
, tloc
,
17587 "Will not turn target construct into a gridified "
17588 " GPGPU kernel because the inner loop contains "
17589 "a simd construct\n");
17591 dump_printf_loc (MSG_NOTE
, tloc
,
17592 "Will not turn target construct into a gridified "
17593 "GPGPU kernel because the inner loop contains "
17594 "statement %s which cannot be transformed\n",
17595 gimple_code_name
[(int) gimple_code (bad
)]);
17600 *group_size_p
= group_size
;
17604 /* Operand walker, used to remap pre-body declarations according to a hash map
17605 provided in DATA. */
17608 grid_remap_prebody_decls (tree
*tp
, int *walk_subtrees
, void *data
)
17612 if (DECL_P (t
) || TYPE_P (t
))
17613 *walk_subtrees
= 0;
17615 *walk_subtrees
= 1;
17617 if (TREE_CODE (t
) == VAR_DECL
)
17619 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17620 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17621 tree
*repl
= declmap
->get (t
);
17628 /* Copy leading register-type assignments to local variables in SRC to just
17629 before DST, Creating temporaries, adjusting mapping of operands in WI and
17630 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17631 Return the first statement that does not conform to
17632 grid_reg_assignment_to_local_var_p or NULL. */
17635 grid_copy_leading_local_assignments (gimple_seq src
, gimple_stmt_iterator
*dst
,
17636 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17638 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17639 gimple_stmt_iterator gsi
;
17640 for (gsi
= gsi_start (src
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17642 gimple
*stmt
= gsi_stmt (gsi
);
17643 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17645 gimple
*r
= grid_copy_leading_local_assignments
17646 (gimple_bind_body (bind
), dst
, tgt_bind
, wi
);
17652 if (!grid_reg_assignment_to_local_var_p (stmt
))
17654 tree lhs
= gimple_assign_lhs (as_a
<gassign
*> (stmt
));
17655 tree repl
= copy_var_decl (lhs
, create_tmp_var_name (NULL
),
17657 DECL_CONTEXT (repl
) = current_function_decl
;
17658 gimple_bind_append_vars (tgt_bind
, repl
);
17660 declmap
->put (lhs
, repl
);
17661 gassign
*copy
= as_a
<gassign
*> (gimple_copy (stmt
));
17662 walk_gimple_op (copy
, grid_remap_prebody_decls
, wi
);
17663 gsi_insert_before (dst
, copy
, GSI_SAME_STMT
);
17668 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17669 components, mark them as part of kernel and return the inner loop, and copy
17670 assignment leading to them just before DST, remapping them using WI and
17671 adding new temporaries to TGT_BIND. */
17674 grid_process_kernel_body_copy (gimple_seq seq
, gimple_stmt_iterator
*dst
,
17675 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17677 gimple
*stmt
= grid_copy_leading_local_assignments (seq
, dst
, tgt_bind
, wi
);
17678 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17679 gcc_assert (teams
);
17680 gimple_omp_teams_set_grid_phony (teams
, true);
17681 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (teams
), dst
,
17683 gcc_checking_assert (stmt
);
17684 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17686 gimple_seq prebody
= gimple_omp_for_pre_body (dist
);
17688 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17689 gimple_omp_for_set_grid_phony (dist
, true);
17690 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (dist
), dst
,
17692 gcc_checking_assert (stmt
);
17694 gomp_parallel
*parallel
= as_a
<gomp_parallel
*> (stmt
);
17695 gimple_omp_parallel_set_grid_phony (parallel
, true);
17696 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (parallel
), dst
,
17698 gomp_for
*inner_loop
= as_a
<gomp_for
*> (stmt
);
17699 gimple_omp_for_set_kind (inner_loop
, GF_OMP_FOR_KIND_GRID_LOOP
);
17700 prebody
= gimple_omp_for_pre_body (inner_loop
);
17702 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17707 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17708 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17709 is the bind into which temporaries inserted before TARGET should be
17713 grid_attempt_target_gridification (gomp_target
*target
,
17714 gimple_stmt_iterator
*gsi
,
17718 if (!target
|| !grid_target_follows_gridifiable_pattern (target
, &group_size
))
17721 location_t loc
= gimple_location (target
);
17722 if (dump_enabled_p ())
17723 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
17724 "Target construct will be turned into a gridified GPGPU "
17727 /* Copy target body to a GPUKERNEL construct: */
17728 gimple_seq kernel_seq
= copy_gimple_seq_and_replace_locals
17729 (gimple_omp_body (target
));
17731 hash_map
<tree
, tree
> *declmap
= new hash_map
<tree
, tree
>;
17732 struct walk_stmt_info wi
;
17733 memset (&wi
, 0, sizeof (struct walk_stmt_info
));
17736 /* Copy assignments in between OMP statements before target, mark OMP
17737 statements within copy appropriatly. */
17738 gomp_for
*inner_loop
= grid_process_kernel_body_copy (kernel_seq
, gsi
,
17741 gbind
*old_bind
= as_a
<gbind
*> (gimple_seq_first (gimple_omp_body (target
)));
17742 gbind
*new_bind
= as_a
<gbind
*> (gimple_seq_first (kernel_seq
));
17743 tree new_block
= gimple_bind_block (new_bind
);
17744 tree enc_block
= BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind
));
17745 BLOCK_CHAIN (new_block
) = BLOCK_SUBBLOCKS (enc_block
);
17746 BLOCK_SUBBLOCKS (enc_block
) = new_block
;
17747 BLOCK_SUPERCONTEXT (new_block
) = enc_block
;
17748 gimple
*gpukernel
= gimple_build_omp_grid_body (kernel_seq
);
17749 gimple_seq_add_stmt
17750 (gimple_bind_body_ptr (as_a
<gbind
*> (gimple_omp_body (target
))),
17753 walk_tree (&group_size
, grid_remap_prebody_decls
, &wi
, NULL
);
17754 push_gimplify_context ();
17755 size_t collapse
= gimple_omp_for_collapse (inner_loop
);
17756 for (size_t i
= 0; i
< collapse
; i
++)
17758 tree itype
, type
= TREE_TYPE (gimple_omp_for_index (inner_loop
, i
));
17759 if (POINTER_TYPE_P (type
))
17760 itype
= signed_type_for (type
);
17764 enum tree_code cond_code
= gimple_omp_for_cond (inner_loop
, i
);
17765 tree n1
= unshare_expr (gimple_omp_for_initial (inner_loop
, i
));
17766 walk_tree (&n1
, grid_remap_prebody_decls
, &wi
, NULL
);
17767 tree n2
= unshare_expr (gimple_omp_for_final (inner_loop
, i
));
17768 walk_tree (&n2
, grid_remap_prebody_decls
, &wi
, NULL
);
17769 adjust_for_condition (loc
, &cond_code
, &n2
);
17771 step
= get_omp_for_step_from_incr (loc
,
17772 gimple_omp_for_incr (inner_loop
, i
));
17773 gimple_seq tmpseq
= NULL
;
17774 n1
= fold_convert (itype
, n1
);
17775 n2
= fold_convert (itype
, n2
);
17776 tree t
= build_int_cst (itype
, (cond_code
== LT_EXPR
? -1 : 1));
17777 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
17778 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
17779 t
= fold_build2 (MINUS_EXPR
, itype
, t
, n1
);
17780 if (TYPE_UNSIGNED (itype
) && cond_code
== GT_EXPR
)
17781 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
17782 fold_build1 (NEGATE_EXPR
, itype
, t
),
17783 fold_build1 (NEGATE_EXPR
, itype
, step
));
17785 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
17786 tree gs
= fold_convert (uint32_type_node
, t
);
17787 gimplify_expr (&gs
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17788 if (!gimple_seq_empty_p (tmpseq
))
17789 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17792 if (i
== 0 && group_size
)
17794 ws
= fold_convert (uint32_type_node
, group_size
);
17796 gimplify_expr (&ws
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17797 if (!gimple_seq_empty_p (tmpseq
))
17798 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17801 ws
= build_zero_cst (uint32_type_node
);
17803 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__GRIDDIM_
);
17804 OMP_CLAUSE__GRIDDIM__DIMENSION (c
) = i
;
17805 OMP_CLAUSE__GRIDDIM__SIZE (c
) = gs
;
17806 OMP_CLAUSE__GRIDDIM__GROUP (c
) = ws
;
17807 OMP_CLAUSE_CHAIN (c
) = gimple_omp_target_clauses (target
);
17808 gimple_omp_target_set_clauses (target
, c
);
17810 pop_gimplify_context (tgt_bind
);
17815 /* Walker function doing all the work for create_target_kernels. */
17818 grid_gridify_all_targets_stmt (gimple_stmt_iterator
*gsi
,
17819 bool *handled_ops_p
,
17820 struct walk_stmt_info
*incoming
)
17822 *handled_ops_p
= false;
17824 gimple
*stmt
= gsi_stmt (*gsi
);
17825 gomp_target
*target
= dyn_cast
<gomp_target
*> (stmt
);
17828 gbind
*tgt_bind
= (gbind
*) incoming
->info
;
17829 gcc_checking_assert (tgt_bind
);
17830 grid_attempt_target_gridification (target
, gsi
, tgt_bind
);
17833 gbind
*bind
= dyn_cast
<gbind
*> (stmt
);
17836 *handled_ops_p
= true;
17837 struct walk_stmt_info wi
;
17838 memset (&wi
, 0, sizeof (wi
));
17840 walk_gimple_seq_mod (gimple_bind_body_ptr (bind
),
17841 grid_gridify_all_targets_stmt
, NULL
, &wi
);
17846 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17847 have their bodies duplicated, with the new copy being put into a
17848 gimple_omp_grid_body statement. All kernel-related construct within the
17849 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17850 re-structuring is often needed, such as copying pre-bodies before the target
17851 construct so that kernel grid sizes can be computed. */
17854 grid_gridify_all_targets (gimple_seq
*body_p
)
17856 struct walk_stmt_info wi
;
17857 memset (&wi
, 0, sizeof (wi
));
17858 walk_gimple_seq_mod (body_p
, grid_gridify_all_targets_stmt
, NULL
, &wi
);
17862 /* Main entry point. */
17864 static unsigned int
17865 execute_lower_omp (void)
17871 /* This pass always runs, to provide PROP_gimple_lomp.
17872 But often, there is nothing to do. */
17873 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
17874 && flag_openmp_simd
== 0)
17877 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
17878 delete_omp_context
);
17880 body
= gimple_body (current_function_decl
);
17882 if (hsa_gen_requested_p ())
17883 grid_gridify_all_targets (&body
);
17885 scan_omp (&body
, NULL
);
17886 gcc_assert (taskreg_nesting_level
== 0);
17887 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
17888 finish_taskreg_scan (ctx
);
17889 taskreg_contexts
.release ();
17891 if (all_contexts
->root
)
17893 if (task_shared_vars
)
17894 push_gimplify_context ();
17895 lower_omp (&body
, NULL
);
17896 if (task_shared_vars
)
17897 pop_gimplify_context (NULL
);
17902 splay_tree_delete (all_contexts
);
17903 all_contexts
= NULL
;
17905 BITMAP_FREE (task_shared_vars
);
17911 const pass_data pass_data_lower_omp
=
17913 GIMPLE_PASS
, /* type */
17914 "omplower", /* name */
17915 OPTGROUP_NONE
, /* optinfo_flags */
17916 TV_NONE
, /* tv_id */
17917 PROP_gimple_any
, /* properties_required */
17918 PROP_gimple_lomp
, /* properties_provided */
17919 0, /* properties_destroyed */
17920 0, /* todo_flags_start */
17921 0, /* todo_flags_finish */
17924 class pass_lower_omp
: public gimple_opt_pass
17927 pass_lower_omp (gcc::context
*ctxt
)
17928 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
17931 /* opt_pass methods: */
17932 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
17934 }; // class pass_lower_omp
17936 } // anon namespace
17939 make_pass_lower_omp (gcc::context
*ctxt
)
17941 return new pass_lower_omp (ctxt
);
17944 /* The following is a utility to diagnose structured block violations.
17945 It is not part of the "omplower" pass, as that's invoked too late. It
17946 should be invoked by the respective front ends after gimplification. */
17948 static splay_tree all_labels
;
17950 /* Check for mismatched contexts and generate an error if needed. Return
17951 true if an error is detected. */
17954 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
17955 gimple
*branch_ctx
, gimple
*label_ctx
)
17957 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
17958 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
17960 if (label_ctx
== branch_ctx
)
17963 const char* kind
= NULL
;
17968 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
17969 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
17971 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
17972 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
17973 kind
= "Cilk Plus";
17977 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
17978 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
17980 gcc_checking_assert (kind
== NULL
);
17986 gcc_checking_assert (flag_openmp
);
17991 Previously we kept track of the label's entire context in diagnose_sb_[12]
17992 so we could traverse it and issue a correct "exit" or "enter" error
17993 message upon a structured block violation.
17995 We built the context by building a list with tree_cons'ing, but there is
17996 no easy counterpart in gimple tuples. It seems like far too much work
17997 for issuing exit/enter error messages. If someone really misses the
17998 distinct error message... patches welcome.
18002 /* Try to avoid confusing the user by producing and error message
18003 with correct "exit" or "enter" verbiage. We prefer "exit"
18004 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
18005 if (branch_ctx
== NULL
)
18011 if (TREE_VALUE (label_ctx
) == branch_ctx
)
18016 label_ctx
= TREE_CHAIN (label_ctx
);
18021 error ("invalid exit from %s structured block", kind
);
18023 error ("invalid entry to %s structured block", kind
);
18026 /* If it's obvious we have an invalid entry, be specific about the error. */
18027 if (branch_ctx
== NULL
)
18028 error ("invalid entry to %s structured block", kind
);
18031 /* Otherwise, be vague and lazy, but efficient. */
18032 error ("invalid branch to/from %s structured block", kind
);
18035 gsi_replace (gsi_p
, gimple_build_nop (), false);
18039 /* Pass 1: Create a minimal tree of structured blocks, and record
18040 where each label is found. */
18043 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18044 struct walk_stmt_info
*wi
)
18046 gimple
*context
= (gimple
*) wi
->info
;
18047 gimple
*inner_context
;
18048 gimple
*stmt
= gsi_stmt (*gsi_p
);
18050 *handled_ops_p
= true;
18052 switch (gimple_code (stmt
))
18056 case GIMPLE_OMP_PARALLEL
:
18057 case GIMPLE_OMP_TASK
:
18058 case GIMPLE_OMP_SECTIONS
:
18059 case GIMPLE_OMP_SINGLE
:
18060 case GIMPLE_OMP_SECTION
:
18061 case GIMPLE_OMP_MASTER
:
18062 case GIMPLE_OMP_ORDERED
:
18063 case GIMPLE_OMP_CRITICAL
:
18064 case GIMPLE_OMP_TARGET
:
18065 case GIMPLE_OMP_TEAMS
:
18066 case GIMPLE_OMP_TASKGROUP
:
18067 /* The minimal context here is just the current OMP construct. */
18068 inner_context
= stmt
;
18069 wi
->info
= inner_context
;
18070 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18071 wi
->info
= context
;
18074 case GIMPLE_OMP_FOR
:
18075 inner_context
= stmt
;
18076 wi
->info
= inner_context
;
18077 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18079 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
18080 diagnose_sb_1
, NULL
, wi
);
18081 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18082 wi
->info
= context
;
18086 splay_tree_insert (all_labels
,
18087 (splay_tree_key
) gimple_label_label (
18088 as_a
<glabel
*> (stmt
)),
18089 (splay_tree_value
) context
);
18099 /* Pass 2: Check each branch and see if its context differs from that of
18100 the destination label's context. */
18103 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18104 struct walk_stmt_info
*wi
)
18106 gimple
*context
= (gimple
*) wi
->info
;
18108 gimple
*stmt
= gsi_stmt (*gsi_p
);
18110 *handled_ops_p
= true;
18112 switch (gimple_code (stmt
))
18116 case GIMPLE_OMP_PARALLEL
:
18117 case GIMPLE_OMP_TASK
:
18118 case GIMPLE_OMP_SECTIONS
:
18119 case GIMPLE_OMP_SINGLE
:
18120 case GIMPLE_OMP_SECTION
:
18121 case GIMPLE_OMP_MASTER
:
18122 case GIMPLE_OMP_ORDERED
:
18123 case GIMPLE_OMP_CRITICAL
:
18124 case GIMPLE_OMP_TARGET
:
18125 case GIMPLE_OMP_TEAMS
:
18126 case GIMPLE_OMP_TASKGROUP
:
18128 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18129 wi
->info
= context
;
18132 case GIMPLE_OMP_FOR
:
18134 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18136 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
18137 diagnose_sb_2
, NULL
, wi
);
18138 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18139 wi
->info
= context
;
18144 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
18145 tree lab
= gimple_cond_true_label (cond_stmt
);
18148 n
= splay_tree_lookup (all_labels
,
18149 (splay_tree_key
) lab
);
18150 diagnose_sb_0 (gsi_p
, context
,
18151 n
? (gimple
*) n
->value
: NULL
);
18153 lab
= gimple_cond_false_label (cond_stmt
);
18156 n
= splay_tree_lookup (all_labels
,
18157 (splay_tree_key
) lab
);
18158 diagnose_sb_0 (gsi_p
, context
,
18159 n
? (gimple
*) n
->value
: NULL
);
18166 tree lab
= gimple_goto_dest (stmt
);
18167 if (TREE_CODE (lab
) != LABEL_DECL
)
18170 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18171 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
18175 case GIMPLE_SWITCH
:
18177 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
18179 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
18181 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
18182 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18183 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
18189 case GIMPLE_RETURN
:
18190 diagnose_sb_0 (gsi_p
, context
, NULL
);
18200 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18203 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
18206 gimple
*last
= last_stmt (bb
);
18207 enum gimple_code code
= gimple_code (last
);
18208 struct omp_region
*cur_region
= *region
;
18209 bool fallthru
= false;
18213 case GIMPLE_OMP_PARALLEL
:
18214 case GIMPLE_OMP_TASK
:
18215 case GIMPLE_OMP_FOR
:
18216 case GIMPLE_OMP_SINGLE
:
18217 case GIMPLE_OMP_TEAMS
:
18218 case GIMPLE_OMP_MASTER
:
18219 case GIMPLE_OMP_TASKGROUP
:
18220 case GIMPLE_OMP_CRITICAL
:
18221 case GIMPLE_OMP_SECTION
:
18222 case GIMPLE_OMP_GRID_BODY
:
18223 cur_region
= new_omp_region (bb
, code
, cur_region
);
18227 case GIMPLE_OMP_ORDERED
:
18228 cur_region
= new_omp_region (bb
, code
, cur_region
);
18230 if (find_omp_clause (gimple_omp_ordered_clauses
18231 (as_a
<gomp_ordered
*> (last
)),
18232 OMP_CLAUSE_DEPEND
))
18233 cur_region
= cur_region
->outer
;
18236 case GIMPLE_OMP_TARGET
:
18237 cur_region
= new_omp_region (bb
, code
, cur_region
);
18239 switch (gimple_omp_target_kind (last
))
18241 case GF_OMP_TARGET_KIND_REGION
:
18242 case GF_OMP_TARGET_KIND_DATA
:
18243 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
18244 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
18245 case GF_OMP_TARGET_KIND_OACC_DATA
:
18246 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
18248 case GF_OMP_TARGET_KIND_UPDATE
:
18249 case GF_OMP_TARGET_KIND_ENTER_DATA
:
18250 case GF_OMP_TARGET_KIND_EXIT_DATA
:
18251 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
18252 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
18253 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
18254 cur_region
= cur_region
->outer
;
18257 gcc_unreachable ();
18261 case GIMPLE_OMP_SECTIONS
:
18262 cur_region
= new_omp_region (bb
, code
, cur_region
);
18266 case GIMPLE_OMP_SECTIONS_SWITCH
:
18270 case GIMPLE_OMP_ATOMIC_LOAD
:
18271 case GIMPLE_OMP_ATOMIC_STORE
:
18275 case GIMPLE_OMP_RETURN
:
18276 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18277 somewhere other than the next block. This will be
18279 cur_region
->exit
= bb
;
18280 if (cur_region
->type
== GIMPLE_OMP_TASK
)
18281 /* Add an edge corresponding to not scheduling the task
18283 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
18284 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
18285 cur_region
= cur_region
->outer
;
18288 case GIMPLE_OMP_CONTINUE
:
18289 cur_region
->cont
= bb
;
18290 switch (cur_region
->type
)
18292 case GIMPLE_OMP_FOR
:
18293 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18294 succs edges as abnormal to prevent splitting
18296 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
18297 /* Make the loopback edge. */
18298 make_edge (bb
, single_succ (cur_region
->entry
),
18301 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18302 corresponds to the case that the body of the loop
18303 is not executed at all. */
18304 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
18305 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
18309 case GIMPLE_OMP_SECTIONS
:
18310 /* Wire up the edges into and out of the nested sections. */
18312 basic_block switch_bb
= single_succ (cur_region
->entry
);
18314 struct omp_region
*i
;
18315 for (i
= cur_region
->inner
; i
; i
= i
->next
)
18317 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
18318 make_edge (switch_bb
, i
->entry
, 0);
18319 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
18322 /* Make the loopback edge to the block with
18323 GIMPLE_OMP_SECTIONS_SWITCH. */
18324 make_edge (bb
, switch_bb
, 0);
18326 /* Make the edge from the switch to exit. */
18327 make_edge (switch_bb
, bb
->next_bb
, 0);
18332 case GIMPLE_OMP_TASK
:
18337 gcc_unreachable ();
18342 gcc_unreachable ();
18345 if (*region
!= cur_region
)
18347 *region
= cur_region
;
18349 *region_idx
= cur_region
->entry
->index
;
18357 static unsigned int
18358 diagnose_omp_structured_block_errors (void)
18360 struct walk_stmt_info wi
;
18361 gimple_seq body
= gimple_body (current_function_decl
);
18363 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
18365 memset (&wi
, 0, sizeof (wi
));
18366 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
18368 memset (&wi
, 0, sizeof (wi
));
18369 wi
.want_locations
= true;
18370 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
18372 gimple_set_body (current_function_decl
, body
);
18374 splay_tree_delete (all_labels
);
18382 const pass_data pass_data_diagnose_omp_blocks
=
18384 GIMPLE_PASS
, /* type */
18385 "*diagnose_omp_blocks", /* name */
18386 OPTGROUP_NONE
, /* optinfo_flags */
18387 TV_NONE
, /* tv_id */
18388 PROP_gimple_any
, /* properties_required */
18389 0, /* properties_provided */
18390 0, /* properties_destroyed */
18391 0, /* todo_flags_start */
18392 0, /* todo_flags_finish */
18395 class pass_diagnose_omp_blocks
: public gimple_opt_pass
18398 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18399 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
18402 /* opt_pass methods: */
18403 virtual bool gate (function
*)
18405 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
18407 virtual unsigned int execute (function
*)
18409 return diagnose_omp_structured_block_errors ();
18412 }; // class pass_diagnose_omp_blocks
18414 } // anon namespace
18417 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18419 return new pass_diagnose_omp_blocks (ctxt
);
18422 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18423 adds their addresses and sizes to constructor-vector V_CTOR. */
18425 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18426 vec
<constructor_elt
, va_gc
> *v_ctor
)
18428 unsigned len
= vec_safe_length (v_decls
);
18429 for (unsigned i
= 0; i
< len
; i
++)
18431 tree it
= (*v_decls
)[i
];
18432 bool is_var
= TREE_CODE (it
) == VAR_DECL
;
18435 #ifdef ACCEL_COMPILER
18436 && DECL_HAS_VALUE_EXPR_P (it
)
18438 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
18440 tree size
= NULL_TREE
;
18442 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
18446 addr
= build_fold_addr_expr (it
);
18449 #ifdef ACCEL_COMPILER
18450 /* For "omp declare target link" vars add address of the pointer to
18451 the target table, instead of address of the var. */
18452 tree value_expr
= DECL_VALUE_EXPR (it
);
18453 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
18454 varpool_node::finalize_decl (link_ptr_decl
);
18455 addr
= build_fold_addr_expr (link_ptr_decl
);
18457 addr
= build_fold_addr_expr (it
);
18460 /* Most significant bit of the size marks "omp declare target link"
18461 vars in host and target tables. */
18462 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
18463 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
18464 * BITS_PER_UNIT
- 1);
18465 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
18468 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
18470 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
18474 /* Create new symbols containing (address, size) pairs for global variables,
18475 marked with "omp declare target" attribute, as well as addresses for the
18476 functions, which are outlined offloading regions. */
18478 omp_finish_file (void)
18480 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18481 unsigned num_vars
= vec_safe_length (offload_vars
);
18483 if (num_funcs
== 0 && num_vars
== 0)
18486 if (targetm_common
.have_named_sections
)
18488 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18489 vec_alloc (v_f
, num_funcs
);
18490 vec_alloc (v_v
, num_vars
* 2);
18492 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18493 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18495 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18497 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18499 SET_TYPE_ALIGN (vars_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
18500 SET_TYPE_ALIGN (funcs_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
18501 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18502 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18503 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18504 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18505 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18506 get_identifier (".offload_func_table"),
18508 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18509 get_identifier (".offload_var_table"),
18511 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18512 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18513 otherwise a joint table in a binary will contain padding between
18514 tables from multiple object files. */
18515 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18516 SET_DECL_ALIGN (funcs_decl
, TYPE_ALIGN (funcs_decl_type
));
18517 SET_DECL_ALIGN (vars_decl
, TYPE_ALIGN (vars_decl_type
));
18518 DECL_INITIAL (funcs_decl
) = ctor_f
;
18519 DECL_INITIAL (vars_decl
) = ctor_v
;
18520 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18521 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18523 varpool_node::finalize_decl (vars_decl
);
18524 varpool_node::finalize_decl (funcs_decl
);
18528 for (unsigned i
= 0; i
< num_funcs
; i
++)
18530 tree it
= (*offload_funcs
)[i
];
18531 targetm
.record_offload_symbol (it
);
18533 for (unsigned i
= 0; i
< num_vars
; i
++)
18535 tree it
= (*offload_vars
)[i
];
18536 targetm
.record_offload_symbol (it
);
18541 /* Find the number of threads (POS = false), or thread number (POS =
18542 true) for an OpenACC region partitioned as MASK. Setup code
18543 required for the calculation is added to SEQ. */
18546 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18548 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18551 /* Start at gang level, and examine relevant dimension indices. */
18552 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18553 if (GOMP_DIM_MASK (ix
) & mask
)
18555 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18559 /* We had an outer index, so scale that by the size of
18561 tree n
= create_tmp_var (integer_type_node
);
18563 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18565 gimple_call_set_lhs (call
, n
);
18566 gimple_seq_add_stmt (seq
, call
);
18567 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18571 /* Determine index in this dimension. */
18572 tree id
= create_tmp_var (integer_type_node
);
18573 gimple
*call
= gimple_build_call_internal
18574 (IFN_GOACC_DIM_POS
, 1, arg
);
18576 gimple_call_set_lhs (call
, id
);
18577 gimple_seq_add_stmt (seq
, call
);
18579 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18585 if (res
== NULL_TREE
)
18586 res
= integer_zero_node
;
18591 /* Transform IFN_GOACC_LOOP calls to actual code. See
18592 expand_oacc_for for where these are generated. At the vector
18593 level, we stride loops, such that each member of a warp will
18594 operate on adjacent iterations. At the worker and gang level,
18595 each gang/warp executes a set of contiguous iterations. Chunking
18596 can override this such that each iteration engine executes a
18597 contiguous chunk, and then moves on to stride to the next chunk. */
18600 oacc_xform_loop (gcall
*call
)
18602 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18603 enum ifn_goacc_loop_kind code
18604 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18605 tree dir
= gimple_call_arg (call
, 1);
18606 tree range
= gimple_call_arg (call
, 2);
18607 tree step
= gimple_call_arg (call
, 3);
18608 tree chunk_size
= NULL_TREE
;
18609 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18610 tree lhs
= gimple_call_lhs (call
);
18611 tree type
= TREE_TYPE (lhs
);
18612 tree diff_type
= TREE_TYPE (range
);
18613 tree r
= NULL_TREE
;
18614 gimple_seq seq
= NULL
;
18615 bool chunking
= false, striding
= true;
18616 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18617 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18619 #ifdef ACCEL_COMPILER
18620 chunk_size
= gimple_call_arg (call
, 4);
18621 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18622 || integer_zerop (chunk_size
)) /* Default (also static). */
18624 /* If we're at the gang level, we want each to execute a
18625 contiguous run of iterations. Otherwise we want each element
18627 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18632 /* Chunk of size 1 is striding. */
18633 striding
= integer_onep (chunk_size
);
18634 chunking
= !striding
;
18638 /* striding=true, chunking=true
18640 striding=true, chunking=false
18642 striding=false,chunking=true
18643 -> chunks=ceil (range/(chunksize*threads*step))
18644 striding=false,chunking=false
18645 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18646 push_gimplify_context (true);
18650 default: gcc_unreachable ();
18652 case IFN_GOACC_LOOP_CHUNKS
:
18654 r
= build_int_cst (type
, 1);
18658 = (range - dir) / (chunks * step * num_threads) + dir */
18659 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18660 per
= fold_convert (type
, per
);
18661 chunk_size
= fold_convert (type
, chunk_size
);
18662 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18663 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18664 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18665 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18666 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18670 case IFN_GOACC_LOOP_STEP
:
18672 /* If striding, step by the entire compute volume, otherwise
18673 step by the inner volume. */
18674 unsigned volume
= striding
? mask
: inner_mask
;
18676 r
= oacc_thread_numbers (false, volume
, &seq
);
18677 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18681 case IFN_GOACC_LOOP_OFFSET
:
18684 r
= oacc_thread_numbers (true, mask
, &seq
);
18685 r
= fold_convert (diff_type
, r
);
18689 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18690 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18691 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18692 inner_size
, outer_size
);
18694 volume
= fold_convert (diff_type
, volume
);
18696 chunk_size
= fold_convert (diff_type
, chunk_size
);
18699 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18701 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18702 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18703 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18706 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18707 fold_convert (diff_type
, inner_size
));
18708 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18709 r
= fold_convert (diff_type
, r
);
18710 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18712 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18713 inner
= fold_convert (diff_type
, inner
);
18714 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18718 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18720 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18721 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18723 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18726 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18727 if (type
!= diff_type
)
18728 r
= fold_convert (type
, r
);
18731 case IFN_GOACC_LOOP_BOUND
:
18736 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18737 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18738 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18739 inner_size
, outer_size
);
18741 volume
= fold_convert (diff_type
, volume
);
18743 chunk_size
= fold_convert (diff_type
, chunk_size
);
18746 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18748 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18749 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18750 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18753 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18754 fold_convert (diff_type
, inner_size
));
18756 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18758 tree offset
= gimple_call_arg (call
, 6);
18759 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18760 fold_convert (diff_type
, offset
));
18761 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18762 diff_type
, r
, range
);
18764 if (diff_type
!= type
)
18765 r
= fold_convert (type
, r
);
18769 gimplify_assign (lhs
, r
, &seq
);
18771 pop_gimplify_context (NULL
);
18773 gsi_replace_with_seq (&gsi
, seq
, true);
18776 /* Default partitioned and minimum partitioned dimensions. */
18778 static int oacc_default_dims
[GOMP_DIM_MAX
];
18779 static int oacc_min_dims
[GOMP_DIM_MAX
];
18781 /* Parse the default dimension parameter. This is a set of
18782 :-separated optional compute dimensions. Each specified dimension
18783 is a positive integer. When device type support is added, it is
18784 planned to be a comma separated list of such compute dimensions,
18785 with all but the first prefixed by the colon-terminated device
18789 oacc_parse_default_dims (const char *dims
)
18793 for (ix
= GOMP_DIM_MAX
; ix
--;)
18795 oacc_default_dims
[ix
] = -1;
18796 oacc_min_dims
[ix
] = 1;
18799 #ifndef ACCEL_COMPILER
18800 /* Cannot be overridden on the host. */
18805 const char *pos
= dims
;
18807 for (ix
= 0; *pos
&& ix
!= GOMP_DIM_MAX
; ix
++)
18822 val
= strtol (pos
, CONST_CAST (char **, &eptr
), 10);
18823 if (errno
|| val
<= 0 || (int) val
!= val
)
18826 oacc_default_dims
[ix
] = (int) val
;
18832 error_at (UNKNOWN_LOCATION
,
18833 "-fopenacc-dim operand is malformed at '%s'", pos
);
18837 /* Allow the backend to validate the dimensions. */
18838 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_default_dims
, -1);
18839 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_min_dims
, -2);
18842 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18843 raw attribute. DIMS is an array of dimensions, which is filled in.
18844 LEVEL is the partitioning level of a routine, or -1 for an offload
18845 region itself. USED is the mask of partitioned execution in the
18849 oacc_validate_dims (tree fn
, tree attrs
, int *dims
, int level
, unsigned used
)
18851 tree purpose
[GOMP_DIM_MAX
];
18853 tree pos
= TREE_VALUE (attrs
);
18854 bool is_kernel
= oacc_fn_attrib_kernels_p (attrs
);
18856 /* Make sure the attribute creator attached the dimension
18860 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18862 purpose
[ix
] = TREE_PURPOSE (pos
);
18863 tree val
= TREE_VALUE (pos
);
18864 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18865 pos
= TREE_CHAIN (pos
);
18868 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, level
);
18870 /* Default anything left to 1 or a partitioned default. */
18871 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18874 /* The OpenACC spec says 'If the [num_gangs] clause is not
18875 specified, an implementation-defined default will be used;
18876 the default may depend on the code within the construct.'
18877 (2.5.6). Thus an implementation is free to choose
18878 non-unity default for a parallel region that doesn't have
18879 any gang-partitioned loops. However, it appears that there
18880 is a sufficient body of user code that expects non-gang
18881 partitioned regions to not execute in gang-redundant mode.
18882 So we (a) don't warn about the non-portability and (b) pick
18883 the minimum permissible dimension size when there is no
18884 partitioned execution. Otherwise we pick the global
18885 default for the dimension, which the user can control. The
18886 same wording and logic applies to num_workers and
18887 vector_length, however the worker- or vector- single
18888 execution doesn't have the same impact as gang-redundant
18889 execution. (If the minimum gang-level partioning is not 1,
18890 the target is probably too confusing.) */
18891 dims
[ix
] = (used
& GOMP_DIM_MASK (ix
)
18892 ? oacc_default_dims
[ix
] : oacc_min_dims
[ix
]);
18898 /* Replace the attribute with new values. */
18900 for (ix
= GOMP_DIM_MAX
; ix
--;)
18902 pos
= tree_cons (purpose
[ix
],
18903 build_int_cst (integer_type_node
, dims
[ix
]),
18906 TREE_PUBLIC (pos
) = 1;
18908 replace_oacc_fn_attrib (fn
, pos
);
18912 /* Create an empty OpenACC loop structure at LOC. */
18915 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18917 oacc_loop
*loop
= XCNEW (oacc_loop
);
18919 loop
->parent
= parent
;
18920 loop
->child
= loop
->sibling
= NULL
;
18924 loop
->sibling
= parent
->child
;
18925 parent
->child
= loop
;
18929 loop
->marker
= NULL
;
18930 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18931 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18932 loop
->routine
= NULL_TREE
;
18934 loop
->mask
= loop
->flags
= loop
->inner
= 0;
18936 loop
->chunk_size
= 0;
18937 loop
->head_end
= NULL
;
18942 /* Create an outermost, dummy OpenACC loop for offloaded function
18946 new_oacc_loop_outer (tree decl
)
18948 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18951 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18952 Link into PARENT loop. Return the new loop. */
18955 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18957 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18959 loop
->marker
= marker
;
18961 /* TODO: This is where device_type flattening would occur for the loop
18964 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18966 tree chunk_size
= integer_zero_node
;
18967 if (loop
->flags
& OLF_GANG_STATIC
)
18968 chunk_size
= gimple_call_arg (marker
, 4);
18969 loop
->chunk_size
= chunk_size
;
18974 /* Create a dummy loop encompassing a call to a openACC routine.
18975 Extract the routine's partitioning requirements. */
18978 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18980 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18981 int level
= oacc_fn_attrib_level (attrs
);
18983 gcc_assert (level
>= 0);
18985 loop
->marker
= call
;
18986 loop
->routine
= decl
;
18987 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18988 ^ (GOMP_DIM_MASK (level
) - 1));
18991 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18992 Return the parent loop. */
18995 finish_oacc_loop (oacc_loop
*loop
)
18997 /* If the loop has been collapsed, don't partition it. */
18999 loop
->mask
= loop
->flags
= 0;
19000 return loop
->parent
;
19003 /* Free all OpenACC loop structures within LOOP (inclusive). */
19006 free_oacc_loop (oacc_loop
*loop
)
19009 free_oacc_loop (loop
->sibling
);
19011 free_oacc_loop (loop
->child
);
19016 /* Dump out the OpenACC loop head or tail beginning at FROM. */
19019 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
19020 const char *title
, int level
)
19022 enum ifn_unique_kind kind
19023 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19025 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
19026 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19028 gimple
*stmt
= gsi_stmt (gsi
);
19030 if (is_gimple_call (stmt
)
19031 && gimple_call_internal_p (stmt
)
19032 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19034 enum ifn_unique_kind k
19035 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
19036 (gimple_call_arg (stmt
, 0)));
19038 if (k
== kind
&& stmt
!= from
)
19041 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
19044 while (gsi_end_p (gsi
))
19045 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19049 /* Dump OpenACC loops LOOP, its siblings and its children. */
19052 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
19056 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
19057 loop
->flags
, loop
->mask
,
19058 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
19061 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
19064 fprintf (file
, "%*sRoutine %s:%u:%s\n",
19065 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
19066 DECL_SOURCE_LINE (loop
->routine
),
19067 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
19069 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
19070 if (loop
->heads
[ix
])
19071 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
19072 for (ix
= GOMP_DIM_MAX
; ix
--;)
19073 if (loop
->tails
[ix
])
19074 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
19077 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
19079 dump_oacc_loop (file
, loop
->sibling
, depth
);
19082 void debug_oacc_loop (oacc_loop
*);
19084 /* Dump loops to stderr. */
19086 DEBUG_FUNCTION
void
19087 debug_oacc_loop (oacc_loop
*loop
)
19089 dump_oacc_loop (stderr
, loop
, 0);
19092 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
19093 structures as we go. By construction these loops are properly
19097 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
19102 if (bb
->flags
& BB_VISITED
)
19106 bb
->flags
|= BB_VISITED
;
19108 /* Scan for loop markers. */
19109 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
19112 gimple
*stmt
= gsi_stmt (gsi
);
19114 if (!is_gimple_call (stmt
))
19117 gcall
*call
= as_a
<gcall
*> (stmt
);
19119 /* If this is a routine, make a dummy loop for it. */
19120 if (tree decl
= gimple_call_fndecl (call
))
19121 if (tree attrs
= get_oacc_fn_attrib (decl
))
19123 gcc_assert (!marker
);
19124 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
19127 if (!gimple_call_internal_p (call
))
19130 switch (gimple_call_internal_fn (call
))
19135 case IFN_GOACC_LOOP
:
19136 /* Count the goacc loop abstraction fns, to determine if the
19137 loop was collapsed already. */
19142 enum ifn_unique_kind kind
19143 = (enum ifn_unique_kind
) (TREE_INT_CST_LOW
19144 (gimple_call_arg (call
, 0)));
19145 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
19146 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19148 if (gimple_call_num_args (call
) == 2)
19150 gcc_assert (marker
&& !remaining
);
19152 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19153 loop
= finish_oacc_loop (loop
);
19155 loop
->head_end
= call
;
19159 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
19163 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19164 loop
= new_oacc_loop (loop
, call
);
19167 gcc_assert (count
== remaining
);
19171 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19172 loop
->heads
[marker
] = call
;
19174 loop
->tails
[remaining
] = call
;
19181 if (remaining
|| marker
)
19183 bb
= single_succ (bb
);
19184 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
19188 /* Walk successor blocks. */
19192 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
19193 oacc_loop_discover_walk (loop
, e
->dest
);
19196 /* LOOP is the first sibling. Reverse the order in place and return
19197 the new first sibling. Recurse to child loops. */
19200 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
19202 oacc_loop
*last
= NULL
;
19206 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
19208 oacc_loop
*next
= loop
->sibling
;
19209 loop
->sibling
= last
;
19218 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19219 the current function. */
19222 oacc_loop_discovery ()
19226 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19227 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19229 /* The siblings were constructed in reverse order, reverse them so
19230 that diagnostics come out in an unsurprising order. */
19231 top
= oacc_loop_sibling_nreverse (top
);
19233 /* Reset the visited flags. */
19234 FOR_ALL_BB_FN (bb
, cfun
)
19235 bb
->flags
&= ~BB_VISITED
;
19240 /* Transform the abstract internal function markers starting at FROM
19241 to be for partitioning level LEVEL. Stop when we meet another HEAD
19245 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19247 enum ifn_unique_kind kind
19248 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19249 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19251 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19253 gimple
*stmt
= gsi_stmt (gsi
);
19255 if (is_gimple_call (stmt
)
19256 && gimple_call_internal_p (stmt
)
19257 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19259 enum ifn_unique_kind k
19260 = ((enum ifn_unique_kind
)
19261 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19263 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19264 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19265 else if (k
== kind
&& stmt
!= from
)
19268 else if (is_gimple_call (stmt
)
19269 && gimple_call_internal_p (stmt
)
19270 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
19271 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
19274 while (gsi_end_p (gsi
))
19275 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19279 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19280 determined partitioning mask and chunking argument. END_MARKER
19281 points at the end IFN_HEAD_TAIL call intgroducing the loop. IFNS
19282 is the number of IFN_GOACC_LOOP calls for the loop. MASK_ARG is
19283 the replacement partitioning mask and CHUNK_ARG is the replacement
19287 oacc_loop_xform_loop (gcall
*end_marker
, unsigned ifns
,
19288 tree mask_arg
, tree chunk_arg
)
19290 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19292 gcc_checking_assert (ifns
);
19295 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19297 gimple
*stmt
= gsi_stmt (gsi
);
19299 if (!is_gimple_call (stmt
))
19302 gcall
*call
= as_a
<gcall
*> (stmt
);
19304 if (!gimple_call_internal_p (call
))
19307 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19310 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19311 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19317 /* The LOOP_BOUND ifn could be in the single successor
19319 basic_block bb
= single_succ (gsi_bb (gsi
));
19320 gsi
= gsi_start_bb (bb
);
19324 /* Process the discovered OpenACC loops, setting the correct
19325 partitioning level etc. */
19328 oacc_loop_process (oacc_loop
*loop
)
19331 oacc_loop_process (loop
->child
);
19333 if (loop
->mask
&& !loop
->routine
)
19336 unsigned mask
= loop
->mask
;
19337 unsigned dim
= GOMP_DIM_GANG
;
19338 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19339 tree chunk_arg
= loop
->chunk_size
;
19341 oacc_loop_xform_loop (loop
->head_end
, loop
->ifns
, mask_arg
, chunk_arg
);
19343 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& mask
; ix
++)
19345 while (!(GOMP_DIM_MASK (dim
) & mask
))
19348 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19349 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19351 mask
^= GOMP_DIM_MASK (dim
);
19356 oacc_loop_process (loop
->sibling
);
19359 /* Walk the OpenACC loop heirarchy checking and assigning the
19360 programmer-specified partitionings. OUTER_MASK is the partitioning
19361 this loop is contained within. Return mask of partitioning
19362 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
19366 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19368 unsigned this_mask
= loop
->mask
;
19369 unsigned mask_all
= 0;
19372 #ifdef ACCEL_COMPILER
19373 /* When device_type is supported, we want the device compiler to be
19374 noisy, if the loop parameters are device_type-specific. */
19378 if (!loop
->routine
)
19380 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19381 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19383 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19384 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19386 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19389 error_at (loop
->loc
,
19391 ? "%<seq%> overrides other OpenACC loop specifiers"
19392 : "%<auto%> conflicts with other OpenACC loop specifiers");
19394 loop
->flags
&= ~OLF_AUTO
;
19398 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19402 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19403 mask_all
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19406 if (this_mask
& outer_mask
)
19408 const oacc_loop
*outer
;
19409 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19410 if (outer
->mask
& this_mask
)
19417 error_at (loop
->loc
,
19418 "%s uses same OpenACC parallelism as containing loop",
19419 loop
->routine
? "routine call" : "inner loop");
19420 inform (outer
->loc
, "containing loop here");
19423 error_at (loop
->loc
,
19424 "%s uses OpenACC parallelism disallowed by containing routine",
19425 loop
->routine
? "routine call" : "loop");
19428 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19429 "routine %qD declared here", loop
->routine
);
19431 this_mask
&= ~outer_mask
;
19435 unsigned outermost
= this_mask
& -this_mask
;
19437 if (outermost
&& outermost
<= outer_mask
)
19441 error_at (loop
->loc
,
19442 "incorrectly nested OpenACC loop parallelism");
19444 const oacc_loop
*outer
;
19445 for (outer
= loop
->parent
;
19446 outer
->flags
&& outer
->flags
< outermost
;
19447 outer
= outer
->parent
)
19449 inform (outer
->loc
, "containing loop here");
19452 this_mask
&= ~outermost
;
19456 loop
->mask
= this_mask
;
19457 mask_all
|= this_mask
;
19461 loop
->inner
= oacc_loop_fixed_partitions (loop
->child
,
19462 outer_mask
| this_mask
);
19463 mask_all
|= loop
->inner
;
19467 mask_all
|= oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
);
19472 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
19473 OUTER_MASK is the partitioning this loop is contained within.
19474 Return the cumulative partitioning used by this loop, siblings and
19478 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19480 bool assign
= (loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
);
19483 #ifdef ACCEL_COMPILER
19484 /* When device_type is supported, we want the device compiler to be
19485 noisy, if the loop parameters are device_type-specific. */
19489 if (assign
&& outer_mask
< GOMP_DIM_MASK (GOMP_DIM_MAX
- 1))
19491 /* Allocate the outermost loop at the outermost available
19493 unsigned this_mask
= outer_mask
+ 1;
19495 if (!(this_mask
& loop
->inner
))
19496 loop
->mask
= this_mask
;
19501 unsigned child_mask
= outer_mask
| loop
->mask
;
19503 if (loop
->mask
|| assign
)
19504 child_mask
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19506 loop
->inner
= oacc_loop_auto_partitions (loop
->child
, child_mask
);
19509 if (assign
&& !loop
->mask
)
19511 /* Allocate the loop at the innermost available level. */
19512 unsigned this_mask
= 0;
19514 /* Determine the outermost partitioning used within this loop. */
19515 this_mask
= loop
->inner
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
19516 this_mask
= (this_mask
& -this_mask
);
19518 /* Pick the partitioning just inside that one. */
19521 /* And avoid picking one use by an outer loop. */
19522 this_mask
&= ~outer_mask
;
19524 if (!this_mask
&& noisy
)
19525 warning_at (loop
->loc
, 0,
19526 "insufficient partitioning available to parallelize loop");
19528 loop
->mask
= this_mask
;
19531 if (assign
&& dump_file
)
19532 fprintf (dump_file
, "Auto loop %s:%d assigned %d\n",
19533 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
19536 unsigned inner_mask
= 0;
19539 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
19541 inner_mask
|= loop
->inner
| loop
->mask
;
19546 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19547 axes. Return mask of partitioning. */
19550 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
19552 unsigned mask_all
= oacc_loop_fixed_partitions (loop
, outer_mask
);
19554 if (mask_all
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
19556 mask_all
^= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19557 mask_all
|= oacc_loop_auto_partitions (loop
, outer_mask
);
19562 /* Default fork/join early expander. Delete the function calls if
19563 there is no RTL expander. */
19566 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19567 const int *ARG_UNUSED (dims
), bool is_fork
)
19570 return targetm
.have_oacc_fork ();
19572 return targetm
.have_oacc_join ();
19575 /* Default goacc.reduction early expander.
19577 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19578 If RES_PTR is not integer-zerop:
19579 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19580 TEARDOWN - emit '*RES_PTR = VAR'
19582 emit 'LHS = VAR' */
19585 default_goacc_reduction (gcall
*call
)
19587 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19588 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19589 tree lhs
= gimple_call_lhs (call
);
19590 tree var
= gimple_call_arg (call
, 2);
19591 gimple_seq seq
= NULL
;
19593 if (code
== IFN_GOACC_REDUCTION_SETUP
19594 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19596 /* Setup and Teardown need to copy from/to the receiver object,
19597 if there is one. */
19598 tree ref_to_res
= gimple_call_arg (call
, 1);
19600 if (!integer_zerop (ref_to_res
))
19602 tree dst
= build_simple_mem_ref (ref_to_res
);
19605 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19611 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19615 /* Copy VAR to LHS, if there is an LHS. */
19617 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19619 gsi_replace_with_seq (&gsi
, seq
, true);
19622 /* Main entry point for oacc transformations which run on the device
19623 compiler after LTO, so we know what the target device is at this
19624 point (including the host fallback). */
19626 static unsigned int
19627 execute_oacc_device_lower ()
19629 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19632 /* Not an offloaded function. */
19635 /* Parse the default dim argument exactly once. */
19636 if ((const void *)flag_openacc_dims
!= &flag_openacc_dims
)
19638 oacc_parse_default_dims (flag_openacc_dims
);
19639 flag_openacc_dims
= (char *)&flag_openacc_dims
;
19642 /* Discover, partition and process the loops. */
19643 oacc_loop
*loops
= oacc_loop_discovery ();
19644 int fn_level
= oacc_fn_attrib_level (attrs
);
19647 fprintf (dump_file
, oacc_fn_attrib_kernels_p (attrs
)
19648 ? "Function is kernels offload\n"
19649 : fn_level
< 0 ? "Function is parallel offload\n"
19650 : "Function is routine level %d\n", fn_level
);
19652 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
19653 unsigned used_mask
= oacc_loop_partition (loops
, outer_mask
);
19654 int dims
[GOMP_DIM_MAX
];
19656 oacc_validate_dims (current_function_decl
, attrs
, dims
, fn_level
, used_mask
);
19660 const char *comma
= "Compute dimensions [";
19661 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, comma
= ", ")
19662 fprintf (dump_file
, "%s%d", comma
, dims
[ix
]);
19663 fprintf (dump_file
, "]\n");
19666 oacc_loop_process (loops
);
19669 fprintf (dump_file
, "OpenACC loops\n");
19670 dump_oacc_loop (dump_file
, loops
, 0);
19671 fprintf (dump_file
, "\n");
19674 /* Offloaded targets may introduce new basic blocks, which require
19675 dominance information to update SSA. */
19676 calculate_dominance_info (CDI_DOMINATORS
);
19678 /* Now lower internal loop functions to target-specific code
19681 FOR_ALL_BB_FN (bb
, cfun
)
19682 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19684 gimple
*stmt
= gsi_stmt (gsi
);
19685 if (!is_gimple_call (stmt
))
19691 gcall
*call
= as_a
<gcall
*> (stmt
);
19692 if (!gimple_call_internal_p (call
))
19698 /* Rewind to allow rescan. */
19700 bool rescan
= false, remove
= false;
19701 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19707 case IFN_GOACC_LOOP
:
19708 oacc_xform_loop (call
);
19712 case IFN_GOACC_REDUCTION
:
19713 /* Mark the function for SSA renaming. */
19714 mark_virtual_operands_for_renaming (cfun
);
19716 /* If the level is -1, this ended up being an unused
19717 axis. Handle as a default. */
19718 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19719 default_goacc_reduction (call
);
19721 targetm
.goacc
.reduction (call
);
19727 enum ifn_unique_kind kind
19728 = ((enum ifn_unique_kind
)
19729 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19734 gcc_unreachable ();
19736 case IFN_UNIQUE_OACC_FORK
:
19737 case IFN_UNIQUE_OACC_JOIN
:
19738 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19740 else if (!targetm
.goacc
.fork_join
19741 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19745 case IFN_UNIQUE_OACC_HEAD_MARK
:
19746 case IFN_UNIQUE_OACC_TAIL_MARK
:
19754 if (gsi_end_p (gsi
))
19755 /* We rewound past the beginning of the BB. */
19756 gsi
= gsi_start_bb (bb
);
19758 /* Undo the rewind. */
19763 if (gimple_vdef (call
))
19764 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19765 if (gimple_call_lhs (call
))
19767 /* Propagate the data dependency var. */
19768 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19769 gimple_call_arg (call
, 1));
19770 gsi_replace (&gsi
, ass
, false);
19773 gsi_remove (&gsi
, true);
19776 /* If not rescanning, advance over the call. */
19780 free_oacc_loop (loops
);
19785 /* Default launch dimension validator. Force everything to 1. A
19786 backend that wants to provide larger dimensions must override this
19790 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19791 int ARG_UNUSED (fn_level
))
19793 bool changed
= false;
19795 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19807 /* Default dimension bound is unknown on accelerator and 1 on host. */
19810 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19812 #ifdef ACCEL_COMPILER
19821 const pass_data pass_data_oacc_device_lower
=
19823 GIMPLE_PASS
, /* type */
19824 "oaccdevlow", /* name */
19825 OPTGROUP_NONE
, /* optinfo_flags */
19826 TV_NONE
, /* tv_id */
19827 PROP_cfg
, /* properties_required */
19828 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19829 0, /* properties_destroyed */
19830 0, /* todo_flags_start */
19831 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19834 class pass_oacc_device_lower
: public gimple_opt_pass
19837 pass_oacc_device_lower (gcc::context
*ctxt
)
19838 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19841 /* opt_pass methods: */
19842 virtual unsigned int execute (function
*)
19844 bool gate
= flag_openacc
!= 0;
19849 return execute_oacc_device_lower ();
19852 }; // class pass_oacc_device_lower
19854 } // anon namespace
19857 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19859 return new pass_oacc_device_lower (ctxt
);
19862 /* "omp declare target link" handling pass. */
19866 const pass_data pass_data_omp_target_link
=
19868 GIMPLE_PASS
, /* type */
19869 "omptargetlink", /* name */
19870 OPTGROUP_NONE
, /* optinfo_flags */
19871 TV_NONE
, /* tv_id */
19872 PROP_ssa
, /* properties_required */
19873 0, /* properties_provided */
19874 0, /* properties_destroyed */
19875 0, /* todo_flags_start */
19876 TODO_update_ssa
, /* todo_flags_finish */
19879 class pass_omp_target_link
: public gimple_opt_pass
19882 pass_omp_target_link (gcc::context
*ctxt
)
19883 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
19886 /* opt_pass methods: */
19887 virtual bool gate (function
*fun
)
19889 #ifdef ACCEL_COMPILER
19890 tree attrs
= DECL_ATTRIBUTES (fun
->decl
);
19891 return lookup_attribute ("omp declare target", attrs
)
19892 || lookup_attribute ("omp target entrypoint", attrs
);
19899 virtual unsigned execute (function
*);
19902 /* Callback for walk_gimple_stmt used to scan for link var operands. */
19905 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
19909 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
)
19910 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
19912 *walk_subtrees
= 0;
19920 pass_omp_target_link::execute (function
*fun
)
19923 FOR_EACH_BB_FN (bb
, fun
)
19925 gimple_stmt_iterator gsi
;
19926 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
19927 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
19928 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
19934 } // anon namespace
19937 make_pass_omp_target_link (gcc::context
*ctxt
)
19939 return new pass_omp_target_link (ctxt
);
19942 #include "gt-omp-low.h"