1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region
*outer
;
107 /* First child region. */
108 struct omp_region
*inner
;
110 /* Next peer region. */
111 struct omp_region
*next
;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec
<tree
, va_gc
> *ws_args
;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type
;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind
;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered
*ord_stmt
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind
;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v
, n1
, n2
, step
;
204 enum tree_code cond_code
;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop
;
217 bool have_nowait
, have_ordered
, simd_schedule
;
218 unsigned char sched_modifiers
;
219 enum omp_clause_schedule_kind sched_kind
;
220 struct omp_for_data_loop
*loops
;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop
*parent
; /* Containing loop. */
230 oacc_loop
*child
; /* First inner loop. */
232 oacc_loop
*sibling
; /* Next loop within same parent. */
234 location_t loc
; /* Location of the loop start. */
236 gcall
*marker
; /* Initial head marker. */
238 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
239 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
241 tree routine
; /* Pseudo-loop enclosing a routine. */
243 unsigned mask
; /* Partitioning mask. */
244 unsigned flags
; /* Partitioning flags. */
245 unsigned ifns
; /* Contained loop abstraction functions. */
246 tree chunk_size
; /* Chunk size. */
247 gcall
*head_end
; /* Final marker of head sequence. */
250 /* Flags for an OpenACC loop. */
252 enum oacc_loop_flags
{
253 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
254 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
255 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
256 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
258 /* Explicitly specified loop axes. */
260 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
261 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
262 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
264 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
268 static splay_tree all_contexts
;
269 static int taskreg_nesting_level
;
270 static int target_nesting_level
;
271 static struct omp_region
*root_omp_region
;
272 static bitmap task_shared_vars
;
273 static vec
<omp_context
*> taskreg_contexts
;
274 static bool omp_any_child_fn_dumped
;
276 static void scan_omp (gimple_seq
*, omp_context
*);
277 static tree
scan_omp_1_op (tree
*, int *, void *);
278 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
280 #define WALK_SUBSTMTS \
284 case GIMPLE_EH_FILTER: \
285 case GIMPLE_TRANSACTION: \
286 /* The sub-statements for these should be walked. */ \
287 *handled_ops_p = false; \
290 /* Return true if CTX corresponds to an oacc parallel region. */
293 is_oacc_parallel (omp_context
*ctx
)
295 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
296 return ((outer_type
== GIMPLE_OMP_TARGET
)
297 && (gimple_omp_target_kind (ctx
->stmt
)
298 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
301 /* Return true if CTX corresponds to an oacc kernels region. */
304 is_oacc_kernels (omp_context
*ctx
)
306 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
307 return ((outer_type
== GIMPLE_OMP_TARGET
)
308 && (gimple_omp_target_kind (ctx
->stmt
)
309 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
312 /* If DECL is the artificial dummy VAR_DECL created for non-static
313 data member privatization, return the underlying "this" parameter,
314 otherwise return NULL. */
317 omp_member_access_dummy_var (tree decl
)
320 || !DECL_ARTIFICIAL (decl
)
321 || !DECL_IGNORED_P (decl
)
322 || !DECL_HAS_VALUE_EXPR_P (decl
)
323 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
326 tree v
= DECL_VALUE_EXPR (decl
);
327 if (TREE_CODE (v
) != COMPONENT_REF
)
331 switch (TREE_CODE (v
))
337 case POINTER_PLUS_EXPR
:
338 v
= TREE_OPERAND (v
, 0);
341 if (DECL_CONTEXT (v
) == current_function_decl
342 && DECL_ARTIFICIAL (v
)
343 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
351 /* Helper for unshare_and_remap, called through walk_tree. */
354 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
356 tree
*pair
= (tree
*) data
;
359 *tp
= unshare_expr (pair
[1]);
362 else if (IS_TYPE_OR_DECL_P (*tp
))
367 /* Return unshare_expr (X) with all occurrences of FROM
371 unshare_and_remap (tree x
, tree from
, tree to
)
373 tree pair
[2] = { from
, to
};
374 x
= unshare_expr (x
);
375 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
379 /* Holds offload tables with decls. */
380 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
382 /* Convenience function for calling scan_omp_1_op on tree operands. */
385 scan_omp_op (tree
*tp
, omp_context
*ctx
)
387 struct walk_stmt_info wi
;
389 memset (&wi
, 0, sizeof (wi
));
391 wi
.want_locations
= true;
393 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
396 static void lower_omp (gimple_seq
*, omp_context
*);
397 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
398 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
400 /* Find an OMP clause of type KIND within CLAUSES. */
403 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
405 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
406 if (OMP_CLAUSE_CODE (clauses
) == kind
)
412 /* Return true if CTX is for an omp parallel. */
415 is_parallel_ctx (omp_context
*ctx
)
417 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
421 /* Return true if CTX is for an omp task. */
424 is_task_ctx (omp_context
*ctx
)
426 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
430 /* Return true if CTX is for an omp taskloop. */
433 is_taskloop_ctx (omp_context
*ctx
)
435 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
436 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
440 /* Return true if CTX is for an omp parallel or omp task. */
443 is_taskreg_ctx (omp_context
*ctx
)
445 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
449 /* Return true if REGION is a combined parallel+workshare region. */
452 is_combined_parallel (struct omp_region
*region
)
454 return region
->is_combined_parallel
;
457 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
461 adjust_for_condition (location_t loc
, enum tree_code
*cond_code
, tree
*n2
)
470 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
471 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, 1);
473 *n2
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (*n2
), *n2
,
474 build_int_cst (TREE_TYPE (*n2
), 1));
475 *cond_code
= LT_EXPR
;
478 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
479 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, -1);
481 *n2
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (*n2
), *n2
,
482 build_int_cst (TREE_TYPE (*n2
), 1));
483 *cond_code
= GT_EXPR
;
490 /* Return the looping step from INCR, extracted from the step of a gimple omp
494 get_omp_for_step_from_incr (location_t loc
, tree incr
)
497 switch (TREE_CODE (incr
))
500 step
= TREE_OPERAND (incr
, 1);
502 case POINTER_PLUS_EXPR
:
503 step
= fold_convert (ssizetype
, TREE_OPERAND (incr
, 1));
506 step
= TREE_OPERAND (incr
, 1);
507 step
= fold_build1_loc (loc
, NEGATE_EXPR
, TREE_TYPE (step
), step
);
515 /* Extract the header elements of parallel loop FOR_STMT and store
519 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
520 struct omp_for_data_loop
*loops
)
522 tree t
, var
, *collapse_iter
, *collapse_count
;
523 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
524 struct omp_for_data_loop
*loop
;
526 struct omp_for_data_loop dummy_loop
;
527 location_t loc
= gimple_location (for_stmt
);
528 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
529 bool distribute
= gimple_omp_for_kind (for_stmt
)
530 == GF_OMP_FOR_KIND_DISTRIBUTE
;
531 bool taskloop
= gimple_omp_for_kind (for_stmt
)
532 == GF_OMP_FOR_KIND_TASKLOOP
;
535 fd
->for_stmt
= for_stmt
;
537 if (gimple_omp_for_collapse (for_stmt
) > 1)
540 fd
->loops
= &fd
->loop
;
542 fd
->have_nowait
= distribute
|| simd
;
543 fd
->have_ordered
= false;
546 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
547 fd
->sched_modifiers
= 0;
548 fd
->chunk_size
= NULL_TREE
;
549 fd
->simd_schedule
= false;
550 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
551 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
552 collapse_iter
= NULL
;
553 collapse_count
= NULL
;
555 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
556 switch (OMP_CLAUSE_CODE (t
))
558 case OMP_CLAUSE_NOWAIT
:
559 fd
->have_nowait
= true;
561 case OMP_CLAUSE_ORDERED
:
562 fd
->have_ordered
= true;
563 if (OMP_CLAUSE_ORDERED_EXPR (t
))
564 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
566 case OMP_CLAUSE_SCHEDULE
:
567 gcc_assert (!distribute
&& !taskloop
);
569 = (enum omp_clause_schedule_kind
)
570 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
571 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
572 & ~OMP_CLAUSE_SCHEDULE_MASK
);
573 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
574 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
576 case OMP_CLAUSE_DIST_SCHEDULE
:
577 gcc_assert (distribute
);
578 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
580 case OMP_CLAUSE_COLLAPSE
:
581 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
582 if (fd
->collapse
> 1)
584 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
585 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
591 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
596 collapse_iter
= &iterv
;
597 collapse_count
= &countv
;
600 /* FIXME: for now map schedule(auto) to schedule(static).
601 There should be analysis to determine whether all iterations
602 are approximately the same amount of work (then schedule(static)
603 is best) or if it varies (then schedule(dynamic,N) is better). */
604 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
606 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
607 gcc_assert (fd
->chunk_size
== NULL
);
609 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
611 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
612 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
613 gcc_assert (fd
->chunk_size
== NULL
);
614 else if (fd
->chunk_size
== NULL
)
616 /* We only need to compute a default chunk size for ordered
617 static loops and dynamic loops. */
618 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
620 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
621 ? integer_zero_node
: integer_one_node
;
624 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
625 for (i
= 0; i
< cnt
; i
++)
627 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
629 else if (loops
!= NULL
)
634 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
635 gcc_assert (SSA_VAR_P (loop
->v
));
636 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
637 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
638 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
639 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
641 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
642 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
643 gcc_assert (loop
->cond_code
!= NE_EXPR
644 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKSIMD
645 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
);
646 adjust_for_condition (loc
, &loop
->cond_code
, &loop
->n2
);
648 t
= gimple_omp_for_incr (for_stmt
, i
);
649 gcc_assert (TREE_OPERAND (t
, 0) == var
);
650 loop
->step
= get_omp_for_step_from_incr (loc
, t
);
653 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
654 && !fd
->have_ordered
))
656 if (fd
->collapse
== 1)
657 iter_type
= TREE_TYPE (loop
->v
);
659 || TYPE_PRECISION (iter_type
)
660 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
662 = build_nonstandard_integer_type
663 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
665 else if (iter_type
!= long_long_unsigned_type_node
)
667 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
668 iter_type
= long_long_unsigned_type_node
;
669 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
670 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
671 >= TYPE_PRECISION (iter_type
))
675 if (loop
->cond_code
== LT_EXPR
)
676 n
= fold_build2_loc (loc
,
677 PLUS_EXPR
, TREE_TYPE (loop
->v
),
678 loop
->n2
, loop
->step
);
681 if (TREE_CODE (n
) != INTEGER_CST
682 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
683 iter_type
= long_long_unsigned_type_node
;
685 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
686 > TYPE_PRECISION (iter_type
))
690 if (loop
->cond_code
== LT_EXPR
)
693 n2
= fold_build2_loc (loc
,
694 PLUS_EXPR
, TREE_TYPE (loop
->v
),
695 loop
->n2
, loop
->step
);
699 n1
= fold_build2_loc (loc
,
700 MINUS_EXPR
, TREE_TYPE (loop
->v
),
701 loop
->n2
, loop
->step
);
704 if (TREE_CODE (n1
) != INTEGER_CST
705 || TREE_CODE (n2
) != INTEGER_CST
706 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
707 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
708 iter_type
= long_long_unsigned_type_node
;
712 if (i
>= fd
->collapse
)
715 if (collapse_count
&& *collapse_count
== NULL
)
717 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
718 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
719 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
720 if (t
&& integer_zerop (t
))
721 count
= build_zero_cst (long_long_unsigned_type_node
);
722 else if ((i
== 0 || count
!= NULL_TREE
)
723 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
724 && TREE_CONSTANT (loop
->n1
)
725 && TREE_CONSTANT (loop
->n2
)
726 && TREE_CODE (loop
->step
) == INTEGER_CST
)
728 tree itype
= TREE_TYPE (loop
->v
);
730 if (POINTER_TYPE_P (itype
))
731 itype
= signed_type_for (itype
);
732 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
733 t
= fold_build2_loc (loc
,
735 fold_convert_loc (loc
, itype
, loop
->step
), t
);
736 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
737 fold_convert_loc (loc
, itype
, loop
->n2
));
738 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
739 fold_convert_loc (loc
, itype
, loop
->n1
));
740 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
741 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
742 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
743 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
744 fold_convert_loc (loc
, itype
,
747 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
748 fold_convert_loc (loc
, itype
, loop
->step
));
749 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
750 if (count
!= NULL_TREE
)
751 count
= fold_build2_loc (loc
,
752 MULT_EXPR
, long_long_unsigned_type_node
,
756 if (TREE_CODE (count
) != INTEGER_CST
)
759 else if (count
&& !integer_zerop (count
))
766 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
767 || fd
->have_ordered
))
769 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
770 iter_type
= long_long_unsigned_type_node
;
772 iter_type
= long_integer_type_node
;
774 else if (collapse_iter
&& *collapse_iter
!= NULL
)
775 iter_type
= TREE_TYPE (*collapse_iter
);
776 fd
->iter_type
= iter_type
;
777 if (collapse_iter
&& *collapse_iter
== NULL
)
778 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
779 if (collapse_count
&& *collapse_count
== NULL
)
782 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
784 *collapse_count
= create_tmp_var (iter_type
, ".count");
787 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
789 fd
->loop
.v
= *collapse_iter
;
790 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
791 fd
->loop
.n2
= *collapse_count
;
792 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
793 fd
->loop
.cond_code
= LT_EXPR
;
800 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
801 is the immediate dominator of PAR_ENTRY_BB, return true if there
802 are no data dependencies that would prevent expanding the parallel
803 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
805 When expanding a combined parallel+workshare region, the call to
806 the child function may need additional arguments in the case of
807 GIMPLE_OMP_FOR regions. In some cases, these arguments are
808 computed out of variables passed in from the parent to the child
809 via 'struct .omp_data_s'. For instance:
811 #pragma omp parallel for schedule (guided, i * 4)
816 # BLOCK 2 (PAR_ENTRY_BB)
818 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
820 # BLOCK 3 (WS_ENTRY_BB)
821 .omp_data_i = &.omp_data_o;
822 D.1667 = .omp_data_i->i;
824 #pragma omp for schedule (guided, D.1598)
826 When we outline the parallel region, the call to the child function
827 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
828 that value is computed *after* the call site. So, in principle we
829 cannot do the transformation.
831 To see whether the code in WS_ENTRY_BB blocks the combined
832 parallel+workshare call, we collect all the variables used in the
833 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
834 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
837 FIXME. If we had the SSA form built at this point, we could merely
838 hoist the code in block 3 into block 2 and be done with it. But at
839 this point we don't have dataflow information and though we could
840 hack something up here, it is really not worth the aggravation. */
843 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
845 struct omp_for_data fd
;
846 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
848 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
851 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
853 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
855 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
857 if (fd
.iter_type
!= long_integer_type_node
)
860 /* FIXME. We give up too easily here. If any of these arguments
861 are not constants, they will likely involve variables that have
862 been mapped into fields of .omp_data_s for sharing with the child
863 function. With appropriate data flow, it would be possible to
865 if (!is_gimple_min_invariant (fd
.loop
.n1
)
866 || !is_gimple_min_invariant (fd
.loop
.n2
)
867 || !is_gimple_min_invariant (fd
.loop
.step
)
868 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
875 static int omp_max_vf (void);
877 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
878 presence (SIMD_SCHEDULE). */
881 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
886 int vf
= omp_max_vf ();
890 tree type
= TREE_TYPE (chunk_size
);
891 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
892 build_int_cst (type
, vf
- 1));
893 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
894 build_int_cst (type
, -vf
));
898 /* Collect additional arguments needed to emit a combined
899 parallel+workshare call. WS_STMT is the workshare directive being
902 static vec
<tree
, va_gc
> *
903 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
906 location_t loc
= gimple_location (ws_stmt
);
907 vec
<tree
, va_gc
> *ws_args
;
909 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
911 struct omp_for_data fd
;
914 extract_omp_for_data (for_stmt
, &fd
, NULL
);
918 if (gimple_omp_for_combined_into_p (for_stmt
))
921 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
922 OMP_CLAUSE__LOOPTEMP_
);
924 n1
= OMP_CLAUSE_DECL (innerc
);
925 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
926 OMP_CLAUSE__LOOPTEMP_
);
928 n2
= OMP_CLAUSE_DECL (innerc
);
931 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
933 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
934 ws_args
->quick_push (t
);
936 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
937 ws_args
->quick_push (t
);
939 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
940 ws_args
->quick_push (t
);
944 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
945 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
946 ws_args
->quick_push (t
);
951 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
953 /* Number of sections is equal to the number of edges from the
954 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
955 the exit of the sections region. */
956 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
957 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
958 vec_alloc (ws_args
, 1);
959 ws_args
->quick_push (t
);
967 /* Discover whether REGION is a combined parallel+workshare region. */
970 determine_parallel_type (struct omp_region
*region
)
972 basic_block par_entry_bb
, par_exit_bb
;
973 basic_block ws_entry_bb
, ws_exit_bb
;
975 if (region
== NULL
|| region
->inner
== NULL
976 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
977 || region
->inner
->cont
== NULL
)
980 /* We only support parallel+for and parallel+sections. */
981 if (region
->type
!= GIMPLE_OMP_PARALLEL
982 || (region
->inner
->type
!= GIMPLE_OMP_FOR
983 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
986 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
987 WS_EXIT_BB -> PAR_EXIT_BB. */
988 par_entry_bb
= region
->entry
;
989 par_exit_bb
= region
->exit
;
990 ws_entry_bb
= region
->inner
->entry
;
991 ws_exit_bb
= region
->inner
->exit
;
993 if (single_succ (par_entry_bb
) == ws_entry_bb
994 && single_succ (ws_exit_bb
) == par_exit_bb
995 && workshare_safe_to_combine_p (ws_entry_bb
)
996 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
997 || (last_and_only_stmt (ws_entry_bb
)
998 && last_and_only_stmt (par_exit_bb
))))
1000 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1001 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1003 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1005 /* If this is a combined parallel loop, we need to determine
1006 whether or not to use the combined library calls. There
1007 are two cases where we do not apply the transformation:
1008 static loops and any kind of ordered loop. In the first
1009 case, we already open code the loop so there is no need
1010 to do anything else. In the latter case, the combined
1011 parallel loop call would still need extra synchronization
1012 to implement ordered semantics, so there would not be any
1013 gain in using the combined call. */
1014 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1015 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1017 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1018 == OMP_CLAUSE_SCHEDULE_STATIC
)
1019 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1021 region
->is_combined_parallel
= false;
1022 region
->inner
->is_combined_parallel
= false;
1027 region
->is_combined_parallel
= true;
1028 region
->inner
->is_combined_parallel
= true;
1029 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1034 /* Return true if EXPR is variable sized. */
1037 is_variable_sized (const_tree expr
)
1039 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1042 /* Return true if DECL is a reference type. */
1045 is_reference (tree decl
)
1047 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1050 /* Return the type of a decl. If the decl is reference type,
1051 return its base type. */
1053 get_base_type (tree decl
)
1055 tree type
= TREE_TYPE (decl
);
1056 if (is_reference (decl
))
1057 type
= TREE_TYPE (type
);
1061 /* Lookup variables. The "maybe" form
1062 allows for the variable form to not have been entered, otherwise we
1063 assert that the variable must have been entered. */
1066 lookup_decl (tree var
, omp_context
*ctx
)
1068 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1073 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1075 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1076 return n
? *n
: NULL_TREE
;
1080 lookup_field (tree var
, omp_context
*ctx
)
1083 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1084 return (tree
) n
->value
;
1088 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1091 n
= splay_tree_lookup (ctx
->sfield_map
1092 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1093 return (tree
) n
->value
;
1097 lookup_sfield (tree var
, omp_context
*ctx
)
1099 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1103 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1106 n
= splay_tree_lookup (ctx
->field_map
, key
);
1107 return n
? (tree
) n
->value
: NULL_TREE
;
1111 maybe_lookup_field (tree var
, omp_context
*ctx
)
1113 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1116 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1117 the parallel context if DECL is to be shared. */
1120 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1122 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1125 /* We can only use copy-in/copy-out semantics for shared variables
1126 when we know the value is not accessible from an outer scope. */
1129 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1131 /* ??? Trivially accessible from anywhere. But why would we even
1132 be passing an address in this case? Should we simply assert
1133 this to be false, or should we have a cleanup pass that removes
1134 these from the list of mappings? */
1135 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1138 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1139 without analyzing the expression whether or not its location
1140 is accessible to anyone else. In the case of nested parallel
1141 regions it certainly may be. */
1142 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1145 /* Do not use copy-in/copy-out for variables that have their
1147 if (TREE_ADDRESSABLE (decl
))
1150 /* lower_send_shared_vars only uses copy-in, but not copy-out
1152 if (TREE_READONLY (decl
)
1153 || ((TREE_CODE (decl
) == RESULT_DECL
1154 || TREE_CODE (decl
) == PARM_DECL
)
1155 && DECL_BY_REFERENCE (decl
)))
1158 /* Disallow copy-in/out in nested parallel if
1159 decl is shared in outer parallel, otherwise
1160 each thread could store the shared variable
1161 in its own copy-in location, making the
1162 variable no longer really shared. */
1163 if (shared_ctx
->is_nested
)
1167 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1168 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1175 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1176 c
; c
= OMP_CLAUSE_CHAIN (c
))
1177 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1178 && OMP_CLAUSE_DECL (c
) == decl
)
1182 goto maybe_mark_addressable_and_ret
;
1186 /* For tasks avoid using copy-in/out. As tasks can be
1187 deferred or executed in different thread, when GOMP_task
1188 returns, the task hasn't necessarily terminated. */
1189 if (is_task_ctx (shared_ctx
))
1192 maybe_mark_addressable_and_ret
:
1193 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1194 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1196 /* Taking address of OUTER in lower_send_shared_vars
1197 might need regimplification of everything that uses the
1199 if (!task_shared_vars
)
1200 task_shared_vars
= BITMAP_ALLOC (NULL
);
1201 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1202 TREE_ADDRESSABLE (outer
) = 1;
1211 /* Construct a new automatic decl similar to VAR. */
1214 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1216 tree copy
= copy_var_decl (var
, name
, type
);
1218 DECL_CONTEXT (copy
) = current_function_decl
;
1219 DECL_CHAIN (copy
) = ctx
->block_vars
;
1220 /* If VAR is listed in task_shared_vars, it means it wasn't
1221 originally addressable and is just because task needs to take
1222 it's address. But we don't need to take address of privatizations
1224 if (TREE_ADDRESSABLE (var
)
1226 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1227 TREE_ADDRESSABLE (copy
) = 0;
1228 ctx
->block_vars
= copy
;
1234 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1236 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1239 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1242 omp_build_component_ref (tree obj
, tree field
)
1244 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1245 if (TREE_THIS_VOLATILE (field
))
1246 TREE_THIS_VOLATILE (ret
) |= 1;
1247 if (TREE_READONLY (field
))
1248 TREE_READONLY (ret
) |= 1;
1252 /* Build tree nodes to access the field for VAR on the receiver side. */
1255 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1257 tree x
, field
= lookup_field (var
, ctx
);
1259 /* If the receiver record type was remapped in the child function,
1260 remap the field into the new record type. */
1261 x
= maybe_lookup_field (field
, ctx
);
1265 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1266 TREE_THIS_NOTRAP (x
) = 1;
1267 x
= omp_build_component_ref (x
, field
);
1270 x
= build_simple_mem_ref (x
);
1271 TREE_THIS_NOTRAP (x
) = 1;
1277 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1278 of a parallel, this is a component reference; for workshare constructs
1279 this is some variable. */
1282 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1286 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1288 else if (is_variable_sized (var
))
1290 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1291 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1292 x
= build_simple_mem_ref (x
);
1294 else if (is_taskreg_ctx (ctx
))
1296 bool by_ref
= use_pointer_for_field (var
, NULL
);
1297 x
= build_receiver_ref (var
, by_ref
, ctx
);
1299 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1300 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1302 /* #pragma omp simd isn't a worksharing construct, and can reference even
1303 private vars in its linear etc. clauses. */
1305 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1306 x
= lookup_decl (var
, ctx
->outer
);
1307 else if (ctx
->outer
)
1308 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1312 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1314 gcc_assert (ctx
->outer
);
1316 = splay_tree_lookup (ctx
->outer
->field_map
,
1317 (splay_tree_key
) &DECL_UID (var
));
1320 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1323 x
= lookup_decl (var
, ctx
->outer
);
1327 tree field
= (tree
) n
->value
;
1328 /* If the receiver record type was remapped in the child function,
1329 remap the field into the new record type. */
1330 x
= maybe_lookup_field (field
, ctx
->outer
);
1334 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1335 x
= omp_build_component_ref (x
, field
);
1336 if (use_pointer_for_field (var
, ctx
->outer
))
1337 x
= build_simple_mem_ref (x
);
1340 else if (ctx
->outer
)
1342 omp_context
*outer
= ctx
->outer
;
1343 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
1345 outer
= outer
->outer
;
1347 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
1349 x
= lookup_decl (var
, outer
);
1351 else if (is_reference (var
))
1352 /* This can happen with orphaned constructs. If var is reference, it is
1353 possible it is shared and as such valid. */
1355 else if (omp_member_access_dummy_var (var
))
1362 tree t
= omp_member_access_dummy_var (var
);
1365 x
= DECL_VALUE_EXPR (var
);
1366 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1368 x
= unshare_and_remap (x
, t
, o
);
1370 x
= unshare_expr (x
);
1374 if (is_reference (var
))
1375 x
= build_simple_mem_ref (x
);
1380 /* Build tree nodes to access the field for VAR on the sender side. */
1383 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1385 tree field
= lookup_sfield (key
, ctx
);
1386 return omp_build_component_ref (ctx
->sender_decl
, field
);
1390 build_sender_ref (tree var
, omp_context
*ctx
)
1392 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1395 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1396 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1399 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
,
1400 bool base_pointers_restrict
= false)
1402 tree field
, type
, sfield
= NULL_TREE
;
1403 splay_tree_key key
= (splay_tree_key
) var
;
1405 if ((mask
& 8) != 0)
1407 key
= (splay_tree_key
) &DECL_UID (var
);
1408 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1410 gcc_assert ((mask
& 1) == 0
1411 || !splay_tree_lookup (ctx
->field_map
, key
));
1412 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1413 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1414 gcc_assert ((mask
& 3) == 3
1415 || !is_gimple_omp_oacc (ctx
->stmt
));
1417 type
= TREE_TYPE (var
);
1418 /* Prevent redeclaring the var in the split-off function with a restrict
1419 pointer type. Note that we only clear type itself, restrict qualifiers in
1420 the pointed-to type will be ignored by points-to analysis. */
1421 if (POINTER_TYPE_P (type
)
1422 && TYPE_RESTRICT (type
))
1423 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
1427 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1428 type
= build_pointer_type (build_pointer_type (type
));
1432 type
= build_pointer_type (type
);
1433 if (base_pointers_restrict
)
1434 type
= build_qualified_type (type
, TYPE_QUAL_RESTRICT
);
1436 else if ((mask
& 3) == 1 && is_reference (var
))
1437 type
= TREE_TYPE (type
);
1439 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1440 FIELD_DECL
, DECL_NAME (var
), type
);
1442 /* Remember what variable this field was created for. This does have a
1443 side effect of making dwarf2out ignore this member, so for helpful
1444 debugging we clear it later in delete_omp_context. */
1445 DECL_ABSTRACT_ORIGIN (field
) = var
;
1446 if (type
== TREE_TYPE (var
))
1448 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1449 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1450 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1453 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1455 if ((mask
& 3) == 3)
1457 insert_field_into_struct (ctx
->record_type
, field
);
1458 if (ctx
->srecord_type
)
1460 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1461 FIELD_DECL
, DECL_NAME (var
), type
);
1462 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1463 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1464 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1465 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1466 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1471 if (ctx
->srecord_type
== NULL_TREE
)
1475 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1476 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1477 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1479 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1480 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1481 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1482 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1483 splay_tree_insert (ctx
->sfield_map
,
1484 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1485 (splay_tree_value
) sfield
);
1489 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1490 : ctx
->srecord_type
, field
);
1494 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1495 if ((mask
& 2) && ctx
->sfield_map
)
1496 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1500 install_var_local (tree var
, omp_context
*ctx
)
1502 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1503 insert_decl_map (&ctx
->cb
, var
, new_var
);
1507 /* Adjust the replacement for DECL in CTX for the new context. This means
1508 copying the DECL_VALUE_EXPR, and fixing up the type. */
1511 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1513 tree new_decl
, size
;
1515 new_decl
= lookup_decl (decl
, ctx
);
1517 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1519 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1520 && DECL_HAS_VALUE_EXPR_P (decl
))
1522 tree ve
= DECL_VALUE_EXPR (decl
);
1523 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1524 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1525 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1528 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1530 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1531 if (size
== error_mark_node
)
1532 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1533 DECL_SIZE (new_decl
) = size
;
1535 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1536 if (size
== error_mark_node
)
1537 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1538 DECL_SIZE_UNIT (new_decl
) = size
;
1542 /* The callback for remap_decl. Search all containing contexts for a
1543 mapping of the variable; this avoids having to duplicate the splay
1544 tree ahead of time. We know a mapping doesn't already exist in the
1545 given context. Create new mappings to implement default semantics. */
1548 omp_copy_decl (tree var
, copy_body_data
*cb
)
1550 omp_context
*ctx
= (omp_context
*) cb
;
1553 if (TREE_CODE (var
) == LABEL_DECL
)
1555 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1556 DECL_CONTEXT (new_var
) = current_function_decl
;
1557 insert_decl_map (&ctx
->cb
, var
, new_var
);
1561 while (!is_taskreg_ctx (ctx
))
1566 new_var
= maybe_lookup_decl (var
, ctx
);
1571 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1574 return error_mark_node
;
1578 /* Debugging dumps for parallel regions. */
1579 void dump_omp_region (FILE *, struct omp_region
*, int);
1580 void debug_omp_region (struct omp_region
*);
1581 void debug_all_omp_regions (void);
1583 /* Dump the parallel region tree rooted at REGION. */
1586 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1588 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1589 gimple_code_name
[region
->type
]);
1592 dump_omp_region (file
, region
->inner
, indent
+ 4);
1596 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1597 region
->cont
->index
);
1601 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1602 region
->exit
->index
);
1604 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1607 dump_omp_region (file
, region
->next
, indent
);
1611 debug_omp_region (struct omp_region
*region
)
1613 dump_omp_region (stderr
, region
, 0);
1617 debug_all_omp_regions (void)
1619 dump_omp_region (stderr
, root_omp_region
, 0);
1623 /* Create a new parallel region starting at STMT inside region PARENT. */
1625 static struct omp_region
*
1626 new_omp_region (basic_block bb
, enum gimple_code type
,
1627 struct omp_region
*parent
)
1629 struct omp_region
*region
= XCNEW (struct omp_region
);
1631 region
->outer
= parent
;
1633 region
->type
= type
;
1637 /* This is a nested region. Add it to the list of inner
1638 regions in PARENT. */
1639 region
->next
= parent
->inner
;
1640 parent
->inner
= region
;
1644 /* This is a toplevel region. Add it to the list of toplevel
1645 regions in ROOT_OMP_REGION. */
1646 region
->next
= root_omp_region
;
1647 root_omp_region
= region
;
1653 /* Release the memory associated with the region tree rooted at REGION. */
1656 free_omp_region_1 (struct omp_region
*region
)
1658 struct omp_region
*i
, *n
;
1660 for (i
= region
->inner
; i
; i
= n
)
1663 free_omp_region_1 (i
);
1669 /* Release the memory for the entire omp region tree. */
1672 free_omp_regions (void)
1674 struct omp_region
*r
, *n
;
1675 for (r
= root_omp_region
; r
; r
= n
)
1678 free_omp_region_1 (r
);
1680 root_omp_region
= NULL
;
1684 /* Create a new context, with OUTER_CTX being the surrounding context. */
1686 static omp_context
*
1687 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1689 omp_context
*ctx
= XCNEW (omp_context
);
1691 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1692 (splay_tree_value
) ctx
);
1697 ctx
->outer
= outer_ctx
;
1698 ctx
->cb
= outer_ctx
->cb
;
1699 ctx
->cb
.block
= NULL
;
1700 ctx
->depth
= outer_ctx
->depth
+ 1;
1704 ctx
->cb
.src_fn
= current_function_decl
;
1705 ctx
->cb
.dst_fn
= current_function_decl
;
1706 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1707 gcc_checking_assert (ctx
->cb
.src_node
);
1708 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1709 ctx
->cb
.src_cfun
= cfun
;
1710 ctx
->cb
.copy_decl
= omp_copy_decl
;
1711 ctx
->cb
.eh_lp_nr
= 0;
1712 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1716 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1721 static gimple_seq
maybe_catch_exception (gimple_seq
);
1723 /* Finalize task copyfn. */
1726 finalize_task_copyfn (gomp_task
*task_stmt
)
1728 struct function
*child_cfun
;
1730 gimple_seq seq
= NULL
, new_seq
;
1733 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1734 if (child_fn
== NULL_TREE
)
1737 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1738 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1740 push_cfun (child_cfun
);
1741 bind
= gimplify_body (child_fn
, false);
1742 gimple_seq_add_stmt (&seq
, bind
);
1743 new_seq
= maybe_catch_exception (seq
);
1746 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1748 gimple_seq_add_stmt (&seq
, bind
);
1750 gimple_set_body (child_fn
, seq
);
1753 /* Inform the callgraph about the new function. */
1754 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1755 node
->parallelized_function
= 1;
1756 cgraph_node::add_new_function (child_fn
, false);
1759 /* Destroy a omp_context data structures. Called through the splay tree
1760 value delete callback. */
1763 delete_omp_context (splay_tree_value value
)
1765 omp_context
*ctx
= (omp_context
*) value
;
1767 delete ctx
->cb
.decl_map
;
1770 splay_tree_delete (ctx
->field_map
);
1771 if (ctx
->sfield_map
)
1772 splay_tree_delete (ctx
->sfield_map
);
1774 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1775 it produces corrupt debug information. */
1776 if (ctx
->record_type
)
1779 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1780 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1782 if (ctx
->srecord_type
)
1785 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1786 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1789 if (is_task_ctx (ctx
))
1790 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1795 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1799 fixup_child_record_type (omp_context
*ctx
)
1801 tree f
, type
= ctx
->record_type
;
1803 if (!ctx
->receiver_decl
)
1805 /* ??? It isn't sufficient to just call remap_type here, because
1806 variably_modified_type_p doesn't work the way we expect for
1807 record types. Testing each field for whether it needs remapping
1808 and creating a new record by hand works, however. */
1809 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1810 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1814 tree name
, new_fields
= NULL
;
1816 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1817 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1818 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1819 TYPE_DECL
, name
, type
);
1820 TYPE_NAME (type
) = name
;
1822 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1824 tree new_f
= copy_node (f
);
1825 DECL_CONTEXT (new_f
) = type
;
1826 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1827 DECL_CHAIN (new_f
) = new_fields
;
1828 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1829 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1831 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1835 /* Arrange to be able to look up the receiver field
1836 given the sender field. */
1837 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1838 (splay_tree_value
) new_f
);
1840 TYPE_FIELDS (type
) = nreverse (new_fields
);
1844 /* In a target region we never modify any of the pointers in *.omp_data_i,
1845 so attempt to help the optimizers. */
1846 if (is_gimple_omp_offloaded (ctx
->stmt
))
1847 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1849 TREE_TYPE (ctx
->receiver_decl
)
1850 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1853 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1854 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1858 scan_sharing_clauses (tree clauses
, omp_context
*ctx
,
1859 bool base_pointers_restrict
= false)
1862 bool scan_array_reductions
= false;
1864 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1868 switch (OMP_CLAUSE_CODE (c
))
1870 case OMP_CLAUSE_PRIVATE
:
1871 decl
= OMP_CLAUSE_DECL (c
);
1872 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1874 else if (!is_variable_sized (decl
))
1875 install_var_local (decl
, ctx
);
1878 case OMP_CLAUSE_SHARED
:
1879 decl
= OMP_CLAUSE_DECL (c
);
1880 /* Ignore shared directives in teams construct. */
1881 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1883 /* Global variables don't need to be copied,
1884 the receiver side will use them directly. */
1885 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1886 if (is_global_var (odecl
))
1888 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1891 gcc_assert (is_taskreg_ctx (ctx
));
1892 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1893 || !is_variable_sized (decl
));
1894 /* Global variables don't need to be copied,
1895 the receiver side will use them directly. */
1896 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1898 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1900 use_pointer_for_field (decl
, ctx
);
1903 by_ref
= use_pointer_for_field (decl
, NULL
);
1904 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1905 || TREE_ADDRESSABLE (decl
)
1907 || is_reference (decl
))
1909 by_ref
= use_pointer_for_field (decl
, ctx
);
1910 install_var_field (decl
, by_ref
, 3, ctx
);
1911 install_var_local (decl
, ctx
);
1914 /* We don't need to copy const scalar vars back. */
1915 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1918 case OMP_CLAUSE_REDUCTION
:
1919 decl
= OMP_CLAUSE_DECL (c
);
1920 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1921 && TREE_CODE (decl
) == MEM_REF
)
1923 tree t
= TREE_OPERAND (decl
, 0);
1924 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1925 t
= TREE_OPERAND (t
, 0);
1926 if (TREE_CODE (t
) == INDIRECT_REF
1927 || TREE_CODE (t
) == ADDR_EXPR
)
1928 t
= TREE_OPERAND (t
, 0);
1929 install_var_local (t
, ctx
);
1930 if (is_taskreg_ctx (ctx
)
1931 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1932 && !is_variable_sized (t
))
1934 by_ref
= use_pointer_for_field (t
, ctx
);
1935 install_var_field (t
, by_ref
, 3, ctx
);
1941 case OMP_CLAUSE_LASTPRIVATE
:
1942 /* Let the corresponding firstprivate clause create
1944 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1948 case OMP_CLAUSE_FIRSTPRIVATE
:
1949 case OMP_CLAUSE_LINEAR
:
1950 decl
= OMP_CLAUSE_DECL (c
);
1952 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1953 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1954 && is_gimple_omp_offloaded (ctx
->stmt
))
1956 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1957 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1958 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1959 install_var_field (decl
, true, 3, ctx
);
1961 install_var_field (decl
, false, 3, ctx
);
1963 if (is_variable_sized (decl
))
1965 if (is_task_ctx (ctx
))
1966 install_var_field (decl
, false, 1, ctx
);
1969 else if (is_taskreg_ctx (ctx
))
1972 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1973 by_ref
= use_pointer_for_field (decl
, NULL
);
1975 if (is_task_ctx (ctx
)
1976 && (global
|| by_ref
|| is_reference (decl
)))
1978 install_var_field (decl
, false, 1, ctx
);
1980 install_var_field (decl
, by_ref
, 2, ctx
);
1983 install_var_field (decl
, by_ref
, 3, ctx
);
1985 install_var_local (decl
, ctx
);
1988 case OMP_CLAUSE_USE_DEVICE_PTR
:
1989 decl
= OMP_CLAUSE_DECL (c
);
1990 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1991 install_var_field (decl
, true, 3, ctx
);
1993 install_var_field (decl
, false, 3, ctx
);
1994 if (DECL_SIZE (decl
)
1995 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1997 tree decl2
= DECL_VALUE_EXPR (decl
);
1998 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1999 decl2
= TREE_OPERAND (decl2
, 0);
2000 gcc_assert (DECL_P (decl2
));
2001 install_var_local (decl2
, ctx
);
2003 install_var_local (decl
, ctx
);
2006 case OMP_CLAUSE_IS_DEVICE_PTR
:
2007 decl
= OMP_CLAUSE_DECL (c
);
2010 case OMP_CLAUSE__LOOPTEMP_
:
2011 gcc_assert (is_taskreg_ctx (ctx
));
2012 decl
= OMP_CLAUSE_DECL (c
);
2013 install_var_field (decl
, false, 3, ctx
);
2014 install_var_local (decl
, ctx
);
2017 case OMP_CLAUSE_COPYPRIVATE
:
2018 case OMP_CLAUSE_COPYIN
:
2019 decl
= OMP_CLAUSE_DECL (c
);
2020 by_ref
= use_pointer_for_field (decl
, NULL
);
2021 install_var_field (decl
, by_ref
, 3, ctx
);
2024 case OMP_CLAUSE_DEFAULT
:
2025 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2028 case OMP_CLAUSE_FINAL
:
2030 case OMP_CLAUSE_NUM_THREADS
:
2031 case OMP_CLAUSE_NUM_TEAMS
:
2032 case OMP_CLAUSE_THREAD_LIMIT
:
2033 case OMP_CLAUSE_DEVICE
:
2034 case OMP_CLAUSE_SCHEDULE
:
2035 case OMP_CLAUSE_DIST_SCHEDULE
:
2036 case OMP_CLAUSE_DEPEND
:
2037 case OMP_CLAUSE_PRIORITY
:
2038 case OMP_CLAUSE_GRAINSIZE
:
2039 case OMP_CLAUSE_NUM_TASKS
:
2040 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2041 case OMP_CLAUSE_NUM_GANGS
:
2042 case OMP_CLAUSE_NUM_WORKERS
:
2043 case OMP_CLAUSE_VECTOR_LENGTH
:
2045 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2049 case OMP_CLAUSE_FROM
:
2050 case OMP_CLAUSE_MAP
:
2052 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2053 decl
= OMP_CLAUSE_DECL (c
);
2054 /* Global variables with "omp declare target" attribute
2055 don't need to be copied, the receiver side will use them
2056 directly. However, global variables with "omp declare target link"
2057 attribute need to be copied. */
2058 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2060 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2061 && (OMP_CLAUSE_MAP_KIND (c
)
2062 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2063 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2064 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2065 && varpool_node::get_create (decl
)->offloadable
2066 && !lookup_attribute ("omp declare target link",
2067 DECL_ATTRIBUTES (decl
)))
2069 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2070 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2072 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2073 not offloaded; there is nothing to map for those. */
2074 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2075 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2076 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2079 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2080 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2081 || (OMP_CLAUSE_MAP_KIND (c
)
2082 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2084 if (TREE_CODE (decl
) == COMPONENT_REF
2085 || (TREE_CODE (decl
) == INDIRECT_REF
2086 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2087 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2088 == REFERENCE_TYPE
)))
2090 if (DECL_SIZE (decl
)
2091 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2093 tree decl2
= DECL_VALUE_EXPR (decl
);
2094 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2095 decl2
= TREE_OPERAND (decl2
, 0);
2096 gcc_assert (DECL_P (decl2
));
2097 install_var_local (decl2
, ctx
);
2099 install_var_local (decl
, ctx
);
2104 if (DECL_SIZE (decl
)
2105 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2107 tree decl2
= DECL_VALUE_EXPR (decl
);
2108 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2109 decl2
= TREE_OPERAND (decl2
, 0);
2110 gcc_assert (DECL_P (decl2
));
2111 install_var_field (decl2
, true, 3, ctx
);
2112 install_var_local (decl2
, ctx
);
2113 install_var_local (decl
, ctx
);
2117 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2118 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2119 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2120 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2121 install_var_field (decl
, true, 7, ctx
);
2123 install_var_field (decl
, true, 3, ctx
,
2124 base_pointers_restrict
);
2125 if (is_gimple_omp_offloaded (ctx
->stmt
)
2126 && !OMP_CLAUSE_MAP_IN_REDUCTION (c
))
2127 install_var_local (decl
, ctx
);
2132 tree base
= get_base_address (decl
);
2133 tree nc
= OMP_CLAUSE_CHAIN (c
);
2136 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2137 && OMP_CLAUSE_DECL (nc
) == base
2138 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2139 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2141 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2142 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2148 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2149 decl
= OMP_CLAUSE_DECL (c
);
2151 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2152 (splay_tree_key
) decl
));
2154 = build_decl (OMP_CLAUSE_LOCATION (c
),
2155 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2156 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2157 insert_field_into_struct (ctx
->record_type
, field
);
2158 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2159 (splay_tree_value
) field
);
2164 case OMP_CLAUSE__GRIDDIM_
:
2167 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
2168 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
2172 case OMP_CLAUSE_NOWAIT
:
2173 case OMP_CLAUSE_ORDERED
:
2174 case OMP_CLAUSE_COLLAPSE
:
2175 case OMP_CLAUSE_UNTIED
:
2176 case OMP_CLAUSE_MERGEABLE
:
2177 case OMP_CLAUSE_PROC_BIND
:
2178 case OMP_CLAUSE_SAFELEN
:
2179 case OMP_CLAUSE_SIMDLEN
:
2180 case OMP_CLAUSE_THREADS
:
2181 case OMP_CLAUSE_SIMD
:
2182 case OMP_CLAUSE_NOGROUP
:
2183 case OMP_CLAUSE_DEFAULTMAP
:
2184 case OMP_CLAUSE_ASYNC
:
2185 case OMP_CLAUSE_WAIT
:
2186 case OMP_CLAUSE_GANG
:
2187 case OMP_CLAUSE_WORKER
:
2188 case OMP_CLAUSE_VECTOR
:
2189 case OMP_CLAUSE_INDEPENDENT
:
2190 case OMP_CLAUSE_AUTO
:
2191 case OMP_CLAUSE_SEQ
:
2194 case OMP_CLAUSE_ALIGNED
:
2195 decl
= OMP_CLAUSE_DECL (c
);
2196 if (is_global_var (decl
)
2197 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2198 install_var_local (decl
, ctx
);
2201 case OMP_CLAUSE_DEVICE_RESIDENT
:
2202 case OMP_CLAUSE_TILE
:
2203 case OMP_CLAUSE__CACHE_
:
2209 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2211 switch (OMP_CLAUSE_CODE (c
))
2213 case OMP_CLAUSE_LASTPRIVATE
:
2214 /* Let the corresponding firstprivate clause create
2216 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2217 scan_array_reductions
= true;
2218 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2222 case OMP_CLAUSE_FIRSTPRIVATE
:
2223 case OMP_CLAUSE_PRIVATE
:
2224 case OMP_CLAUSE_LINEAR
:
2225 case OMP_CLAUSE_IS_DEVICE_PTR
:
2226 decl
= OMP_CLAUSE_DECL (c
);
2227 if (is_variable_sized (decl
))
2229 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2230 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2231 && is_gimple_omp_offloaded (ctx
->stmt
))
2233 tree decl2
= DECL_VALUE_EXPR (decl
);
2234 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2235 decl2
= TREE_OPERAND (decl2
, 0);
2236 gcc_assert (DECL_P (decl2
));
2237 install_var_local (decl2
, ctx
);
2238 fixup_remapped_decl (decl2
, ctx
, false);
2240 install_var_local (decl
, ctx
);
2242 fixup_remapped_decl (decl
, ctx
,
2243 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2244 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2245 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2246 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2247 scan_array_reductions
= true;
2250 case OMP_CLAUSE_REDUCTION
:
2251 decl
= OMP_CLAUSE_DECL (c
);
2252 if (TREE_CODE (decl
) != MEM_REF
)
2254 if (is_variable_sized (decl
))
2255 install_var_local (decl
, ctx
);
2256 fixup_remapped_decl (decl
, ctx
, false);
2258 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2259 scan_array_reductions
= true;
2262 case OMP_CLAUSE_SHARED
:
2263 /* Ignore shared directives in teams construct. */
2264 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2266 decl
= OMP_CLAUSE_DECL (c
);
2267 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2269 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2271 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2274 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2275 install_var_field (decl
, by_ref
, 11, ctx
);
2278 fixup_remapped_decl (decl
, ctx
, false);
2281 case OMP_CLAUSE_MAP
:
2282 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2284 decl
= OMP_CLAUSE_DECL (c
);
2286 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2287 && (OMP_CLAUSE_MAP_KIND (c
)
2288 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2289 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2290 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2291 && varpool_node::get_create (decl
)->offloadable
)
2295 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2296 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2297 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2298 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2300 tree new_decl
= lookup_decl (decl
, ctx
);
2301 TREE_TYPE (new_decl
)
2302 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2304 else if (DECL_SIZE (decl
)
2305 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2307 tree decl2
= DECL_VALUE_EXPR (decl
);
2308 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2309 decl2
= TREE_OPERAND (decl2
, 0);
2310 gcc_assert (DECL_P (decl2
));
2311 fixup_remapped_decl (decl2
, ctx
, false);
2312 fixup_remapped_decl (decl
, ctx
, true);
2315 fixup_remapped_decl (decl
, ctx
, false);
2319 case OMP_CLAUSE_COPYPRIVATE
:
2320 case OMP_CLAUSE_COPYIN
:
2321 case OMP_CLAUSE_DEFAULT
:
2323 case OMP_CLAUSE_NUM_THREADS
:
2324 case OMP_CLAUSE_NUM_TEAMS
:
2325 case OMP_CLAUSE_THREAD_LIMIT
:
2326 case OMP_CLAUSE_DEVICE
:
2327 case OMP_CLAUSE_SCHEDULE
:
2328 case OMP_CLAUSE_DIST_SCHEDULE
:
2329 case OMP_CLAUSE_NOWAIT
:
2330 case OMP_CLAUSE_ORDERED
:
2331 case OMP_CLAUSE_COLLAPSE
:
2332 case OMP_CLAUSE_UNTIED
:
2333 case OMP_CLAUSE_FINAL
:
2334 case OMP_CLAUSE_MERGEABLE
:
2335 case OMP_CLAUSE_PROC_BIND
:
2336 case OMP_CLAUSE_SAFELEN
:
2337 case OMP_CLAUSE_SIMDLEN
:
2338 case OMP_CLAUSE_ALIGNED
:
2339 case OMP_CLAUSE_DEPEND
:
2340 case OMP_CLAUSE__LOOPTEMP_
:
2342 case OMP_CLAUSE_FROM
:
2343 case OMP_CLAUSE_PRIORITY
:
2344 case OMP_CLAUSE_GRAINSIZE
:
2345 case OMP_CLAUSE_NUM_TASKS
:
2346 case OMP_CLAUSE_THREADS
:
2347 case OMP_CLAUSE_SIMD
:
2348 case OMP_CLAUSE_NOGROUP
:
2349 case OMP_CLAUSE_DEFAULTMAP
:
2350 case OMP_CLAUSE_USE_DEVICE_PTR
:
2351 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2352 case OMP_CLAUSE_ASYNC
:
2353 case OMP_CLAUSE_WAIT
:
2354 case OMP_CLAUSE_NUM_GANGS
:
2355 case OMP_CLAUSE_NUM_WORKERS
:
2356 case OMP_CLAUSE_VECTOR_LENGTH
:
2357 case OMP_CLAUSE_GANG
:
2358 case OMP_CLAUSE_WORKER
:
2359 case OMP_CLAUSE_VECTOR
:
2360 case OMP_CLAUSE_INDEPENDENT
:
2361 case OMP_CLAUSE_AUTO
:
2362 case OMP_CLAUSE_SEQ
:
2363 case OMP_CLAUSE__GRIDDIM_
:
2366 case OMP_CLAUSE_DEVICE_RESIDENT
:
2367 case OMP_CLAUSE_TILE
:
2368 case OMP_CLAUSE__CACHE_
:
2374 gcc_checking_assert (!scan_array_reductions
2375 || !is_gimple_omp_oacc (ctx
->stmt
));
2376 if (scan_array_reductions
)
2378 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2379 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2380 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2382 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2383 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2385 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2386 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2387 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2388 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2389 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2390 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2394 /* Create a new name for omp child function. Returns an identifier. If
2395 IS_CILK_FOR is true then the suffix for the child function is
2399 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2402 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2403 return clone_function_name (current_function_decl
,
2404 task_copy
? "_omp_cpyfn" : "_omp_fn");
2407 /* Returns the type of the induction variable for the child function for
2408 _Cilk_for and the types for _high and _low variables based on TYPE. */
2411 cilk_for_check_loop_diff_type (tree type
)
2413 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2415 if (TYPE_UNSIGNED (type
))
2416 return uint32_type_node
;
2418 return integer_type_node
;
2422 if (TYPE_UNSIGNED (type
))
2423 return uint64_type_node
;
2425 return long_long_integer_type_node
;
2429 /* Build a decl for the omp child function. It'll not contain a body
2430 yet, just the bare decl. */
2433 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2435 tree decl
, type
, name
, t
;
2438 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2439 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2440 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2441 tree cilk_var_type
= NULL_TREE
;
2443 name
= create_omp_child_function_name (task_copy
,
2444 cilk_for_count
!= NULL_TREE
);
2446 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2447 ptr_type_node
, NULL_TREE
);
2448 else if (cilk_for_count
)
2450 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2451 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2452 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2453 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2456 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2458 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2460 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2463 ctx
->cb
.dst_fn
= decl
;
2465 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2467 TREE_STATIC (decl
) = 1;
2468 TREE_USED (decl
) = 1;
2469 DECL_ARTIFICIAL (decl
) = 1;
2470 DECL_IGNORED_P (decl
) = 0;
2471 TREE_PUBLIC (decl
) = 0;
2472 DECL_UNINLINABLE (decl
) = 1;
2473 DECL_EXTERNAL (decl
) = 0;
2474 DECL_CONTEXT (decl
) = NULL_TREE
;
2475 DECL_INITIAL (decl
) = make_node (BLOCK
);
2476 if (cgraph_node::get (current_function_decl
)->offloadable
)
2477 cgraph_node::get_create (decl
)->offloadable
= 1;
2481 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2482 if (is_gimple_omp_offloaded (octx
->stmt
))
2484 cgraph_node::get_create (decl
)->offloadable
= 1;
2485 if (ENABLE_OFFLOADING
)
2486 g
->have_offload
= true;
2492 if (cgraph_node::get_create (decl
)->offloadable
2493 && !lookup_attribute ("omp declare target",
2494 DECL_ATTRIBUTES (current_function_decl
)))
2495 DECL_ATTRIBUTES (decl
)
2496 = tree_cons (get_identifier ("omp target entrypoint"),
2497 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2499 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2500 RESULT_DECL
, NULL_TREE
, void_type_node
);
2501 DECL_ARTIFICIAL (t
) = 1;
2502 DECL_IGNORED_P (t
) = 1;
2503 DECL_CONTEXT (t
) = decl
;
2504 DECL_RESULT (decl
) = t
;
2506 /* _Cilk_for's child function requires two extra parameters called
2507 __low and __high that are set the by Cilk runtime when it calls this
2511 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2512 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2513 DECL_ARTIFICIAL (t
) = 1;
2514 DECL_NAMELESS (t
) = 1;
2515 DECL_ARG_TYPE (t
) = ptr_type_node
;
2516 DECL_CONTEXT (t
) = current_function_decl
;
2518 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2519 DECL_ARGUMENTS (decl
) = t
;
2521 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2522 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2523 DECL_ARTIFICIAL (t
) = 1;
2524 DECL_NAMELESS (t
) = 1;
2525 DECL_ARG_TYPE (t
) = ptr_type_node
;
2526 DECL_CONTEXT (t
) = current_function_decl
;
2528 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2529 DECL_ARGUMENTS (decl
) = t
;
2532 tree data_name
= get_identifier (".omp_data_i");
2533 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2535 DECL_ARTIFICIAL (t
) = 1;
2536 DECL_NAMELESS (t
) = 1;
2537 DECL_ARG_TYPE (t
) = ptr_type_node
;
2538 DECL_CONTEXT (t
) = current_function_decl
;
2540 TREE_READONLY (t
) = 1;
2542 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2543 DECL_ARGUMENTS (decl
) = t
;
2545 ctx
->receiver_decl
= t
;
2548 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2549 PARM_DECL
, get_identifier (".omp_data_o"),
2551 DECL_ARTIFICIAL (t
) = 1;
2552 DECL_NAMELESS (t
) = 1;
2553 DECL_ARG_TYPE (t
) = ptr_type_node
;
2554 DECL_CONTEXT (t
) = current_function_decl
;
2556 TREE_ADDRESSABLE (t
) = 1;
2557 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2558 DECL_ARGUMENTS (decl
) = t
;
2561 /* Allocate memory for the function structure. The call to
2562 allocate_struct_function clobbers CFUN, so we need to restore
2564 push_struct_function (decl
);
2565 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2569 /* Callback for walk_gimple_seq. Check if combined parallel
2570 contains gimple_omp_for_combined_into_p OMP_FOR. */
2573 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2574 bool *handled_ops_p
,
2575 struct walk_stmt_info
*wi
)
2577 gimple
*stmt
= gsi_stmt (*gsi_p
);
2579 *handled_ops_p
= true;
2580 switch (gimple_code (stmt
))
2584 case GIMPLE_OMP_FOR
:
2585 if (gimple_omp_for_combined_into_p (stmt
)
2586 && gimple_omp_for_kind (stmt
)
2587 == *(const enum gf_mask
*) (wi
->info
))
2590 return integer_zero_node
;
2599 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2602 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2603 omp_context
*outer_ctx
)
2605 struct walk_stmt_info wi
;
2607 memset (&wi
, 0, sizeof (wi
));
2609 wi
.info
= (void *) &msk
;
2610 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2611 if (wi
.info
!= (void *) &msk
)
2613 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2614 struct omp_for_data fd
;
2615 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2616 /* We need two temporaries with fd.loop.v type (istart/iend)
2617 and then (fd.collapse - 1) temporaries with the same
2618 type for count2 ... countN-1 vars if not constant. */
2619 size_t count
= 2, i
;
2620 tree type
= fd
.iter_type
;
2622 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2624 count
+= fd
.collapse
- 1;
2625 /* If there are lastprivate clauses on the inner
2626 GIMPLE_OMP_FOR, add one more temporaries for the total number
2627 of iterations (product of count1 ... countN-1). */
2628 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2629 OMP_CLAUSE_LASTPRIVATE
))
2631 else if (msk
== GF_OMP_FOR_KIND_FOR
2632 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2633 OMP_CLAUSE_LASTPRIVATE
))
2636 for (i
= 0; i
< count
; i
++)
2638 tree temp
= create_tmp_var (type
);
2639 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2640 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2641 OMP_CLAUSE_DECL (c
) = temp
;
2642 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2643 gimple_omp_taskreg_set_clauses (stmt
, c
);
2648 /* Scan an OpenMP parallel directive. */
2651 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2655 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2657 /* Ignore parallel directives with empty bodies, unless there
2658 are copyin clauses. */
2660 && empty_body_p (gimple_omp_body (stmt
))
2661 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2662 OMP_CLAUSE_COPYIN
) == NULL
)
2664 gsi_replace (gsi
, gimple_build_nop (), false);
2668 if (gimple_omp_parallel_combined_p (stmt
))
2669 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2671 ctx
= new_omp_context (stmt
, outer_ctx
);
2672 taskreg_contexts
.safe_push (ctx
);
2673 if (taskreg_nesting_level
> 1)
2674 ctx
->is_nested
= true;
2675 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2676 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2677 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2678 name
= create_tmp_var_name (".omp_data_s");
2679 name
= build_decl (gimple_location (stmt
),
2680 TYPE_DECL
, name
, ctx
->record_type
);
2681 DECL_ARTIFICIAL (name
) = 1;
2682 DECL_NAMELESS (name
) = 1;
2683 TYPE_NAME (ctx
->record_type
) = name
;
2684 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2685 if (!gimple_omp_parallel_grid_phony (stmt
))
2687 create_omp_child_function (ctx
, false);
2688 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2691 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2692 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2694 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2695 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2698 /* Scan an OpenMP task directive. */
2701 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2705 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2707 /* Ignore task directives with empty bodies. */
2709 && empty_body_p (gimple_omp_body (stmt
)))
2711 gsi_replace (gsi
, gimple_build_nop (), false);
2715 if (gimple_omp_task_taskloop_p (stmt
))
2716 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2718 ctx
= new_omp_context (stmt
, outer_ctx
);
2719 taskreg_contexts
.safe_push (ctx
);
2720 if (taskreg_nesting_level
> 1)
2721 ctx
->is_nested
= true;
2722 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2723 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2724 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2725 name
= create_tmp_var_name (".omp_data_s");
2726 name
= build_decl (gimple_location (stmt
),
2727 TYPE_DECL
, name
, ctx
->record_type
);
2728 DECL_ARTIFICIAL (name
) = 1;
2729 DECL_NAMELESS (name
) = 1;
2730 TYPE_NAME (ctx
->record_type
) = name
;
2731 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2732 create_omp_child_function (ctx
, false);
2733 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2735 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2737 if (ctx
->srecord_type
)
2739 name
= create_tmp_var_name (".omp_data_a");
2740 name
= build_decl (gimple_location (stmt
),
2741 TYPE_DECL
, name
, ctx
->srecord_type
);
2742 DECL_ARTIFICIAL (name
) = 1;
2743 DECL_NAMELESS (name
) = 1;
2744 TYPE_NAME (ctx
->srecord_type
) = name
;
2745 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2746 create_omp_child_function (ctx
, true);
2749 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2751 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2753 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2754 t
= build_int_cst (long_integer_type_node
, 0);
2755 gimple_omp_task_set_arg_size (stmt
, t
);
2756 t
= build_int_cst (long_integer_type_node
, 1);
2757 gimple_omp_task_set_arg_align (stmt
, t
);
2762 /* If any decls have been made addressable during scan_omp,
2763 adjust their fields if needed, and layout record types
2764 of parallel/task constructs. */
2767 finish_taskreg_scan (omp_context
*ctx
)
2769 if (ctx
->record_type
== NULL_TREE
)
2772 /* If any task_shared_vars were needed, verify all
2773 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2774 statements if use_pointer_for_field hasn't changed
2775 because of that. If it did, update field types now. */
2776 if (task_shared_vars
)
2780 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2781 c
; c
= OMP_CLAUSE_CHAIN (c
))
2782 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2783 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2785 tree decl
= OMP_CLAUSE_DECL (c
);
2787 /* Global variables don't need to be copied,
2788 the receiver side will use them directly. */
2789 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2791 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2792 || !use_pointer_for_field (decl
, ctx
))
2794 tree field
= lookup_field (decl
, ctx
);
2795 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2796 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2798 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2799 TREE_THIS_VOLATILE (field
) = 0;
2800 DECL_USER_ALIGN (field
) = 0;
2801 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2802 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2803 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2804 if (ctx
->srecord_type
)
2806 tree sfield
= lookup_sfield (decl
, ctx
);
2807 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2808 TREE_THIS_VOLATILE (sfield
) = 0;
2809 DECL_USER_ALIGN (sfield
) = 0;
2810 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2811 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2812 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2817 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2819 layout_type (ctx
->record_type
);
2820 fixup_child_record_type (ctx
);
2824 location_t loc
= gimple_location (ctx
->stmt
);
2825 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2826 /* Move VLA fields to the end. */
2827 p
= &TYPE_FIELDS (ctx
->record_type
);
2829 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2830 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2833 *p
= TREE_CHAIN (*p
);
2834 TREE_CHAIN (*q
) = NULL_TREE
;
2835 q
= &TREE_CHAIN (*q
);
2838 p
= &DECL_CHAIN (*p
);
2840 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2842 /* Move fields corresponding to first and second _looptemp_
2843 clause first. There are filled by GOMP_taskloop
2844 and thus need to be in specific positions. */
2845 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2846 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2847 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2848 OMP_CLAUSE__LOOPTEMP_
);
2849 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2850 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2851 p
= &TYPE_FIELDS (ctx
->record_type
);
2853 if (*p
== f1
|| *p
== f2
)
2854 *p
= DECL_CHAIN (*p
);
2856 p
= &DECL_CHAIN (*p
);
2857 DECL_CHAIN (f1
) = f2
;
2858 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2859 TYPE_FIELDS (ctx
->record_type
) = f1
;
2860 if (ctx
->srecord_type
)
2862 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2863 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2864 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2866 if (*p
== f1
|| *p
== f2
)
2867 *p
= DECL_CHAIN (*p
);
2869 p
= &DECL_CHAIN (*p
);
2870 DECL_CHAIN (f1
) = f2
;
2871 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2872 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2875 layout_type (ctx
->record_type
);
2876 fixup_child_record_type (ctx
);
2877 if (ctx
->srecord_type
)
2878 layout_type (ctx
->srecord_type
);
2879 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2880 TYPE_SIZE_UNIT (ctx
->record_type
));
2881 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2882 t
= build_int_cst (long_integer_type_node
,
2883 TYPE_ALIGN_UNIT (ctx
->record_type
));
2884 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2888 /* Find the enclosing offload context. */
2890 static omp_context
*
2891 enclosing_target_ctx (omp_context
*ctx
)
2893 for (; ctx
; ctx
= ctx
->outer
)
2894 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2900 /* Return true if ctx is part of an oacc kernels region. */
2903 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2905 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2907 gimple
*stmt
= ctx
->stmt
;
2908 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2909 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2916 /* Check the parallelism clauses inside a kernels regions.
2917 Until kernels handling moves to use the same loop indirection
2918 scheme as parallel, we need to do this checking early. */
2921 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2923 bool checking
= true;
2924 unsigned outer_mask
= 0;
2925 unsigned this_mask
= 0;
2926 bool has_seq
= false, has_auto
= false;
2929 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2933 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2935 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2938 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2940 switch (OMP_CLAUSE_CODE (c
))
2942 case OMP_CLAUSE_GANG
:
2943 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2945 case OMP_CLAUSE_WORKER
:
2946 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2948 case OMP_CLAUSE_VECTOR
:
2949 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2951 case OMP_CLAUSE_SEQ
:
2954 case OMP_CLAUSE_AUTO
:
2964 if (has_seq
&& (this_mask
|| has_auto
))
2965 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2966 " OpenACC loop specifiers");
2967 else if (has_auto
&& this_mask
)
2968 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2969 " OpenACC loop specifiers");
2971 if (this_mask
& outer_mask
)
2972 error_at (gimple_location (stmt
), "inner loop uses same"
2973 " OpenACC parallelism as containing loop");
2976 return outer_mask
| this_mask
;
2979 /* Scan a GIMPLE_OMP_FOR. */
2982 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2986 tree clauses
= gimple_omp_for_clauses (stmt
);
2988 ctx
= new_omp_context (stmt
, outer_ctx
);
2990 if (is_gimple_omp_oacc (stmt
))
2992 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2994 if (!tgt
|| is_oacc_parallel (tgt
))
2995 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2997 char const *check
= NULL
;
2999 switch (OMP_CLAUSE_CODE (c
))
3001 case OMP_CLAUSE_GANG
:
3005 case OMP_CLAUSE_WORKER
:
3009 case OMP_CLAUSE_VECTOR
:
3017 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3018 error_at (gimple_location (stmt
),
3019 "argument not permitted on %qs clause in"
3020 " OpenACC %<parallel%>", check
);
3023 if (tgt
&& is_oacc_kernels (tgt
))
3025 /* Strip out reductions, as they are not handled yet. */
3026 tree
*prev_ptr
= &clauses
;
3028 while (tree probe
= *prev_ptr
)
3030 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3032 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3033 *prev_ptr
= *next_ptr
;
3035 prev_ptr
= next_ptr
;
3038 gimple_omp_for_set_clauses (stmt
, clauses
);
3039 check_oacc_kernel_gwv (stmt
, ctx
);
3043 scan_sharing_clauses (clauses
, ctx
);
3045 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3046 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3048 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3049 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3050 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3051 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3053 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3056 /* Scan an OpenMP sections directive. */
3059 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3063 ctx
= new_omp_context (stmt
, outer_ctx
);
3064 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3065 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3068 /* Scan an OpenMP single directive. */
3071 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3076 ctx
= new_omp_context (stmt
, outer_ctx
);
3077 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3078 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3079 name
= create_tmp_var_name (".omp_copy_s");
3080 name
= build_decl (gimple_location (stmt
),
3081 TYPE_DECL
, name
, ctx
->record_type
);
3082 TYPE_NAME (ctx
->record_type
) = name
;
3084 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3085 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3087 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3088 ctx
->record_type
= NULL
;
3090 layout_type (ctx
->record_type
);
3093 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3094 used in the corresponding offloaded function are restrict. */
3097 omp_target_base_pointers_restrict_p (tree clauses
)
3099 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3101 if (flag_openacc
== 0)
3104 /* I. Basic example:
3108 unsigned int a[2], b[2];
3110 #pragma acc kernels \
3119 After gimplification, we have:
3121 #pragma omp target oacc_kernels \
3122 map(force_from:a [len: 8]) \
3123 map(force_from:b [len: 8])
3129 Because both mappings have the force prefix, we know that they will be
3130 allocated when calling the corresponding offloaded function, which means we
3131 can mark the base pointers for a and b in the offloaded function as
3135 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3137 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
3140 switch (OMP_CLAUSE_MAP_KIND (c
))
3142 case GOMP_MAP_FORCE_ALLOC
:
3143 case GOMP_MAP_FORCE_TO
:
3144 case GOMP_MAP_FORCE_FROM
:
3145 case GOMP_MAP_FORCE_TOFROM
:
3155 /* Scan a GIMPLE_OMP_TARGET. */
3158 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3162 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3163 tree clauses
= gimple_omp_target_clauses (stmt
);
3165 ctx
= new_omp_context (stmt
, outer_ctx
);
3166 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3167 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3168 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3169 name
= create_tmp_var_name (".omp_data_t");
3170 name
= build_decl (gimple_location (stmt
),
3171 TYPE_DECL
, name
, ctx
->record_type
);
3172 DECL_ARTIFICIAL (name
) = 1;
3173 DECL_NAMELESS (name
) = 1;
3174 TYPE_NAME (ctx
->record_type
) = name
;
3175 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3177 bool base_pointers_restrict
= false;
3180 create_omp_child_function (ctx
, false);
3181 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3183 base_pointers_restrict
= omp_target_base_pointers_restrict_p (clauses
);
3184 if (base_pointers_restrict
3185 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3187 "Base pointers in offloaded function are restrict\n");
3190 scan_sharing_clauses (clauses
, ctx
, base_pointers_restrict
);
3191 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3193 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3194 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3197 TYPE_FIELDS (ctx
->record_type
)
3198 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3201 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3202 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3204 field
= DECL_CHAIN (field
))
3205 gcc_assert (DECL_ALIGN (field
) == align
);
3207 layout_type (ctx
->record_type
);
3209 fixup_child_record_type (ctx
);
3213 /* Scan an OpenMP teams directive. */
3216 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3218 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3219 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3220 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3223 /* Check nesting restrictions. */
3225 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3229 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
3230 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3231 the original copy of its contents. */
3234 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3235 inside an OpenACC CTX. */
3236 if (!(is_gimple_omp (stmt
)
3237 && is_gimple_omp_oacc (stmt
))
3238 /* Except for atomic codes that we share with OpenMP. */
3239 && !(gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3240 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3242 if (get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3244 error_at (gimple_location (stmt
),
3245 "non-OpenACC construct inside of OpenACC routine");
3249 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3250 if (is_gimple_omp (octx
->stmt
)
3251 && is_gimple_omp_oacc (octx
->stmt
))
3253 error_at (gimple_location (stmt
),
3254 "non-OpenACC construct inside of OpenACC region");
3261 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3262 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3265 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3267 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3268 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3270 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3271 && (ctx
->outer
== NULL
3272 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3273 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3274 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3275 != GF_OMP_FOR_KIND_FOR
)
3276 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3278 error_at (gimple_location (stmt
),
3279 "%<ordered simd threads%> must be closely "
3280 "nested inside of %<for simd%> region");
3286 error_at (gimple_location (stmt
),
3287 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3288 " may not be nested inside %<simd%> region");
3291 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3293 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3294 || (gimple_omp_for_kind (stmt
)
3295 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3296 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3298 error_at (gimple_location (stmt
),
3299 "only %<distribute%> or %<parallel%> regions are "
3300 "allowed to be strictly nested inside %<teams%> "
3306 switch (gimple_code (stmt
))
3308 case GIMPLE_OMP_FOR
:
3309 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3311 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3313 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3315 error_at (gimple_location (stmt
),
3316 "%<distribute%> region must be strictly nested "
3317 "inside %<teams%> construct");
3322 /* We split taskloop into task and nested taskloop in it. */
3323 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3325 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3330 switch (gimple_code (ctx
->stmt
))
3332 case GIMPLE_OMP_FOR
:
3333 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3334 == GF_OMP_FOR_KIND_OACC_LOOP
);
3337 case GIMPLE_OMP_TARGET
:
3338 switch (gimple_omp_target_kind (ctx
->stmt
))
3340 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3341 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3352 else if (get_oacc_fn_attrib (current_function_decl
))
3356 error_at (gimple_location (stmt
),
3357 "OpenACC loop directive must be associated with"
3358 " an OpenACC compute region");
3364 if (is_gimple_call (stmt
)
3365 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3366 == BUILT_IN_GOMP_CANCEL
3367 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3368 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3370 const char *bad
= NULL
;
3371 const char *kind
= NULL
;
3372 const char *construct
3373 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3374 == BUILT_IN_GOMP_CANCEL
)
3375 ? "#pragma omp cancel"
3376 : "#pragma omp cancellation point";
3379 error_at (gimple_location (stmt
), "orphaned %qs construct",
3383 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3384 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3388 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3389 bad
= "#pragma omp parallel";
3390 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3391 == BUILT_IN_GOMP_CANCEL
3392 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3393 ctx
->cancellable
= true;
3397 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3398 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3399 bad
= "#pragma omp for";
3400 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3401 == BUILT_IN_GOMP_CANCEL
3402 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3404 ctx
->cancellable
= true;
3405 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3407 warning_at (gimple_location (stmt
), 0,
3408 "%<#pragma omp cancel for%> inside "
3409 "%<nowait%> for construct");
3410 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3411 OMP_CLAUSE_ORDERED
))
3412 warning_at (gimple_location (stmt
), 0,
3413 "%<#pragma omp cancel for%> inside "
3414 "%<ordered%> for construct");
3419 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3420 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3421 bad
= "#pragma omp sections";
3422 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3423 == BUILT_IN_GOMP_CANCEL
3424 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3426 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3428 ctx
->cancellable
= true;
3429 if (find_omp_clause (gimple_omp_sections_clauses
3432 warning_at (gimple_location (stmt
), 0,
3433 "%<#pragma omp cancel sections%> inside "
3434 "%<nowait%> sections construct");
3438 gcc_assert (ctx
->outer
3439 && gimple_code (ctx
->outer
->stmt
)
3440 == GIMPLE_OMP_SECTIONS
);
3441 ctx
->outer
->cancellable
= true;
3442 if (find_omp_clause (gimple_omp_sections_clauses
3445 warning_at (gimple_location (stmt
), 0,
3446 "%<#pragma omp cancel sections%> inside "
3447 "%<nowait%> sections construct");
3453 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3454 bad
= "#pragma omp task";
3457 for (omp_context
*octx
= ctx
->outer
;
3458 octx
; octx
= octx
->outer
)
3460 switch (gimple_code (octx
->stmt
))
3462 case GIMPLE_OMP_TASKGROUP
:
3464 case GIMPLE_OMP_TARGET
:
3465 if (gimple_omp_target_kind (octx
->stmt
)
3466 != GF_OMP_TARGET_KIND_REGION
)
3469 case GIMPLE_OMP_PARALLEL
:
3470 case GIMPLE_OMP_TEAMS
:
3471 error_at (gimple_location (stmt
),
3472 "%<%s taskgroup%> construct not closely "
3473 "nested inside of %<taskgroup%> region",
3481 ctx
->cancellable
= true;
3486 error_at (gimple_location (stmt
), "invalid arguments");
3491 error_at (gimple_location (stmt
),
3492 "%<%s %s%> construct not closely nested inside of %qs",
3493 construct
, kind
, bad
);
3498 case GIMPLE_OMP_SECTIONS
:
3499 case GIMPLE_OMP_SINGLE
:
3500 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3501 switch (gimple_code (ctx
->stmt
))
3503 case GIMPLE_OMP_FOR
:
3504 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3505 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3508 case GIMPLE_OMP_SECTIONS
:
3509 case GIMPLE_OMP_SINGLE
:
3510 case GIMPLE_OMP_ORDERED
:
3511 case GIMPLE_OMP_MASTER
:
3512 case GIMPLE_OMP_TASK
:
3513 case GIMPLE_OMP_CRITICAL
:
3514 if (is_gimple_call (stmt
))
3516 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3517 != BUILT_IN_GOMP_BARRIER
)
3519 error_at (gimple_location (stmt
),
3520 "barrier region may not be closely nested inside "
3521 "of work-sharing, %<critical%>, %<ordered%>, "
3522 "%<master%>, explicit %<task%> or %<taskloop%> "
3526 error_at (gimple_location (stmt
),
3527 "work-sharing region may not be closely nested inside "
3528 "of work-sharing, %<critical%>, %<ordered%>, "
3529 "%<master%>, explicit %<task%> or %<taskloop%> region");
3531 case GIMPLE_OMP_PARALLEL
:
3532 case GIMPLE_OMP_TEAMS
:
3534 case GIMPLE_OMP_TARGET
:
3535 if (gimple_omp_target_kind (ctx
->stmt
)
3536 == GF_OMP_TARGET_KIND_REGION
)
3543 case GIMPLE_OMP_MASTER
:
3544 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3545 switch (gimple_code (ctx
->stmt
))
3547 case GIMPLE_OMP_FOR
:
3548 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3549 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3552 case GIMPLE_OMP_SECTIONS
:
3553 case GIMPLE_OMP_SINGLE
:
3554 case GIMPLE_OMP_TASK
:
3555 error_at (gimple_location (stmt
),
3556 "%<master%> region may not be closely nested inside "
3557 "of work-sharing, explicit %<task%> or %<taskloop%> "
3560 case GIMPLE_OMP_PARALLEL
:
3561 case GIMPLE_OMP_TEAMS
:
3563 case GIMPLE_OMP_TARGET
:
3564 if (gimple_omp_target_kind (ctx
->stmt
)
3565 == GF_OMP_TARGET_KIND_REGION
)
3572 case GIMPLE_OMP_TASK
:
3573 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3574 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3575 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3576 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3578 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3579 error_at (OMP_CLAUSE_LOCATION (c
),
3580 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3581 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3585 case GIMPLE_OMP_ORDERED
:
3586 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3587 c
; c
= OMP_CLAUSE_CHAIN (c
))
3589 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3591 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3592 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3595 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3596 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3597 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3600 /* Look for containing ordered(N) loop. */
3602 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3604 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3605 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3607 error_at (OMP_CLAUSE_LOCATION (c
),
3608 "%<ordered%> construct with %<depend%> clause "
3609 "must be closely nested inside an %<ordered%> "
3613 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3615 error_at (OMP_CLAUSE_LOCATION (c
),
3616 "%<ordered%> construct with %<depend%> clause "
3617 "must be closely nested inside a loop with "
3618 "%<ordered%> clause with a parameter");
3624 error_at (OMP_CLAUSE_LOCATION (c
),
3625 "invalid depend kind in omp %<ordered%> %<depend%>");
3629 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3630 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3632 /* ordered simd must be closely nested inside of simd region,
3633 and simd region must not encounter constructs other than
3634 ordered simd, therefore ordered simd may be either orphaned,
3635 or ctx->stmt must be simd. The latter case is handled already
3639 error_at (gimple_location (stmt
),
3640 "%<ordered%> %<simd%> must be closely nested inside "
3645 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3646 switch (gimple_code (ctx
->stmt
))
3648 case GIMPLE_OMP_CRITICAL
:
3649 case GIMPLE_OMP_TASK
:
3650 case GIMPLE_OMP_ORDERED
:
3651 ordered_in_taskloop
:
3652 error_at (gimple_location (stmt
),
3653 "%<ordered%> region may not be closely nested inside "
3654 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3655 "%<taskloop%> region");
3657 case GIMPLE_OMP_FOR
:
3658 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3659 goto ordered_in_taskloop
;
3660 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3661 OMP_CLAUSE_ORDERED
) == NULL
)
3663 error_at (gimple_location (stmt
),
3664 "%<ordered%> region must be closely nested inside "
3665 "a loop region with an %<ordered%> clause");
3669 case GIMPLE_OMP_TARGET
:
3670 if (gimple_omp_target_kind (ctx
->stmt
)
3671 != GF_OMP_TARGET_KIND_REGION
)
3674 case GIMPLE_OMP_PARALLEL
:
3675 case GIMPLE_OMP_TEAMS
:
3676 error_at (gimple_location (stmt
),
3677 "%<ordered%> region must be closely nested inside "
3678 "a loop region with an %<ordered%> clause");
3684 case GIMPLE_OMP_CRITICAL
:
3687 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3688 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3689 if (gomp_critical
*other_crit
3690 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3691 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3693 error_at (gimple_location (stmt
),
3694 "%<critical%> region may not be nested inside "
3695 "a %<critical%> region with the same name");
3700 case GIMPLE_OMP_TEAMS
:
3702 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3703 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3705 error_at (gimple_location (stmt
),
3706 "%<teams%> construct not closely nested inside of "
3707 "%<target%> construct");
3711 case GIMPLE_OMP_TARGET
:
3712 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3713 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3714 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3715 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3717 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3718 error_at (OMP_CLAUSE_LOCATION (c
),
3719 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3720 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3723 if (is_gimple_omp_offloaded (stmt
)
3724 && get_oacc_fn_attrib (cfun
->decl
) != NULL
)
3726 error_at (gimple_location (stmt
),
3727 "OpenACC region inside of OpenACC routine, nested "
3728 "parallelism not supported yet");
3731 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3733 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3735 if (is_gimple_omp (stmt
)
3736 && is_gimple_omp_oacc (stmt
)
3737 && is_gimple_omp (ctx
->stmt
))
3739 error_at (gimple_location (stmt
),
3740 "OpenACC construct inside of non-OpenACC region");
3746 const char *stmt_name
, *ctx_stmt_name
;
3747 switch (gimple_omp_target_kind (stmt
))
3749 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3750 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3751 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3752 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3753 stmt_name
= "target enter data"; break;
3754 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3755 stmt_name
= "target exit data"; break;
3756 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3757 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3758 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3759 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3760 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3761 stmt_name
= "enter/exit data"; break;
3762 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3764 default: gcc_unreachable ();
3766 switch (gimple_omp_target_kind (ctx
->stmt
))
3768 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3769 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3770 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3771 ctx_stmt_name
= "parallel"; break;
3772 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3773 ctx_stmt_name
= "kernels"; break;
3774 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3775 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3776 ctx_stmt_name
= "host_data"; break;
3777 default: gcc_unreachable ();
3780 /* OpenACC/OpenMP mismatch? */
3781 if (is_gimple_omp_oacc (stmt
)
3782 != is_gimple_omp_oacc (ctx
->stmt
))
3784 error_at (gimple_location (stmt
),
3785 "%s %qs construct inside of %s %qs region",
3786 (is_gimple_omp_oacc (stmt
)
3787 ? "OpenACC" : "OpenMP"), stmt_name
,
3788 (is_gimple_omp_oacc (ctx
->stmt
)
3789 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3792 if (is_gimple_omp_offloaded (ctx
->stmt
))
3794 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3795 if (is_gimple_omp_oacc (ctx
->stmt
))
3797 error_at (gimple_location (stmt
),
3798 "%qs construct inside of %qs region",
3799 stmt_name
, ctx_stmt_name
);
3804 warning_at (gimple_location (stmt
), 0,
3805 "%qs construct inside of %qs region",
3806 stmt_name
, ctx_stmt_name
);
3818 /* Helper function scan_omp.
3820 Callback for walk_tree or operators in walk_gimple_stmt used to
3821 scan for OMP directives in TP. */
3824 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3826 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3827 omp_context
*ctx
= (omp_context
*) wi
->info
;
3830 switch (TREE_CODE (t
))
3838 tree repl
= remap_decl (t
, &ctx
->cb
);
3839 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3845 if (ctx
&& TYPE_P (t
))
3846 *tp
= remap_type (t
, &ctx
->cb
);
3847 else if (!DECL_P (t
))
3852 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3853 if (tem
!= TREE_TYPE (t
))
3855 if (TREE_CODE (t
) == INTEGER_CST
)
3856 *tp
= wide_int_to_tree (tem
, t
);
3858 TREE_TYPE (t
) = tem
;
3868 /* Return true if FNDECL is a setjmp or a longjmp. */
3871 setjmp_or_longjmp_p (const_tree fndecl
)
3873 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3874 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3875 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3878 tree declname
= DECL_NAME (fndecl
);
3881 const char *name
= IDENTIFIER_POINTER (declname
);
3882 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3886 /* Helper function for scan_omp.
3888 Callback for walk_gimple_stmt used to scan for OMP directives in
3889 the current statement in GSI. */
3892 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3893 struct walk_stmt_info
*wi
)
3895 gimple
*stmt
= gsi_stmt (*gsi
);
3896 omp_context
*ctx
= (omp_context
*) wi
->info
;
3898 if (gimple_has_location (stmt
))
3899 input_location
= gimple_location (stmt
);
3901 /* Check the nesting restrictions. */
3902 bool remove
= false;
3903 if (is_gimple_omp (stmt
))
3904 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3905 else if (is_gimple_call (stmt
))
3907 tree fndecl
= gimple_call_fndecl (stmt
);
3910 if (setjmp_or_longjmp_p (fndecl
)
3912 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3913 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3916 error_at (gimple_location (stmt
),
3917 "setjmp/longjmp inside simd construct");
3919 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3920 switch (DECL_FUNCTION_CODE (fndecl
))
3922 case BUILT_IN_GOMP_BARRIER
:
3923 case BUILT_IN_GOMP_CANCEL
:
3924 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3925 case BUILT_IN_GOMP_TASKYIELD
:
3926 case BUILT_IN_GOMP_TASKWAIT
:
3927 case BUILT_IN_GOMP_TASKGROUP_START
:
3928 case BUILT_IN_GOMP_TASKGROUP_END
:
3929 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3938 stmt
= gimple_build_nop ();
3939 gsi_replace (gsi
, stmt
, false);
3942 *handled_ops_p
= true;
3944 switch (gimple_code (stmt
))
3946 case GIMPLE_OMP_PARALLEL
:
3947 taskreg_nesting_level
++;
3948 scan_omp_parallel (gsi
, ctx
);
3949 taskreg_nesting_level
--;
3952 case GIMPLE_OMP_TASK
:
3953 taskreg_nesting_level
++;
3954 scan_omp_task (gsi
, ctx
);
3955 taskreg_nesting_level
--;
3958 case GIMPLE_OMP_FOR
:
3959 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3962 case GIMPLE_OMP_SECTIONS
:
3963 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3966 case GIMPLE_OMP_SINGLE
:
3967 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3970 case GIMPLE_OMP_SECTION
:
3971 case GIMPLE_OMP_MASTER
:
3972 case GIMPLE_OMP_TASKGROUP
:
3973 case GIMPLE_OMP_ORDERED
:
3974 case GIMPLE_OMP_CRITICAL
:
3975 case GIMPLE_OMP_GRID_BODY
:
3976 ctx
= new_omp_context (stmt
, ctx
);
3977 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3980 case GIMPLE_OMP_TARGET
:
3981 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3984 case GIMPLE_OMP_TEAMS
:
3985 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3992 *handled_ops_p
= false;
3994 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3996 var
= DECL_CHAIN (var
))
3997 insert_decl_map (&ctx
->cb
, var
, var
);
4001 *handled_ops_p
= false;
4009 /* Scan all the statements starting at the current statement. CTX
4010 contains context information about the OMP directives and
4011 clauses found during the scan. */
4014 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
4016 location_t saved_location
;
4017 struct walk_stmt_info wi
;
4019 memset (&wi
, 0, sizeof (wi
));
4021 wi
.want_locations
= true;
4023 saved_location
= input_location
;
4024 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
4025 input_location
= saved_location
;
4028 /* Re-gimplification and code generation routines. */
4030 /* Build a call to GOMP_barrier. */
4033 build_omp_barrier (tree lhs
)
4035 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
4036 : BUILT_IN_GOMP_BARRIER
);
4037 gcall
*g
= gimple_build_call (fndecl
, 0);
4039 gimple_call_set_lhs (g
, lhs
);
4043 /* If a context was created for STMT when it was scanned, return it. */
4045 static omp_context
*
4046 maybe_lookup_ctx (gimple
*stmt
)
4049 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
4050 return n
? (omp_context
*) n
->value
: NULL
;
4054 /* Find the mapping for DECL in CTX or the immediately enclosing
4055 context that has a mapping for DECL.
4057 If CTX is a nested parallel directive, we may have to use the decl
4058 mappings created in CTX's parent context. Suppose that we have the
4059 following parallel nesting (variable UIDs showed for clarity):
4062 #omp parallel shared(iD.1562) -> outer parallel
4063 iD.1562 = iD.1562 + 1;
4065 #omp parallel shared (iD.1562) -> inner parallel
4066 iD.1562 = iD.1562 - 1;
4068 Each parallel structure will create a distinct .omp_data_s structure
4069 for copying iD.1562 in/out of the directive:
4071 outer parallel .omp_data_s.1.i -> iD.1562
4072 inner parallel .omp_data_s.2.i -> iD.1562
4074 A shared variable mapping will produce a copy-out operation before
4075 the parallel directive and a copy-in operation after it. So, in
4076 this case we would have:
4079 .omp_data_o.1.i = iD.1562;
4080 #omp parallel shared(iD.1562) -> outer parallel
4081 .omp_data_i.1 = &.omp_data_o.1
4082 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4084 .omp_data_o.2.i = iD.1562; -> **
4085 #omp parallel shared(iD.1562) -> inner parallel
4086 .omp_data_i.2 = &.omp_data_o.2
4087 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4090 ** This is a problem. The symbol iD.1562 cannot be referenced
4091 inside the body of the outer parallel region. But since we are
4092 emitting this copy operation while expanding the inner parallel
4093 directive, we need to access the CTX structure of the outer
4094 parallel directive to get the correct mapping:
4096 .omp_data_o.2.i = .omp_data_i.1->i
4098 Since there may be other workshare or parallel directives enclosing
4099 the parallel directive, it may be necessary to walk up the context
4100 parent chain. This is not a problem in general because nested
4101 parallelism happens only rarely. */
4104 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4109 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4110 t
= maybe_lookup_decl (decl
, up
);
4112 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
4114 return t
? t
: decl
;
4118 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4119 in outer contexts. */
4122 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4127 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4128 t
= maybe_lookup_decl (decl
, up
);
4130 return t
? t
: decl
;
4134 /* Construct the initialization value for reduction operation OP. */
4137 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4146 case TRUTH_ORIF_EXPR
:
4147 case TRUTH_XOR_EXPR
:
4149 return build_zero_cst (type
);
4152 case TRUTH_AND_EXPR
:
4153 case TRUTH_ANDIF_EXPR
:
4155 return fold_convert_loc (loc
, type
, integer_one_node
);
4158 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4161 if (SCALAR_FLOAT_TYPE_P (type
))
4163 REAL_VALUE_TYPE max
, min
;
4164 if (HONOR_INFINITIES (type
))
4167 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4170 real_maxval (&min
, 1, TYPE_MODE (type
));
4171 return build_real (type
, min
);
4173 else if (POINTER_TYPE_P (type
))
4176 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4177 return wide_int_to_tree (type
, min
);
4181 gcc_assert (INTEGRAL_TYPE_P (type
));
4182 return TYPE_MIN_VALUE (type
);
4186 if (SCALAR_FLOAT_TYPE_P (type
))
4188 REAL_VALUE_TYPE max
;
4189 if (HONOR_INFINITIES (type
))
4192 real_maxval (&max
, 0, TYPE_MODE (type
));
4193 return build_real (type
, max
);
4195 else if (POINTER_TYPE_P (type
))
4198 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4199 return wide_int_to_tree (type
, max
);
4203 gcc_assert (INTEGRAL_TYPE_P (type
));
4204 return TYPE_MAX_VALUE (type
);
4212 /* Construct the initialization value for reduction CLAUSE. */
4215 omp_reduction_init (tree clause
, tree type
)
4217 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4218 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4221 /* Return alignment to be assumed for var in CLAUSE, which should be
4222 OMP_CLAUSE_ALIGNED. */
4225 omp_clause_aligned_alignment (tree clause
)
4227 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4228 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4230 /* Otherwise return implementation defined alignment. */
4231 unsigned int al
= 1;
4232 machine_mode mode
, vmode
;
4233 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4235 vs
= 1 << floor_log2 (vs
);
4236 static enum mode_class classes
[]
4237 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4238 for (int i
= 0; i
< 4; i
+= 2)
4239 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4241 mode
= GET_MODE_WIDER_MODE (mode
))
4243 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4244 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4247 && GET_MODE_SIZE (vmode
) < vs
4248 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4249 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4251 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4252 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4254 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4255 / GET_MODE_SIZE (mode
));
4256 if (TYPE_MODE (type
) != vmode
)
4258 if (TYPE_ALIGN_UNIT (type
) > al
)
4259 al
= TYPE_ALIGN_UNIT (type
);
4261 return build_int_cst (integer_type_node
, al
);
4264 /* Return maximum possible vectorization factor for the target. */
4271 || !flag_tree_loop_optimize
4272 || (!flag_tree_loop_vectorize
4273 && (global_options_set
.x_flag_tree_loop_vectorize
4274 || global_options_set
.x_flag_tree_vectorize
)))
4277 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4280 vs
= 1 << floor_log2 (vs
);
4283 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4284 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4285 return GET_MODE_NUNITS (vqimode
);
4289 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4293 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4294 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4298 max_vf
= omp_max_vf ();
4301 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4302 OMP_CLAUSE_SAFELEN
);
4303 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4305 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4307 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4311 idx
= create_tmp_var (unsigned_type_node
);
4312 lane
= create_tmp_var (unsigned_type_node
);
4318 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4319 tree avar
= create_tmp_var_raw (atype
);
4320 if (TREE_ADDRESSABLE (new_var
))
4321 TREE_ADDRESSABLE (avar
) = 1;
4322 DECL_ATTRIBUTES (avar
)
4323 = tree_cons (get_identifier ("omp simd array"), NULL
,
4324 DECL_ATTRIBUTES (avar
));
4325 gimple_add_tmp_var (avar
);
4326 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4327 NULL_TREE
, NULL_TREE
);
4328 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4329 NULL_TREE
, NULL_TREE
);
4330 if (DECL_P (new_var
))
4332 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4333 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4338 /* Helper function of lower_rec_input_clauses. For a reference
4339 in simd reduction, add an underlying variable it will reference. */
4342 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4344 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4345 if (TREE_CONSTANT (z
))
4347 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4348 get_name (new_vard
));
4349 gimple_add_tmp_var (z
);
4350 TREE_ADDRESSABLE (z
) = 1;
4351 z
= build_fold_addr_expr_loc (loc
, z
);
4352 gimplify_assign (new_vard
, z
, ilist
);
4356 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4357 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4358 private variables. Initialization statements go in ILIST, while calls
4359 to destructors go in DLIST. */
4362 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4363 omp_context
*ctx
, struct omp_for_data
*fd
)
4365 tree c
, dtor
, copyin_seq
, x
, ptr
;
4366 bool copyin_by_ref
= false;
4367 bool lastprivate_firstprivate
= false;
4368 bool reduction_omp_orig_ref
= false;
4370 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4371 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4373 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4374 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4375 gimple_seq llist
[2] = { NULL
, NULL
};
4379 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4380 with data sharing clauses referencing variable sized vars. That
4381 is unnecessarily hard to support and very unlikely to result in
4382 vectorized code anyway. */
4384 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4385 switch (OMP_CLAUSE_CODE (c
))
4387 case OMP_CLAUSE_LINEAR
:
4388 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4391 case OMP_CLAUSE_PRIVATE
:
4392 case OMP_CLAUSE_FIRSTPRIVATE
:
4393 case OMP_CLAUSE_LASTPRIVATE
:
4394 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4397 case OMP_CLAUSE_REDUCTION
:
4398 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4399 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4406 /* Do all the fixed sized types in the first pass, and the variable sized
4407 types in the second pass. This makes sure that the scalar arguments to
4408 the variable sized types are processed before we use them in the
4409 variable sized operations. */
4410 for (pass
= 0; pass
< 2; ++pass
)
4412 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4414 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4417 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4421 case OMP_CLAUSE_PRIVATE
:
4422 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4425 case OMP_CLAUSE_SHARED
:
4426 /* Ignore shared directives in teams construct. */
4427 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4429 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4431 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4432 || is_global_var (OMP_CLAUSE_DECL (c
)));
4435 case OMP_CLAUSE_FIRSTPRIVATE
:
4436 case OMP_CLAUSE_COPYIN
:
4438 case OMP_CLAUSE_LINEAR
:
4439 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4440 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4441 lastprivate_firstprivate
= true;
4443 case OMP_CLAUSE_REDUCTION
:
4444 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4445 reduction_omp_orig_ref
= true;
4447 case OMP_CLAUSE__LOOPTEMP_
:
4448 /* Handle _looptemp_ clauses only on parallel/task. */
4452 case OMP_CLAUSE_LASTPRIVATE
:
4453 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4455 lastprivate_firstprivate
= true;
4456 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4459 /* Even without corresponding firstprivate, if
4460 decl is Fortran allocatable, it needs outer var
4463 && lang_hooks
.decls
.omp_private_outer_ref
4464 (OMP_CLAUSE_DECL (c
)))
4465 lastprivate_firstprivate
= true;
4467 case OMP_CLAUSE_ALIGNED
:
4470 var
= OMP_CLAUSE_DECL (c
);
4471 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4472 && !is_global_var (var
))
4474 new_var
= maybe_lookup_decl (var
, ctx
);
4475 if (new_var
== NULL_TREE
)
4476 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4477 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4478 tree alarg
= omp_clause_aligned_alignment (c
);
4479 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4480 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
, alarg
);
4481 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4482 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4483 gimplify_and_add (x
, ilist
);
4485 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4486 && is_global_var (var
))
4488 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4489 new_var
= lookup_decl (var
, ctx
);
4490 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4491 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4492 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4493 tree alarg
= omp_clause_aligned_alignment (c
);
4494 alarg
= fold_convert_loc (clause_loc
, size_type_node
, alarg
);
4495 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
, alarg
);
4496 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4497 x
= create_tmp_var (ptype
);
4498 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4499 gimplify_and_add (t
, ilist
);
4500 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4501 SET_DECL_VALUE_EXPR (new_var
, t
);
4502 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4509 new_var
= var
= OMP_CLAUSE_DECL (c
);
4510 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4512 var
= TREE_OPERAND (var
, 0);
4513 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4514 var
= TREE_OPERAND (var
, 0);
4515 if (TREE_CODE (var
) == INDIRECT_REF
4516 || TREE_CODE (var
) == ADDR_EXPR
)
4517 var
= TREE_OPERAND (var
, 0);
4518 if (is_variable_sized (var
))
4520 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4521 var
= DECL_VALUE_EXPR (var
);
4522 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4523 var
= TREE_OPERAND (var
, 0);
4524 gcc_assert (DECL_P (var
));
4528 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4529 new_var
= lookup_decl (var
, ctx
);
4531 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4536 /* C/C++ array section reductions. */
4537 else if (c_kind
== OMP_CLAUSE_REDUCTION
4538 && var
!= OMP_CLAUSE_DECL (c
))
4543 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4544 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4545 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4547 tree b
= TREE_OPERAND (orig_var
, 1);
4548 b
= maybe_lookup_decl (b
, ctx
);
4551 b
= TREE_OPERAND (orig_var
, 1);
4552 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4554 if (integer_zerop (bias
))
4558 bias
= fold_convert_loc (clause_loc
,
4559 TREE_TYPE (b
), bias
);
4560 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4561 TREE_TYPE (b
), b
, bias
);
4563 orig_var
= TREE_OPERAND (orig_var
, 0);
4565 if (TREE_CODE (orig_var
) == INDIRECT_REF
4566 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4567 orig_var
= TREE_OPERAND (orig_var
, 0);
4568 tree d
= OMP_CLAUSE_DECL (c
);
4569 tree type
= TREE_TYPE (d
);
4570 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4571 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4572 const char *name
= get_name (orig_var
);
4573 if (TREE_CONSTANT (v
))
4575 x
= create_tmp_var_raw (type
, name
);
4576 gimple_add_tmp_var (x
);
4577 TREE_ADDRESSABLE (x
) = 1;
4578 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4583 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4584 tree t
= maybe_lookup_decl (v
, ctx
);
4588 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4589 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4590 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4592 build_int_cst (TREE_TYPE (v
), 1));
4593 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4595 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4596 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4597 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4600 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4601 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4602 tree y
= create_tmp_var (ptype
, name
);
4603 gimplify_assign (y
, x
, ilist
);
4607 if (!integer_zerop (bias
))
4609 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4611 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4613 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4614 pointer_sized_int_node
, yb
, bias
);
4615 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4616 yb
= create_tmp_var (ptype
, name
);
4617 gimplify_assign (yb
, x
, ilist
);
4621 d
= TREE_OPERAND (d
, 0);
4622 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4623 d
= TREE_OPERAND (d
, 0);
4624 if (TREE_CODE (d
) == ADDR_EXPR
)
4626 if (orig_var
!= var
)
4628 gcc_assert (is_variable_sized (orig_var
));
4629 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4631 gimplify_assign (new_var
, x
, ilist
);
4632 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4633 tree t
= build_fold_indirect_ref (new_var
);
4634 DECL_IGNORED_P (new_var
) = 0;
4635 TREE_THIS_NOTRAP (t
);
4636 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4637 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4641 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4642 build_int_cst (ptype
, 0));
4643 SET_DECL_VALUE_EXPR (new_var
, x
);
4644 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4649 gcc_assert (orig_var
== var
);
4650 if (TREE_CODE (d
) == INDIRECT_REF
)
4652 x
= create_tmp_var (ptype
, name
);
4653 TREE_ADDRESSABLE (x
) = 1;
4654 gimplify_assign (x
, yb
, ilist
);
4655 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4657 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4658 gimplify_assign (new_var
, x
, ilist
);
4660 tree y1
= create_tmp_var (ptype
, NULL
);
4661 gimplify_assign (y1
, y
, ilist
);
4662 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4663 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4664 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4665 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4667 y2
= create_tmp_var (ptype
, NULL
);
4668 gimplify_assign (y2
, y
, ilist
);
4669 tree ref
= build_outer_var_ref (var
, ctx
);
4670 /* For ref build_outer_var_ref already performs this. */
4671 if (TREE_CODE (d
) == INDIRECT_REF
)
4672 gcc_assert (is_reference (var
));
4673 else if (TREE_CODE (d
) == ADDR_EXPR
)
4674 ref
= build_fold_addr_expr (ref
);
4675 else if (is_reference (var
))
4676 ref
= build_fold_addr_expr (ref
);
4677 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4678 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4679 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4681 y3
= create_tmp_var (ptype
, NULL
);
4682 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4686 y4
= create_tmp_var (ptype
, NULL
);
4687 gimplify_assign (y4
, ref
, dlist
);
4690 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4691 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4692 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4693 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4694 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4697 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4698 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4699 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4700 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4701 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4703 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4705 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4706 tree decl_placeholder
4707 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4708 SET_DECL_VALUE_EXPR (decl_placeholder
,
4709 build_simple_mem_ref (y1
));
4710 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4711 SET_DECL_VALUE_EXPR (placeholder
,
4712 y3
? build_simple_mem_ref (y3
)
4714 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4715 x
= lang_hooks
.decls
.omp_clause_default_ctor
4716 (c
, build_simple_mem_ref (y1
),
4717 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4719 gimplify_and_add (x
, ilist
);
4720 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4722 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4723 lower_omp (&tseq
, ctx
);
4724 gimple_seq_add_seq (ilist
, tseq
);
4726 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4729 SET_DECL_VALUE_EXPR (decl_placeholder
,
4730 build_simple_mem_ref (y2
));
4731 SET_DECL_VALUE_EXPR (placeholder
,
4732 build_simple_mem_ref (y4
));
4733 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4734 lower_omp (&tseq
, ctx
);
4735 gimple_seq_add_seq (dlist
, tseq
);
4736 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4738 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4739 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4740 x
= lang_hooks
.decls
.omp_clause_dtor
4741 (c
, build_simple_mem_ref (y2
));
4744 gimple_seq tseq
= NULL
;
4746 gimplify_stmt (&dtor
, &tseq
);
4747 gimple_seq_add_seq (dlist
, tseq
);
4752 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4753 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4755 /* reduction(-:var) sums up the partial results, so it
4756 acts identically to reduction(+:var). */
4757 if (code
== MINUS_EXPR
)
4760 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4763 x
= build2 (code
, TREE_TYPE (type
),
4764 build_simple_mem_ref (y4
),
4765 build_simple_mem_ref (y2
));
4766 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4770 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4771 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4772 gimple_seq_add_stmt (ilist
, g
);
4775 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4776 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4777 gimple_seq_add_stmt (ilist
, g
);
4779 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4780 build_int_cst (TREE_TYPE (i
), 1));
4781 gimple_seq_add_stmt (ilist
, g
);
4782 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4783 gimple_seq_add_stmt (ilist
, g
);
4784 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4787 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4788 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4789 gimple_seq_add_stmt (dlist
, g
);
4792 g
= gimple_build_assign
4793 (y4
, POINTER_PLUS_EXPR
, y4
,
4794 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4795 gimple_seq_add_stmt (dlist
, g
);
4797 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4798 build_int_cst (TREE_TYPE (i2
), 1));
4799 gimple_seq_add_stmt (dlist
, g
);
4800 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4801 gimple_seq_add_stmt (dlist
, g
);
4802 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4806 else if (is_variable_sized (var
))
4808 /* For variable sized types, we need to allocate the
4809 actual storage here. Call alloca and store the
4810 result in the pointer decl that we created elsewhere. */
4814 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4819 ptr
= DECL_VALUE_EXPR (new_var
);
4820 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4821 ptr
= TREE_OPERAND (ptr
, 0);
4822 gcc_assert (DECL_P (ptr
));
4823 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4825 /* void *tmp = __builtin_alloca */
4826 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4827 stmt
= gimple_build_call (atmp
, 2, x
,
4828 size_int (DECL_ALIGN (var
)));
4829 tmp
= create_tmp_var_raw (ptr_type_node
);
4830 gimple_add_tmp_var (tmp
);
4831 gimple_call_set_lhs (stmt
, tmp
);
4833 gimple_seq_add_stmt (ilist
, stmt
);
4835 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4836 gimplify_assign (ptr
, x
, ilist
);
4839 else if (is_reference (var
))
4841 /* For references that are being privatized for Fortran,
4842 allocate new backing storage for the new pointer
4843 variable. This allows us to avoid changing all the
4844 code that expects a pointer to something that expects
4845 a direct variable. */
4849 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4850 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4852 x
= build_receiver_ref (var
, false, ctx
);
4853 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4855 else if (TREE_CONSTANT (x
))
4857 /* For reduction in SIMD loop, defer adding the
4858 initialization of the reference, because if we decide
4859 to use SIMD array for it, the initilization could cause
4861 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4865 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4867 gimple_add_tmp_var (x
);
4868 TREE_ADDRESSABLE (x
) = 1;
4869 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4875 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4876 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4877 tree al
= size_int (TYPE_ALIGN (rtype
));
4878 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4883 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4884 gimplify_assign (new_var
, x
, ilist
);
4887 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4889 else if (c_kind
== OMP_CLAUSE_REDUCTION
4890 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4898 switch (OMP_CLAUSE_CODE (c
))
4900 case OMP_CLAUSE_SHARED
:
4901 /* Ignore shared directives in teams construct. */
4902 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4904 /* Shared global vars are just accessed directly. */
4905 if (is_global_var (new_var
))
4907 /* For taskloop firstprivate/lastprivate, represented
4908 as firstprivate and shared clause on the task, new_var
4909 is the firstprivate var. */
4910 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4912 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4913 needs to be delayed until after fixup_child_record_type so
4914 that we get the correct type during the dereference. */
4915 by_ref
= use_pointer_for_field (var
, ctx
);
4916 x
= build_receiver_ref (var
, by_ref
, ctx
);
4917 SET_DECL_VALUE_EXPR (new_var
, x
);
4918 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4920 /* ??? If VAR is not passed by reference, and the variable
4921 hasn't been initialized yet, then we'll get a warning for
4922 the store into the omp_data_s structure. Ideally, we'd be
4923 able to notice this and not store anything at all, but
4924 we're generating code too early. Suppress the warning. */
4926 TREE_NO_WARNING (var
) = 1;
4929 case OMP_CLAUSE_LASTPRIVATE
:
4930 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4934 case OMP_CLAUSE_PRIVATE
:
4935 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4936 x
= build_outer_var_ref (var
, ctx
);
4937 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4939 if (is_task_ctx (ctx
))
4940 x
= build_receiver_ref (var
, false, ctx
);
4942 x
= build_outer_var_ref (var
, ctx
);
4948 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4949 (c
, unshare_expr (new_var
), x
);
4952 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4953 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4954 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4955 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4956 idx
, lane
, ivar
, lvar
))
4959 x
= lang_hooks
.decls
.omp_clause_default_ctor
4960 (c
, unshare_expr (ivar
), x
);
4962 gimplify_and_add (x
, &llist
[0]);
4965 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4968 gimple_seq tseq
= NULL
;
4971 gimplify_stmt (&dtor
, &tseq
);
4972 gimple_seq_add_seq (&llist
[1], tseq
);
4979 gimplify_and_add (nx
, ilist
);
4983 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4986 gimple_seq tseq
= NULL
;
4989 gimplify_stmt (&dtor
, &tseq
);
4990 gimple_seq_add_seq (dlist
, tseq
);
4994 case OMP_CLAUSE_LINEAR
:
4995 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4996 goto do_firstprivate
;
4997 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
5000 x
= build_outer_var_ref (var
, ctx
);
5003 case OMP_CLAUSE_FIRSTPRIVATE
:
5004 if (is_task_ctx (ctx
))
5006 if (is_reference (var
) || is_variable_sized (var
))
5008 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
5010 || use_pointer_for_field (var
, NULL
))
5012 x
= build_receiver_ref (var
, false, ctx
);
5013 SET_DECL_VALUE_EXPR (new_var
, x
);
5014 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5019 x
= build_outer_var_ref (var
, ctx
);
5022 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5023 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5025 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5026 tree stept
= TREE_TYPE (t
);
5027 tree ct
= find_omp_clause (clauses
,
5028 OMP_CLAUSE__LOOPTEMP_
);
5030 tree l
= OMP_CLAUSE_DECL (ct
);
5031 tree n1
= fd
->loop
.n1
;
5032 tree step
= fd
->loop
.step
;
5033 tree itype
= TREE_TYPE (l
);
5034 if (POINTER_TYPE_P (itype
))
5035 itype
= signed_type_for (itype
);
5036 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5037 if (TYPE_UNSIGNED (itype
)
5038 && fd
->loop
.cond_code
== GT_EXPR
)
5039 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5040 fold_build1 (NEGATE_EXPR
, itype
, l
),
5041 fold_build1 (NEGATE_EXPR
,
5044 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5045 t
= fold_build2 (MULT_EXPR
, stept
,
5046 fold_convert (stept
, l
), t
);
5048 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5050 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5052 gimplify_and_add (x
, ilist
);
5056 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5057 x
= fold_build2 (POINTER_PLUS_EXPR
,
5058 TREE_TYPE (x
), x
, t
);
5060 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5063 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5064 || TREE_ADDRESSABLE (new_var
))
5065 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5066 idx
, lane
, ivar
, lvar
))
5068 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5070 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5071 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5072 gimplify_and_add (x
, ilist
);
5073 gimple_stmt_iterator gsi
5074 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5076 = gimple_build_assign (unshare_expr (lvar
), iv
);
5077 gsi_insert_before_without_update (&gsi
, g
,
5079 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5080 enum tree_code code
= PLUS_EXPR
;
5081 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5082 code
= POINTER_PLUS_EXPR
;
5083 g
= gimple_build_assign (iv
, code
, iv
, t
);
5084 gsi_insert_before_without_update (&gsi
, g
,
5088 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5089 (c
, unshare_expr (ivar
), x
);
5090 gimplify_and_add (x
, &llist
[0]);
5091 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5094 gimple_seq tseq
= NULL
;
5097 gimplify_stmt (&dtor
, &tseq
);
5098 gimple_seq_add_seq (&llist
[1], tseq
);
5103 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5104 (c
, unshare_expr (new_var
), x
);
5105 gimplify_and_add (x
, ilist
);
5108 case OMP_CLAUSE__LOOPTEMP_
:
5109 gcc_assert (is_taskreg_ctx (ctx
));
5110 x
= build_outer_var_ref (var
, ctx
);
5111 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5112 gimplify_and_add (x
, ilist
);
5115 case OMP_CLAUSE_COPYIN
:
5116 by_ref
= use_pointer_for_field (var
, NULL
);
5117 x
= build_receiver_ref (var
, by_ref
, ctx
);
5118 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5119 append_to_statement_list (x
, ©in_seq
);
5120 copyin_by_ref
|= by_ref
;
5123 case OMP_CLAUSE_REDUCTION
:
5124 /* OpenACC reductions are initialized using the
5125 GOACC_REDUCTION internal function. */
5126 if (is_gimple_omp_oacc (ctx
->stmt
))
5128 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5130 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5132 x
= build_outer_var_ref (var
, ctx
);
5134 if (is_reference (var
)
5135 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5137 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5138 SET_DECL_VALUE_EXPR (placeholder
, x
);
5139 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5140 tree new_vard
= new_var
;
5141 if (is_reference (var
))
5143 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5144 new_vard
= TREE_OPERAND (new_var
, 0);
5145 gcc_assert (DECL_P (new_vard
));
5148 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5149 idx
, lane
, ivar
, lvar
))
5151 if (new_vard
== new_var
)
5153 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5154 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5158 SET_DECL_VALUE_EXPR (new_vard
,
5159 build_fold_addr_expr (ivar
));
5160 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5162 x
= lang_hooks
.decls
.omp_clause_default_ctor
5163 (c
, unshare_expr (ivar
),
5164 build_outer_var_ref (var
, ctx
));
5166 gimplify_and_add (x
, &llist
[0]);
5167 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5169 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5170 lower_omp (&tseq
, ctx
);
5171 gimple_seq_add_seq (&llist
[0], tseq
);
5173 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5174 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5175 lower_omp (&tseq
, ctx
);
5176 gimple_seq_add_seq (&llist
[1], tseq
);
5177 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5178 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5179 if (new_vard
== new_var
)
5180 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5182 SET_DECL_VALUE_EXPR (new_vard
,
5183 build_fold_addr_expr (lvar
));
5184 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5189 gimplify_stmt (&dtor
, &tseq
);
5190 gimple_seq_add_seq (&llist
[1], tseq
);
5194 /* If this is a reference to constant size reduction var
5195 with placeholder, we haven't emitted the initializer
5196 for it because it is undesirable if SIMD arrays are used.
5197 But if they aren't used, we need to emit the deferred
5198 initialization now. */
5199 else if (is_reference (var
) && is_simd
)
5200 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5201 x
= lang_hooks
.decls
.omp_clause_default_ctor
5202 (c
, unshare_expr (new_var
),
5203 build_outer_var_ref (var
, ctx
));
5205 gimplify_and_add (x
, ilist
);
5206 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5208 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5209 lower_omp (&tseq
, ctx
);
5210 gimple_seq_add_seq (ilist
, tseq
);
5212 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5215 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5216 lower_omp (&tseq
, ctx
);
5217 gimple_seq_add_seq (dlist
, tseq
);
5218 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5220 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5225 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5226 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5227 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5229 /* reduction(-:var) sums up the partial results, so it
5230 acts identically to reduction(+:var). */
5231 if (code
== MINUS_EXPR
)
5234 tree new_vard
= new_var
;
5235 if (is_simd
&& is_reference (var
))
5237 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5238 new_vard
= TREE_OPERAND (new_var
, 0);
5239 gcc_assert (DECL_P (new_vard
));
5242 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5243 idx
, lane
, ivar
, lvar
))
5245 tree ref
= build_outer_var_ref (var
, ctx
);
5247 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5249 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5250 ref
= build_outer_var_ref (var
, ctx
);
5251 gimplify_assign (ref
, x
, &llist
[1]);
5253 if (new_vard
!= new_var
)
5255 SET_DECL_VALUE_EXPR (new_vard
,
5256 build_fold_addr_expr (lvar
));
5257 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5262 if (is_reference (var
) && is_simd
)
5263 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5264 gimplify_assign (new_var
, x
, ilist
);
5267 tree ref
= build_outer_var_ref (var
, ctx
);
5269 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5270 ref
= build_outer_var_ref (var
, ctx
);
5271 gimplify_assign (ref
, x
, dlist
);
5285 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5286 /* Don't want uninit warnings on simduid, it is always uninitialized,
5287 but we use it not for the value, but for the DECL_UID only. */
5288 TREE_NO_WARNING (uid
) = 1;
5290 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5291 gimple_call_set_lhs (g
, lane
);
5292 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5293 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5294 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5295 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5296 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5297 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5298 g
= gimple_build_assign (lane
, INTEGER_CST
,
5299 build_int_cst (unsigned_type_node
, 0));
5300 gimple_seq_add_stmt (ilist
, g
);
5301 for (int i
= 0; i
< 2; i
++)
5304 tree vf
= create_tmp_var (unsigned_type_node
);
5305 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5306 gimple_call_set_lhs (g
, vf
);
5307 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5308 gimple_seq_add_stmt (seq
, g
);
5309 tree t
= build_int_cst (unsigned_type_node
, 0);
5310 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5311 gimple_seq_add_stmt (seq
, g
);
5312 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5313 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5314 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5315 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5316 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5317 gimple_seq_add_seq (seq
, llist
[i
]);
5318 t
= build_int_cst (unsigned_type_node
, 1);
5319 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5320 gimple_seq_add_stmt (seq
, g
);
5321 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5322 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5323 gimple_seq_add_stmt (seq
, g
);
5324 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5328 /* The copyin sequence is not to be executed by the main thread, since
5329 that would result in self-copies. Perhaps not visible to scalars,
5330 but it certainly is to C++ operator=. */
5333 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5335 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5336 build_int_cst (TREE_TYPE (x
), 0));
5337 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5338 gimplify_and_add (x
, ilist
);
5341 /* If any copyin variable is passed by reference, we must ensure the
5342 master thread doesn't modify it before it is copied over in all
5343 threads. Similarly for variables in both firstprivate and
5344 lastprivate clauses we need to ensure the lastprivate copying
5345 happens after firstprivate copying in all threads. And similarly
5346 for UDRs if initializer expression refers to omp_orig. */
5347 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5349 /* Don't add any barrier for #pragma omp simd or
5350 #pragma omp distribute. */
5351 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5352 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5353 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5356 /* If max_vf is non-zero, then we can use only a vectorization factor
5357 up to the max_vf we chose. So stick it into the safelen clause. */
5360 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5361 OMP_CLAUSE_SAFELEN
);
5363 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5364 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5367 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5368 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5370 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5371 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5377 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5378 both parallel and workshare constructs. PREDICATE may be NULL if it's
5382 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5385 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5386 bool par_clauses
= false;
5387 tree simduid
= NULL
, lastlane
= NULL
;
5389 /* Early exit if there are no lastprivate or linear clauses. */
5390 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5391 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5392 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5393 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5395 if (clauses
== NULL
)
5397 /* If this was a workshare clause, see if it had been combined
5398 with its parallel. In that case, look for the clauses on the
5399 parallel statement itself. */
5400 if (is_parallel_ctx (ctx
))
5404 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5407 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5408 OMP_CLAUSE_LASTPRIVATE
);
5409 if (clauses
== NULL
)
5417 tree label_true
, arm1
, arm2
;
5419 label
= create_artificial_label (UNKNOWN_LOCATION
);
5420 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5421 arm1
= TREE_OPERAND (predicate
, 0);
5422 arm2
= TREE_OPERAND (predicate
, 1);
5423 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5424 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5425 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5427 gimple_seq_add_stmt (stmt_list
, stmt
);
5428 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5431 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5432 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5434 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5436 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5439 for (c
= clauses
; c
;)
5442 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5444 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5445 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5446 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5448 var
= OMP_CLAUSE_DECL (c
);
5449 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5450 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5451 && is_taskloop_ctx (ctx
))
5453 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5454 new_var
= lookup_decl (var
, ctx
->outer
);
5457 new_var
= lookup_decl (var
, ctx
);
5459 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5461 tree val
= DECL_VALUE_EXPR (new_var
);
5462 if (TREE_CODE (val
) == ARRAY_REF
5463 && VAR_P (TREE_OPERAND (val
, 0))
5464 && lookup_attribute ("omp simd array",
5465 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5468 if (lastlane
== NULL
)
5470 lastlane
= create_tmp_var (unsigned_type_node
);
5472 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5474 TREE_OPERAND (val
, 1));
5475 gimple_call_set_lhs (g
, lastlane
);
5476 gimple_seq_add_stmt (stmt_list
, g
);
5478 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5479 TREE_OPERAND (val
, 0), lastlane
,
5480 NULL_TREE
, NULL_TREE
);
5484 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5485 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5487 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5488 gimple_seq_add_seq (stmt_list
,
5489 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5490 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5492 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5493 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5495 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5496 gimple_seq_add_seq (stmt_list
,
5497 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5498 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5502 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5503 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5505 gcc_checking_assert (is_taskloop_ctx (ctx
));
5506 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5508 if (is_global_var (ovar
))
5512 x
= build_outer_var_ref (var
, ctx
, true);
5513 if (is_reference (var
))
5514 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5515 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5516 gimplify_and_add (x
, stmt_list
);
5518 c
= OMP_CLAUSE_CHAIN (c
);
5519 if (c
== NULL
&& !par_clauses
)
5521 /* If this was a workshare clause, see if it had been combined
5522 with its parallel. In that case, continue looking for the
5523 clauses also on the parallel statement itself. */
5524 if (is_parallel_ctx (ctx
))
5528 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5531 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5532 OMP_CLAUSE_LASTPRIVATE
);
5538 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5541 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5542 (which might be a placeholder). INNER is true if this is an inner
5543 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5544 join markers. Generate the before-loop forking sequence in
5545 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5546 general form of these sequences is
5548 GOACC_REDUCTION_SETUP
5550 GOACC_REDUCTION_INIT
5552 GOACC_REDUCTION_FINI
5554 GOACC_REDUCTION_TEARDOWN. */
5557 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5558 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5559 gimple_seq
*join_seq
, omp_context
*ctx
)
5561 gimple_seq before_fork
= NULL
;
5562 gimple_seq after_fork
= NULL
;
5563 gimple_seq before_join
= NULL
;
5564 gimple_seq after_join
= NULL
;
5565 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5566 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5567 unsigned offset
= 0;
5569 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5570 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5572 tree orig
= OMP_CLAUSE_DECL (c
);
5573 tree var
= maybe_lookup_decl (orig
, ctx
);
5574 tree ref_to_res
= NULL_TREE
;
5575 tree incoming
, outgoing
, v1
, v2
, v3
;
5576 bool is_private
= false;
5578 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5579 if (rcode
== MINUS_EXPR
)
5581 else if (rcode
== TRUTH_ANDIF_EXPR
)
5582 rcode
= BIT_AND_EXPR
;
5583 else if (rcode
== TRUTH_ORIF_EXPR
)
5584 rcode
= BIT_IOR_EXPR
;
5585 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5590 incoming
= outgoing
= var
;
5594 /* See if an outer construct also reduces this variable. */
5595 omp_context
*outer
= ctx
;
5597 while (omp_context
*probe
= outer
->outer
)
5599 enum gimple_code type
= gimple_code (probe
->stmt
);
5604 case GIMPLE_OMP_FOR
:
5605 cls
= gimple_omp_for_clauses (probe
->stmt
);
5608 case GIMPLE_OMP_TARGET
:
5609 if (gimple_omp_target_kind (probe
->stmt
)
5610 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5613 cls
= gimple_omp_target_clauses (probe
->stmt
);
5621 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5622 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5623 && orig
== OMP_CLAUSE_DECL (cls
))
5625 incoming
= outgoing
= lookup_decl (orig
, probe
);
5626 goto has_outer_reduction
;
5628 else if ((OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_FIRSTPRIVATE
5629 || OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_PRIVATE
)
5630 && orig
== OMP_CLAUSE_DECL (cls
))
5638 /* This is the outermost construct with this reduction,
5639 see if there's a mapping for it. */
5640 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5641 && maybe_lookup_field (orig
, outer
) && !is_private
)
5643 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5644 if (is_reference (orig
))
5645 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5647 tree type
= TREE_TYPE (var
);
5648 if (POINTER_TYPE_P (type
))
5649 type
= TREE_TYPE (type
);
5652 incoming
= omp_reduction_init_op (loc
, rcode
, type
);
5654 else if (ctx
->outer
)
5655 incoming
= outgoing
= lookup_decl (orig
, ctx
->outer
);
5657 incoming
= outgoing
= orig
;
5659 has_outer_reduction
:;
5663 ref_to_res
= integer_zero_node
;
5665 if (is_reference (orig
))
5667 tree type
= TREE_TYPE (var
);
5668 const char *id
= IDENTIFIER_POINTER (DECL_NAME (var
));
5672 tree x
= create_tmp_var (TREE_TYPE (type
), id
);
5673 gimplify_assign (var
, build_fold_addr_expr (x
), fork_seq
);
5676 v1
= create_tmp_var (type
, id
);
5677 v2
= create_tmp_var (type
, id
);
5678 v3
= create_tmp_var (type
, id
);
5680 gimplify_assign (v1
, var
, fork_seq
);
5681 gimplify_assign (v2
, var
, fork_seq
);
5682 gimplify_assign (v3
, var
, fork_seq
);
5684 var
= build_simple_mem_ref (var
);
5685 v1
= build_simple_mem_ref (v1
);
5686 v2
= build_simple_mem_ref (v2
);
5687 v3
= build_simple_mem_ref (v3
);
5688 outgoing
= build_simple_mem_ref (outgoing
);
5690 if (!TREE_CONSTANT (incoming
))
5691 incoming
= build_simple_mem_ref (incoming
);
5696 /* Determine position in reduction buffer, which may be used
5698 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5699 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5700 offset
= (offset
+ align
- 1) & ~(align
- 1);
5701 tree off
= build_int_cst (sizetype
, offset
);
5702 offset
+= GET_MODE_SIZE (mode
);
5706 init_code
= build_int_cst (integer_type_node
,
5707 IFN_GOACC_REDUCTION_INIT
);
5708 fini_code
= build_int_cst (integer_type_node
,
5709 IFN_GOACC_REDUCTION_FINI
);
5710 setup_code
= build_int_cst (integer_type_node
,
5711 IFN_GOACC_REDUCTION_SETUP
);
5712 teardown_code
= build_int_cst (integer_type_node
,
5713 IFN_GOACC_REDUCTION_TEARDOWN
);
5717 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5718 TREE_TYPE (var
), 6, setup_code
,
5719 unshare_expr (ref_to_res
),
5720 incoming
, level
, op
, off
);
5722 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5723 TREE_TYPE (var
), 6, init_code
,
5724 unshare_expr (ref_to_res
),
5725 v1
, level
, op
, off
);
5727 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5728 TREE_TYPE (var
), 6, fini_code
,
5729 unshare_expr (ref_to_res
),
5730 v2
, level
, op
, off
);
5732 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5733 TREE_TYPE (var
), 6, teardown_code
,
5734 ref_to_res
, v3
, level
, op
, off
);
5736 gimplify_assign (v1
, setup_call
, &before_fork
);
5737 gimplify_assign (v2
, init_call
, &after_fork
);
5738 gimplify_assign (v3
, fini_call
, &before_join
);
5739 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5742 /* Now stitch things together. */
5743 gimple_seq_add_seq (fork_seq
, before_fork
);
5745 gimple_seq_add_stmt (fork_seq
, fork
);
5746 gimple_seq_add_seq (fork_seq
, after_fork
);
5748 gimple_seq_add_seq (join_seq
, before_join
);
5750 gimple_seq_add_stmt (join_seq
, join
);
5751 gimple_seq_add_seq (join_seq
, after_join
);
5754 /* Generate code to implement the REDUCTION clauses. */
5757 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5759 gimple_seq sub_seq
= NULL
;
5764 /* OpenACC loop reductions are handled elsewhere. */
5765 if (is_gimple_omp_oacc (ctx
->stmt
))
5768 /* SIMD reductions are handled in lower_rec_input_clauses. */
5769 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5770 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5773 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5774 update in that case, otherwise use a lock. */
5775 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5776 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5778 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5779 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5781 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5791 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5793 tree var
, ref
, new_var
, orig_var
;
5794 enum tree_code code
;
5795 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5797 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5800 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5801 if (TREE_CODE (var
) == MEM_REF
)
5803 var
= TREE_OPERAND (var
, 0);
5804 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5805 var
= TREE_OPERAND (var
, 0);
5806 if (TREE_CODE (var
) == INDIRECT_REF
5807 || TREE_CODE (var
) == ADDR_EXPR
)
5808 var
= TREE_OPERAND (var
, 0);
5810 if (is_variable_sized (var
))
5812 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5813 var
= DECL_VALUE_EXPR (var
);
5814 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5815 var
= TREE_OPERAND (var
, 0);
5816 gcc_assert (DECL_P (var
));
5819 new_var
= lookup_decl (var
, ctx
);
5820 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5821 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5822 ref
= build_outer_var_ref (var
, ctx
);
5823 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5825 /* reduction(-:var) sums up the partial results, so it acts
5826 identically to reduction(+:var). */
5827 if (code
== MINUS_EXPR
)
5832 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5834 addr
= save_expr (addr
);
5835 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5836 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5837 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5838 gimplify_and_add (x
, stmt_seqp
);
5841 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5843 tree d
= OMP_CLAUSE_DECL (c
);
5844 tree type
= TREE_TYPE (d
);
5845 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5846 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5847 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5848 tree bias
= TREE_OPERAND (d
, 1);
5849 d
= TREE_OPERAND (d
, 0);
5850 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5852 tree b
= TREE_OPERAND (d
, 1);
5853 b
= maybe_lookup_decl (b
, ctx
);
5856 b
= TREE_OPERAND (d
, 1);
5857 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5859 if (integer_zerop (bias
))
5863 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5864 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5865 TREE_TYPE (b
), b
, bias
);
5867 d
= TREE_OPERAND (d
, 0);
5869 /* For ref build_outer_var_ref already performs this, so
5870 only new_var needs a dereference. */
5871 if (TREE_CODE (d
) == INDIRECT_REF
)
5873 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5874 gcc_assert (is_reference (var
) && var
== orig_var
);
5876 else if (TREE_CODE (d
) == ADDR_EXPR
)
5878 if (orig_var
== var
)
5880 new_var
= build_fold_addr_expr (new_var
);
5881 ref
= build_fold_addr_expr (ref
);
5886 gcc_assert (orig_var
== var
);
5887 if (is_reference (var
))
5888 ref
= build_fold_addr_expr (ref
);
5892 tree t
= maybe_lookup_decl (v
, ctx
);
5896 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5897 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5899 if (!integer_zerop (bias
))
5901 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5902 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5903 TREE_TYPE (new_var
), new_var
,
5904 unshare_expr (bias
));
5905 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5906 TREE_TYPE (ref
), ref
, bias
);
5908 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5909 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5910 tree m
= create_tmp_var (ptype
, NULL
);
5911 gimplify_assign (m
, new_var
, stmt_seqp
);
5913 m
= create_tmp_var (ptype
, NULL
);
5914 gimplify_assign (m
, ref
, stmt_seqp
);
5916 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5917 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5918 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5919 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5920 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5921 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5922 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5924 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5925 tree decl_placeholder
5926 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5927 SET_DECL_VALUE_EXPR (placeholder
, out
);
5928 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5929 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5930 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5931 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5932 gimple_seq_add_seq (&sub_seq
,
5933 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5934 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5935 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5936 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5940 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5941 out
= unshare_expr (out
);
5942 gimplify_assign (out
, x
, &sub_seq
);
5944 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5945 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5946 gimple_seq_add_stmt (&sub_seq
, g
);
5947 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5948 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5949 gimple_seq_add_stmt (&sub_seq
, g
);
5950 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5951 build_int_cst (TREE_TYPE (i
), 1));
5952 gimple_seq_add_stmt (&sub_seq
, g
);
5953 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5954 gimple_seq_add_stmt (&sub_seq
, g
);
5955 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5957 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5959 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5961 if (is_reference (var
)
5962 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5964 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5965 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5966 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5967 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5968 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5969 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5970 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5974 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5975 ref
= build_outer_var_ref (var
, ctx
);
5976 gimplify_assign (ref
, x
, &sub_seq
);
5980 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5982 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5984 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5986 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5988 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5992 /* Generate code to implement the COPYPRIVATE clauses. */
5995 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
6000 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6002 tree var
, new_var
, ref
, x
;
6004 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6006 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
6009 var
= OMP_CLAUSE_DECL (c
);
6010 by_ref
= use_pointer_for_field (var
, NULL
);
6012 ref
= build_sender_ref (var
, ctx
);
6013 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
6016 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
6017 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
6019 gimplify_assign (ref
, x
, slist
);
6021 ref
= build_receiver_ref (var
, false, ctx
);
6024 ref
= fold_convert_loc (clause_loc
,
6025 build_pointer_type (TREE_TYPE (new_var
)),
6027 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
6029 if (is_reference (var
))
6031 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
6032 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
6033 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
6035 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
6036 gimplify_and_add (x
, rlist
);
6041 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
6042 and REDUCTION from the sender (aka parent) side. */
6045 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
6049 int ignored_looptemp
= 0;
6050 bool is_taskloop
= false;
6052 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
6053 by GOMP_taskloop. */
6054 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
6056 ignored_looptemp
= 2;
6060 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6062 tree val
, ref
, x
, var
;
6063 bool by_ref
, do_in
= false, do_out
= false;
6064 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6066 switch (OMP_CLAUSE_CODE (c
))
6068 case OMP_CLAUSE_PRIVATE
:
6069 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
6072 case OMP_CLAUSE_FIRSTPRIVATE
:
6073 case OMP_CLAUSE_COPYIN
:
6074 case OMP_CLAUSE_LASTPRIVATE
:
6075 case OMP_CLAUSE_REDUCTION
:
6077 case OMP_CLAUSE_SHARED
:
6078 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6081 case OMP_CLAUSE__LOOPTEMP_
:
6082 if (ignored_looptemp
)
6092 val
= OMP_CLAUSE_DECL (c
);
6093 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6094 && TREE_CODE (val
) == MEM_REF
)
6096 val
= TREE_OPERAND (val
, 0);
6097 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
6098 val
= TREE_OPERAND (val
, 0);
6099 if (TREE_CODE (val
) == INDIRECT_REF
6100 || TREE_CODE (val
) == ADDR_EXPR
)
6101 val
= TREE_OPERAND (val
, 0);
6102 if (is_variable_sized (val
))
6106 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6107 outer taskloop region. */
6108 omp_context
*ctx_for_o
= ctx
;
6110 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
6111 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6112 ctx_for_o
= ctx
->outer
;
6114 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
6116 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
6117 && is_global_var (var
))
6120 t
= omp_member_access_dummy_var (var
);
6123 var
= DECL_VALUE_EXPR (var
);
6124 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
6126 var
= unshare_and_remap (var
, t
, o
);
6128 var
= unshare_expr (var
);
6131 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
6133 /* Handle taskloop firstprivate/lastprivate, where the
6134 lastprivate on GIMPLE_OMP_TASK is represented as
6135 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6136 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
6137 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
6138 if (use_pointer_for_field (val
, ctx
))
6139 var
= build_fold_addr_expr (var
);
6140 gimplify_assign (x
, var
, ilist
);
6141 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
6145 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6146 || val
== OMP_CLAUSE_DECL (c
))
6147 && is_variable_sized (val
))
6149 by_ref
= use_pointer_for_field (val
, NULL
);
6151 switch (OMP_CLAUSE_CODE (c
))
6153 case OMP_CLAUSE_FIRSTPRIVATE
:
6154 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
)
6156 && is_task_ctx (ctx
))
6157 TREE_NO_WARNING (var
) = 1;
6161 case OMP_CLAUSE_PRIVATE
:
6162 case OMP_CLAUSE_COPYIN
:
6163 case OMP_CLAUSE__LOOPTEMP_
:
6167 case OMP_CLAUSE_LASTPRIVATE
:
6168 if (by_ref
|| is_reference (val
))
6170 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
6177 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
6182 case OMP_CLAUSE_REDUCTION
:
6184 if (val
== OMP_CLAUSE_DECL (c
))
6185 do_out
= !(by_ref
|| is_reference (val
));
6187 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
6196 ref
= build_sender_ref (val
, ctx
);
6197 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
6198 gimplify_assign (ref
, x
, ilist
);
6199 if (is_task_ctx (ctx
))
6200 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
6205 ref
= build_sender_ref (val
, ctx
);
6206 gimplify_assign (var
, ref
, olist
);
6211 /* Generate code to implement SHARED from the sender (aka parent)
6212 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6213 list things that got automatically shared. */
6216 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6218 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6220 if (ctx
->record_type
== NULL
)
6223 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6224 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6226 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6227 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6230 nvar
= maybe_lookup_decl (ovar
, ctx
);
6231 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6234 /* If CTX is a nested parallel directive. Find the immediately
6235 enclosing parallel or workshare construct that contains a
6236 mapping for OVAR. */
6237 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6239 t
= omp_member_access_dummy_var (var
);
6242 var
= DECL_VALUE_EXPR (var
);
6243 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6245 var
= unshare_and_remap (var
, t
, o
);
6247 var
= unshare_expr (var
);
6250 if (use_pointer_for_field (ovar
, ctx
))
6252 x
= build_sender_ref (ovar
, ctx
);
6253 var
= build_fold_addr_expr (var
);
6254 gimplify_assign (x
, var
, ilist
);
6258 x
= build_sender_ref (ovar
, ctx
);
6259 gimplify_assign (x
, var
, ilist
);
6261 if (!TREE_READONLY (var
)
6262 /* We don't need to receive a new reference to a result
6263 or parm decl. In fact we may not store to it as we will
6264 invalidate any pending RSO and generate wrong gimple
6266 && !((TREE_CODE (var
) == RESULT_DECL
6267 || TREE_CODE (var
) == PARM_DECL
)
6268 && DECL_BY_REFERENCE (var
)))
6270 x
= build_sender_ref (ovar
, ctx
);
6271 gimplify_assign (var
, x
, olist
);
6277 /* Emit an OpenACC head marker call, encapulating the partitioning and
6278 other information that must be processed by the target compiler.
6279 Return the maximum number of dimensions the associated loop might
6280 be partitioned over. */
6283 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6284 gimple_seq
*seq
, omp_context
*ctx
)
6286 unsigned levels
= 0;
6288 tree gang_static
= NULL_TREE
;
6289 auto_vec
<tree
, 5> args
;
6291 args
.quick_push (build_int_cst
6292 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6293 args
.quick_push (ddvar
);
6294 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6296 switch (OMP_CLAUSE_CODE (c
))
6298 case OMP_CLAUSE_GANG
:
6299 tag
|= OLF_DIM_GANG
;
6300 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6301 /* static:* is represented by -1, and we can ignore it, as
6302 scheduling is always static. */
6303 if (gang_static
&& integer_minus_onep (gang_static
))
6304 gang_static
= NULL_TREE
;
6308 case OMP_CLAUSE_WORKER
:
6309 tag
|= OLF_DIM_WORKER
;
6313 case OMP_CLAUSE_VECTOR
:
6314 tag
|= OLF_DIM_VECTOR
;
6318 case OMP_CLAUSE_SEQ
:
6322 case OMP_CLAUSE_AUTO
:
6326 case OMP_CLAUSE_INDEPENDENT
:
6327 tag
|= OLF_INDEPENDENT
;
6337 if (DECL_P (gang_static
))
6338 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6339 tag
|= OLF_GANG_STATIC
;
6342 /* In a parallel region, loops are implicitly INDEPENDENT. */
6343 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6344 if (!tgt
|| is_oacc_parallel (tgt
))
6345 tag
|= OLF_INDEPENDENT
;
6347 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6348 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6352 /* Ensure at least one level. */
6356 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6357 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6359 args
.quick_push (gang_static
);
6361 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6362 gimple_set_location (call
, loc
);
6363 gimple_set_lhs (call
, ddvar
);
6364 gimple_seq_add_stmt (seq
, call
);
6369 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6370 partitioning level of the enclosed region. */
6373 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6374 tree tofollow
, gimple_seq
*seq
)
6376 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6377 : IFN_UNIQUE_OACC_TAIL_MARK
);
6378 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6379 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6380 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6381 marker
, ddvar
, tofollow
);
6382 gimple_set_location (call
, loc
);
6383 gimple_set_lhs (call
, ddvar
);
6384 gimple_seq_add_stmt (seq
, call
);
6387 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6388 the loop clauses, from which we extract reductions. Initialize
6392 lower_oacc_head_tail (location_t loc
, tree clauses
,
6393 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6396 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6397 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6399 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6401 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6403 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6404 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6406 for (unsigned done
= 1; count
; count
--, done
++)
6408 gimple_seq fork_seq
= NULL
;
6409 gimple_seq join_seq
= NULL
;
6411 tree place
= build_int_cst (integer_type_node
, -1);
6412 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6413 fork_kind
, ddvar
, place
);
6414 gimple_set_location (fork
, loc
);
6415 gimple_set_lhs (fork
, ddvar
);
6417 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6418 join_kind
, ddvar
, place
);
6419 gimple_set_location (join
, loc
);
6420 gimple_set_lhs (join
, ddvar
);
6422 /* Mark the beginning of this level sequence. */
6424 lower_oacc_loop_marker (loc
, ddvar
, true,
6425 build_int_cst (integer_type_node
, count
),
6427 lower_oacc_loop_marker (loc
, ddvar
, false,
6428 build_int_cst (integer_type_node
, done
),
6431 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6432 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6434 /* Append this level to head. */
6435 gimple_seq_add_seq (head
, fork_seq
);
6436 /* Prepend it to tail. */
6437 gimple_seq_add_seq (&join_seq
, *tail
);
6443 /* Mark the end of the sequence. */
6444 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6445 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6448 /* A convenience function to build an empty GIMPLE_COND with just the
6452 gimple_build_cond_empty (tree cond
)
6454 enum tree_code pred_code
;
6457 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6458 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6461 /* Return true if a parallel REGION is within a declare target function or
6462 within a target region and is not a part of a gridified target. */
6465 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
6467 bool indirect
= false;
6468 for (region
= region
->outer
; region
; region
= region
->outer
)
6470 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6472 else if (region
->type
== GIMPLE_OMP_TARGET
)
6474 gomp_target
*tgt_stmt
6475 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
6477 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
6478 OMP_CLAUSE__GRIDDIM_
))
6485 if (lookup_attribute ("omp declare target",
6486 DECL_ATTRIBUTES (current_function_decl
)))
6492 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6495 /* Build the function calls to GOMP_parallel_start etc to actually
6496 generate the parallel operation. REGION is the parallel region
6497 being expanded. BB is the block where to insert the code. WS_ARGS
6498 will be set if this is a call to a combined parallel+workshare
6499 construct, it contains the list of additional arguments needed by
6500 the workshare construct. */
6503 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6504 gomp_parallel
*entry_stmt
,
6505 vec
<tree
, va_gc
> *ws_args
)
6507 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6508 gimple_stmt_iterator gsi
;
6510 enum built_in_function start_ix
;
6512 location_t clause_loc
;
6513 vec
<tree
, va_gc
> *args
;
6515 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6517 /* Determine what flavor of GOMP_parallel we will be
6519 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6520 if (is_combined_parallel (region
))
6522 switch (region
->inner
->type
)
6524 case GIMPLE_OMP_FOR
:
6525 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6526 switch (region
->inner
->sched_kind
)
6528 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6531 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6532 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6533 if (region
->inner
->sched_modifiers
6534 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6536 start_ix2
= 3 + region
->inner
->sched_kind
;
6541 start_ix2
= region
->inner
->sched_kind
;
6544 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6545 start_ix
= (enum built_in_function
) start_ix2
;
6547 case GIMPLE_OMP_SECTIONS
:
6548 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6555 /* By default, the value of NUM_THREADS is zero (selected at run time)
6556 and there is no conditional. */
6558 val
= build_int_cst (unsigned_type_node
, 0);
6559 flags
= build_int_cst (unsigned_type_node
, 0);
6561 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6563 cond
= OMP_CLAUSE_IF_EXPR (c
);
6565 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6568 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6569 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6572 clause_loc
= gimple_location (entry_stmt
);
6574 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6576 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6578 /* Ensure 'val' is of the correct type. */
6579 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6581 /* If we found the clause 'if (cond)', build either
6582 (cond != 0) or (cond ? val : 1u). */
6585 cond
= gimple_boolify (cond
);
6587 if (integer_zerop (val
))
6588 val
= fold_build2_loc (clause_loc
,
6589 EQ_EXPR
, unsigned_type_node
, cond
,
6590 build_int_cst (TREE_TYPE (cond
), 0));
6593 basic_block cond_bb
, then_bb
, else_bb
;
6594 edge e
, e_then
, e_else
;
6595 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6597 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6598 if (gimple_in_ssa_p (cfun
))
6600 tmp_then
= make_ssa_name (tmp_var
);
6601 tmp_else
= make_ssa_name (tmp_var
);
6602 tmp_join
= make_ssa_name (tmp_var
);
6611 e
= split_block_after_labels (bb
);
6616 then_bb
= create_empty_bb (cond_bb
);
6617 else_bb
= create_empty_bb (then_bb
);
6618 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6619 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6621 stmt
= gimple_build_cond_empty (cond
);
6622 gsi
= gsi_start_bb (cond_bb
);
6623 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6625 gsi
= gsi_start_bb (then_bb
);
6626 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6628 gsi
= gsi_start_bb (else_bb
);
6629 expand_omp_build_assign (&gsi
, tmp_else
,
6630 build_int_cst (unsigned_type_node
, 1),
6633 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6634 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6635 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6636 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6637 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6638 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6640 if (gimple_in_ssa_p (cfun
))
6642 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6643 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6644 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6650 gsi
= gsi_start_bb (bb
);
6651 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6652 false, GSI_CONTINUE_LINKING
);
6655 gsi
= gsi_last_bb (bb
);
6656 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6658 t1
= null_pointer_node
;
6660 t1
= build_fold_addr_expr (t
);
6661 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
6662 t2
= build_fold_addr_expr (child_fndecl
);
6664 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6665 args
->quick_push (t2
);
6666 args
->quick_push (t1
);
6667 args
->quick_push (val
);
6669 args
->splice (*ws_args
);
6670 args
->quick_push (flags
);
6672 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6673 builtin_decl_explicit (start_ix
), args
);
6675 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6676 false, GSI_CONTINUE_LINKING
);
6678 if (hsa_gen_requested_p ()
6679 && parallel_needs_hsa_kernel_p (region
))
6681 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
6682 hsa_register_kernel (child_cnode
);
6686 /* Insert a function call whose name is FUNC_NAME with the information from
6687 ENTRY_STMT into the basic_block BB. */
6690 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6691 vec
<tree
, va_gc
> *ws_args
)
6694 gimple_stmt_iterator gsi
;
6695 vec
<tree
, va_gc
> *args
;
6697 gcc_assert (vec_safe_length (ws_args
) == 2);
6698 tree func_name
= (*ws_args
)[0];
6699 tree grain
= (*ws_args
)[1];
6701 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6702 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6703 gcc_assert (count
!= NULL_TREE
);
6704 count
= OMP_CLAUSE_OPERAND (count
, 0);
6706 gsi
= gsi_last_bb (bb
);
6707 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6709 t1
= null_pointer_node
;
6711 t1
= build_fold_addr_expr (t
);
6712 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6714 vec_alloc (args
, 4);
6715 args
->quick_push (t2
);
6716 args
->quick_push (t1
);
6717 args
->quick_push (count
);
6718 args
->quick_push (grain
);
6719 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6721 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6722 GSI_CONTINUE_LINKING
);
6725 /* Build the function call to GOMP_task to actually
6726 generate the task operation. BB is the block where to insert the code. */
6729 expand_task_call (struct omp_region
*region
, basic_block bb
,
6730 gomp_task
*entry_stmt
)
6733 gimple_stmt_iterator gsi
;
6734 location_t loc
= gimple_location (entry_stmt
);
6736 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6738 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6739 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6740 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6741 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6742 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6743 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6746 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6747 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6748 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6750 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6751 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6752 tree num_tasks
= NULL_TREE
;
6756 gimple
*g
= last_stmt (region
->outer
->entry
);
6757 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6758 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6759 struct omp_for_data fd
;
6760 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6761 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6762 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6763 OMP_CLAUSE__LOOPTEMP_
);
6764 startvar
= OMP_CLAUSE_DECL (startvar
);
6765 endvar
= OMP_CLAUSE_DECL (endvar
);
6766 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6767 if (fd
.loop
.cond_code
== LT_EXPR
)
6768 iflags
|= GOMP_TASK_FLAG_UP
;
6769 tree tclauses
= gimple_omp_for_clauses (g
);
6770 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6772 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6775 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6778 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6779 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6782 num_tasks
= integer_zero_node
;
6784 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6785 if (ifc
== NULL_TREE
)
6786 iflags
|= GOMP_TASK_FLAG_IF
;
6787 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6788 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6789 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6792 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6794 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6796 tree cond
= boolean_true_node
;
6801 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6802 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6803 build_int_cst (unsigned_type_node
,
6805 build_int_cst (unsigned_type_node
, 0));
6806 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6810 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6815 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6816 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6817 build_int_cst (unsigned_type_node
,
6818 GOMP_TASK_FLAG_FINAL
),
6819 build_int_cst (unsigned_type_node
, 0));
6820 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6823 depend
= OMP_CLAUSE_DECL (depend
);
6825 depend
= build_int_cst (ptr_type_node
, 0);
6827 priority
= fold_convert (integer_type_node
,
6828 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6830 priority
= integer_zero_node
;
6832 gsi
= gsi_last_bb (bb
);
6833 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6835 t2
= null_pointer_node
;
6837 t2
= build_fold_addr_expr_loc (loc
, t
);
6838 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6839 t
= gimple_omp_task_copy_fn (entry_stmt
);
6841 t3
= null_pointer_node
;
6843 t3
= build_fold_addr_expr_loc (loc
, t
);
6846 t
= build_call_expr (ull
6847 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6848 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6850 gimple_omp_task_arg_size (entry_stmt
),
6851 gimple_omp_task_arg_align (entry_stmt
), flags
,
6852 num_tasks
, priority
, startvar
, endvar
, step
);
6854 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6856 gimple_omp_task_arg_size (entry_stmt
),
6857 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6860 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6861 false, GSI_CONTINUE_LINKING
);
6865 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6866 catch handler and return it. This prevents programs from violating the
6867 structured block semantics with throws. */
6870 maybe_catch_exception (gimple_seq body
)
6875 if (!flag_exceptions
)
6878 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6879 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6881 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6883 g
= gimple_build_eh_must_not_throw (decl
);
6884 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6887 return gimple_seq_alloc_with_stmt (g
);
6890 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6893 vec2chain (vec
<tree
, va_gc
> *v
)
6895 tree chain
= NULL_TREE
, t
;
6898 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6900 DECL_CHAIN (t
) = chain
;
6908 /* Remove barriers in REGION->EXIT's block. Note that this is only
6909 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6910 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6911 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6915 remove_exit_barrier (struct omp_region
*region
)
6917 gimple_stmt_iterator gsi
;
6918 basic_block exit_bb
;
6922 int any_addressable_vars
= -1;
6924 exit_bb
= region
->exit
;
6926 /* If the parallel region doesn't return, we don't have REGION->EXIT
6931 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6932 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6933 statements that can appear in between are extremely limited -- no
6934 memory operations at all. Here, we allow nothing at all, so the
6935 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6936 gsi
= gsi_last_bb (exit_bb
);
6937 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6939 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6942 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6944 gsi
= gsi_last_bb (e
->src
);
6945 if (gsi_end_p (gsi
))
6947 stmt
= gsi_stmt (gsi
);
6948 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6949 && !gimple_omp_return_nowait_p (stmt
))
6951 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6952 in many cases. If there could be tasks queued, the barrier
6953 might be needed to let the tasks run before some local
6954 variable of the parallel that the task uses as shared
6955 runs out of scope. The task can be spawned either
6956 from within current function (this would be easy to check)
6957 or from some function it calls and gets passed an address
6958 of such a variable. */
6959 if (any_addressable_vars
< 0)
6961 gomp_parallel
*parallel_stmt
6962 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6963 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6964 tree local_decls
, block
, decl
;
6967 any_addressable_vars
= 0;
6968 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6969 if (TREE_ADDRESSABLE (decl
))
6971 any_addressable_vars
= 1;
6974 for (block
= gimple_block (stmt
);
6975 !any_addressable_vars
6977 && TREE_CODE (block
) == BLOCK
;
6978 block
= BLOCK_SUPERCONTEXT (block
))
6980 for (local_decls
= BLOCK_VARS (block
);
6982 local_decls
= DECL_CHAIN (local_decls
))
6983 if (TREE_ADDRESSABLE (local_decls
))
6985 any_addressable_vars
= 1;
6988 if (block
== gimple_block (parallel_stmt
))
6992 if (!any_addressable_vars
)
6993 gimple_omp_return_set_nowait (stmt
);
6999 remove_exit_barriers (struct omp_region
*region
)
7001 if (region
->type
== GIMPLE_OMP_PARALLEL
)
7002 remove_exit_barrier (region
);
7006 region
= region
->inner
;
7007 remove_exit_barriers (region
);
7008 while (region
->next
)
7010 region
= region
->next
;
7011 remove_exit_barriers (region
);
7016 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
7017 calls. These can't be declared as const functions, but
7018 within one parallel body they are constant, so they can be
7019 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
7020 which are declared const. Similarly for task body, except
7021 that in untied task omp_get_thread_num () can change at any task
7022 scheduling point. */
7025 optimize_omp_library_calls (gimple
*entry_stmt
)
7028 gimple_stmt_iterator gsi
;
7029 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7030 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
7031 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7032 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
7033 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
7034 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
7035 OMP_CLAUSE_UNTIED
) != NULL
);
7037 FOR_EACH_BB_FN (bb
, cfun
)
7038 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
7040 gimple
*call
= gsi_stmt (gsi
);
7043 if (is_gimple_call (call
)
7044 && (decl
= gimple_call_fndecl (call
))
7045 && DECL_EXTERNAL (decl
)
7046 && TREE_PUBLIC (decl
)
7047 && DECL_INITIAL (decl
) == NULL
)
7051 if (DECL_NAME (decl
) == thr_num_id
)
7053 /* In #pragma omp task untied omp_get_thread_num () can change
7054 during the execution of the task region. */
7057 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7059 else if (DECL_NAME (decl
) == num_thr_id
)
7060 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7064 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
7065 || gimple_call_num_args (call
) != 0)
7068 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
7071 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
7072 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
7073 TREE_TYPE (TREE_TYPE (built_in
))))
7076 gimple_call_set_fndecl (call
, built_in
);
7081 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7085 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
7089 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7090 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
7093 if (TREE_CODE (t
) == ADDR_EXPR
)
7094 recompute_tree_invariant_for_addr_expr (t
);
7096 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
7100 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7103 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
7106 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
7107 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
7108 !after
, after
? GSI_CONTINUE_LINKING
7110 gimple
*stmt
= gimple_build_assign (to
, from
);
7112 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
7114 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
7115 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
7116 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
7118 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
7119 gimple_regimplify_operands (stmt
, &gsi
);
7123 /* Expand the OpenMP parallel or task directive starting at REGION. */
7126 expand_omp_taskreg (struct omp_region
*region
)
7128 basic_block entry_bb
, exit_bb
, new_bb
;
7129 struct function
*child_cfun
;
7130 tree child_fn
, block
, t
;
7131 gimple_stmt_iterator gsi
;
7132 gimple
*entry_stmt
, *stmt
;
7134 vec
<tree
, va_gc
> *ws_args
;
7136 entry_stmt
= last_stmt (region
->entry
);
7137 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
7138 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7140 entry_bb
= region
->entry
;
7141 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
7142 exit_bb
= region
->cont
;
7144 exit_bb
= region
->exit
;
7148 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
7149 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
7150 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
7153 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7154 and the inner statement contains the name of the built-in function
7156 ws_args
= region
->inner
->ws_args
;
7157 else if (is_combined_parallel (region
))
7158 ws_args
= region
->ws_args
;
7162 if (child_cfun
->cfg
)
7164 /* Due to inlining, it may happen that we have already outlined
7165 the region, in which case all we need to do is make the
7166 sub-graph unreachable and emit the parallel call. */
7167 edge entry_succ_e
, exit_succ_e
;
7169 entry_succ_e
= single_succ_edge (entry_bb
);
7171 gsi
= gsi_last_bb (entry_bb
);
7172 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
7173 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
7174 gsi_remove (&gsi
, true);
7179 exit_succ_e
= single_succ_edge (exit_bb
);
7180 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
7182 remove_edge_and_dominated_blocks (entry_succ_e
);
7186 unsigned srcidx
, dstidx
, num
;
7188 /* If the parallel region needs data sent from the parent
7189 function, then the very first statement (except possible
7190 tree profile counter updates) of the parallel body
7191 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7192 &.OMP_DATA_O is passed as an argument to the child function,
7193 we need to replace it with the argument as seen by the child
7196 In most cases, this will end up being the identity assignment
7197 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7198 a function call that has been inlined, the original PARM_DECL
7199 .OMP_DATA_I may have been converted into a different local
7200 variable. In which case, we need to keep the assignment. */
7201 if (gimple_omp_taskreg_data_arg (entry_stmt
))
7203 basic_block entry_succ_bb
7204 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
7205 : FALLTHRU_EDGE (entry_bb
)->dest
;
7207 gimple
*parcopy_stmt
= NULL
;
7209 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7213 gcc_assert (!gsi_end_p (gsi
));
7214 stmt
= gsi_stmt (gsi
);
7215 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7218 if (gimple_num_ops (stmt
) == 2)
7220 tree arg
= gimple_assign_rhs1 (stmt
);
7222 /* We're ignore the subcode because we're
7223 effectively doing a STRIP_NOPS. */
7225 if (TREE_CODE (arg
) == ADDR_EXPR
7226 && TREE_OPERAND (arg
, 0)
7227 == gimple_omp_taskreg_data_arg (entry_stmt
))
7229 parcopy_stmt
= stmt
;
7235 gcc_assert (parcopy_stmt
!= NULL
);
7236 arg
= DECL_ARGUMENTS (child_fn
);
7238 if (!gimple_in_ssa_p (cfun
))
7240 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
7241 gsi_remove (&gsi
, true);
7244 /* ?? Is setting the subcode really necessary ?? */
7245 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
7246 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7251 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
7252 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
7253 /* We'd like to set the rhs to the default def in the child_fn,
7254 but it's too early to create ssa names in the child_fn.
7255 Instead, we set the rhs to the parm. In
7256 move_sese_region_to_fn, we introduce a default def for the
7257 parm, map the parm to it's default def, and once we encounter
7258 this stmt, replace the parm with the default def. */
7259 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7260 update_stmt (parcopy_stmt
);
7264 /* Declare local variables needed in CHILD_CFUN. */
7265 block
= DECL_INITIAL (child_fn
);
7266 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7267 /* The gimplifier could record temporaries in parallel/task block
7268 rather than in containing function's local_decls chain,
7269 which would mean cgraph missed finalizing them. Do it now. */
7270 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7271 if (TREE_CODE (t
) == VAR_DECL
7273 && !DECL_EXTERNAL (t
))
7274 varpool_node::finalize_decl (t
);
7275 DECL_SAVED_TREE (child_fn
) = NULL
;
7276 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7277 gimple_set_body (child_fn
, NULL
);
7278 TREE_USED (block
) = 1;
7280 /* Reset DECL_CONTEXT on function arguments. */
7281 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7282 DECL_CONTEXT (t
) = child_fn
;
7284 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7285 so that it can be moved to the child function. */
7286 gsi
= gsi_last_bb (entry_bb
);
7287 stmt
= gsi_stmt (gsi
);
7288 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7289 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7290 e
= split_block (entry_bb
, stmt
);
7291 gsi_remove (&gsi
, true);
7294 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7295 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7298 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7299 gcc_assert (e2
->dest
== region
->exit
);
7300 remove_edge (BRANCH_EDGE (entry_bb
));
7301 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7302 gsi
= gsi_last_bb (region
->exit
);
7303 gcc_assert (!gsi_end_p (gsi
)
7304 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7305 gsi_remove (&gsi
, true);
7308 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7311 gsi
= gsi_last_bb (exit_bb
);
7312 gcc_assert (!gsi_end_p (gsi
)
7313 && (gimple_code (gsi_stmt (gsi
))
7314 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7315 stmt
= gimple_build_return (NULL
);
7316 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7317 gsi_remove (&gsi
, true);
7320 /* Move the parallel region into CHILD_CFUN. */
7322 if (gimple_in_ssa_p (cfun
))
7324 init_tree_ssa (child_cfun
);
7325 init_ssa_operands (child_cfun
);
7326 child_cfun
->gimple_df
->in_ssa_p
= true;
7330 block
= gimple_block (entry_stmt
);
7332 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7334 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7337 basic_block dest_bb
= e2
->dest
;
7339 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7341 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7343 /* When the OMP expansion process cannot guarantee an up-to-date
7344 loop tree arrange for the child function to fixup loops. */
7345 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7346 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7348 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7349 num
= vec_safe_length (child_cfun
->local_decls
);
7350 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7352 t
= (*child_cfun
->local_decls
)[srcidx
];
7353 if (DECL_CONTEXT (t
) == cfun
->decl
)
7355 if (srcidx
!= dstidx
)
7356 (*child_cfun
->local_decls
)[dstidx
] = t
;
7360 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7362 /* Inform the callgraph about the new function. */
7363 child_cfun
->curr_properties
= cfun
->curr_properties
;
7364 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7365 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7366 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7367 node
->parallelized_function
= 1;
7368 cgraph_node::add_new_function (child_fn
, true);
7370 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7371 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7373 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7374 fixed in a following pass. */
7375 push_cfun (child_cfun
);
7377 assign_assembler_name_if_neeeded (child_fn
);
7380 optimize_omp_library_calls (entry_stmt
);
7381 cgraph_edge::rebuild_edges ();
7383 /* Some EH regions might become dead, see PR34608. If
7384 pass_cleanup_cfg isn't the first pass to happen with the
7385 new child, these dead EH edges might cause problems.
7386 Clean them up now. */
7387 if (flag_exceptions
)
7390 bool changed
= false;
7392 FOR_EACH_BB_FN (bb
, cfun
)
7393 changed
|= gimple_purge_dead_eh_edges (bb
);
7395 cleanup_tree_cfg ();
7397 if (gimple_in_ssa_p (cfun
))
7398 update_ssa (TODO_update_ssa
);
7399 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7400 verify_loop_structure ();
7403 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7405 omp_any_child_fn_dumped
= true;
7406 dump_function_header (dump_file
, child_fn
, dump_flags
);
7407 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7411 /* Emit a library call to launch the children threads. */
7413 expand_cilk_for_call (new_bb
,
7414 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7415 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7416 expand_parallel_call (region
, new_bb
,
7417 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7419 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7420 if (gimple_in_ssa_p (cfun
))
7421 update_ssa (TODO_update_ssa_only_virtuals
);
7424 /* Information about members of an OpenACC collapsed loop nest. */
7426 struct oacc_collapse
7428 tree base
; /* Base value. */
7429 tree iters
; /* Number of steps. */
7430 tree step
; /* step size. */
7433 /* Helper for expand_oacc_for. Determine collapsed loop information.
7434 Fill in COUNTS array. Emit any initialization code before GSI.
7435 Return the calculated outer loop bound of BOUND_TYPE. */
7438 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7439 gimple_stmt_iterator
*gsi
,
7440 oacc_collapse
*counts
, tree bound_type
)
7442 tree total
= build_int_cst (bound_type
, 1);
7445 gcc_assert (integer_onep (fd
->loop
.step
));
7446 gcc_assert (integer_zerop (fd
->loop
.n1
));
7448 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7450 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7452 tree iter_type
= TREE_TYPE (loop
->v
);
7453 tree diff_type
= iter_type
;
7454 tree plus_type
= iter_type
;
7456 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7458 if (POINTER_TYPE_P (iter_type
))
7459 plus_type
= sizetype
;
7460 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7461 diff_type
= signed_type_for (diff_type
);
7465 tree s
= loop
->step
;
7466 bool up
= loop
->cond_code
== LT_EXPR
;
7467 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7471 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7472 true, GSI_SAME_STMT
);
7473 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7474 true, GSI_SAME_STMT
);
7476 /* Convert the step, avoiding possible unsigned->signed overflow. */
7477 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7479 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7480 s
= fold_convert (diff_type
, s
);
7482 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7483 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7484 true, GSI_SAME_STMT
);
7486 /* Determine the range, avoiding possible unsigned->signed overflow. */
7487 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7488 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7489 fold_convert (plus_type
, negating
? b
: e
),
7490 fold_convert (plus_type
, negating
? e
: b
));
7491 expr
= fold_convert (diff_type
, expr
);
7493 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7494 tree range
= force_gimple_operand_gsi
7495 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7497 /* Determine number of iterations. */
7498 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7499 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7500 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7502 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7503 true, GSI_SAME_STMT
);
7505 counts
[ix
].base
= b
;
7506 counts
[ix
].iters
= iters
;
7507 counts
[ix
].step
= s
;
7509 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7510 fold_convert (bound_type
, iters
));
7516 /* Emit initializers for collapsed loop members. IVAR is the outer
7517 loop iteration variable, from which collapsed loop iteration values
7518 are calculated. COUNTS array has been initialized by
7519 expand_oacc_collapse_inits. */
7522 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7523 gimple_stmt_iterator
*gsi
,
7524 const oacc_collapse
*counts
, tree ivar
)
7526 tree ivar_type
= TREE_TYPE (ivar
);
7528 /* The most rapidly changing iteration variable is the innermost
7530 for (int ix
= fd
->collapse
; ix
--;)
7532 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7533 const oacc_collapse
*collapse
= &counts
[ix
];
7534 tree iter_type
= TREE_TYPE (loop
->v
);
7535 tree diff_type
= TREE_TYPE (collapse
->step
);
7536 tree plus_type
= iter_type
;
7537 enum tree_code plus_code
= PLUS_EXPR
;
7540 if (POINTER_TYPE_P (iter_type
))
7542 plus_code
= POINTER_PLUS_EXPR
;
7543 plus_type
= sizetype
;
7546 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7547 fold_convert (ivar_type
, collapse
->iters
));
7548 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7550 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7551 fold_convert (plus_type
, expr
));
7552 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7553 true, GSI_SAME_STMT
);
7554 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7555 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7559 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7560 fold_convert (ivar_type
, collapse
->iters
));
7561 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7562 true, GSI_SAME_STMT
);
7568 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7569 of the combined collapse > 1 loop constructs, generate code like:
7570 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7575 count3 = (adj + N32 - N31) / STEP3;
7576 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7581 count2 = (adj + N22 - N21) / STEP2;
7582 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7587 count1 = (adj + N12 - N11) / STEP1;
7588 count = count1 * count2 * count3;
7589 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7591 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7592 of the combined loop constructs, just initialize COUNTS array
7593 from the _looptemp_ clauses. */
7595 /* NOTE: It *could* be better to moosh all of the BBs together,
7596 creating one larger BB with all the computation and the unexpected
7597 jump at the end. I.e.
7599 bool zero3, zero2, zero1, zero;
7602 count3 = (N32 - N31) /[cl] STEP3;
7604 count2 = (N22 - N21) /[cl] STEP2;
7606 count1 = (N12 - N11) /[cl] STEP1;
7607 zero = zero3 || zero2 || zero1;
7608 count = count1 * count2 * count3;
7609 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7611 After all, we expect the zero=false, and thus we expect to have to
7612 evaluate all of the comparison expressions, so short-circuiting
7613 oughtn't be a win. Since the condition isn't protecting a
7614 denominator, we're not concerned about divide-by-zero, so we can
7615 fully evaluate count even if a numerator turned out to be wrong.
7617 It seems like putting this all together would create much better
7618 scheduling opportunities, and less pressure on the chip's branch
7622 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7623 basic_block
&entry_bb
, tree
*counts
,
7624 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7625 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7626 basic_block
&l2_dom_bb
)
7628 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7632 /* Collapsed loops need work for expansion into SSA form. */
7633 gcc_assert (!gimple_in_ssa_p (cfun
));
7635 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7636 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7638 gcc_assert (fd
->ordered
== 0);
7639 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7640 isn't supposed to be handled, as the inner loop doesn't
7642 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7643 OMP_CLAUSE__LOOPTEMP_
);
7644 gcc_assert (innerc
);
7645 for (i
= 0; i
< fd
->collapse
; i
++)
7647 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7648 OMP_CLAUSE__LOOPTEMP_
);
7649 gcc_assert (innerc
);
7651 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7653 counts
[0] = NULL_TREE
;
7658 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7660 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7661 counts
[i
] = NULL_TREE
;
7662 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7663 fold_convert (itype
, fd
->loops
[i
].n1
),
7664 fold_convert (itype
, fd
->loops
[i
].n2
));
7665 if (t
&& integer_zerop (t
))
7667 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7668 counts
[i
] = build_int_cst (type
, 0);
7672 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7674 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7676 if (i
>= fd
->collapse
&& counts
[i
])
7678 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7679 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7680 fold_convert (itype
, fd
->loops
[i
].n1
),
7681 fold_convert (itype
, fd
->loops
[i
].n2
)))
7682 == NULL_TREE
|| !integer_onep (t
)))
7686 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7687 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7688 true, GSI_SAME_STMT
);
7689 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7690 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7691 true, GSI_SAME_STMT
);
7692 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7693 NULL_TREE
, NULL_TREE
);
7694 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7695 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7696 expand_omp_regimplify_p
, NULL
, NULL
)
7697 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7698 expand_omp_regimplify_p
, NULL
, NULL
))
7700 *gsi
= gsi_for_stmt (cond_stmt
);
7701 gimple_regimplify_operands (cond_stmt
, gsi
);
7703 e
= split_block (entry_bb
, cond_stmt
);
7704 basic_block
&zero_iter_bb
7705 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7706 int &first_zero_iter
7707 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7708 if (zero_iter_bb
== NULL
)
7710 gassign
*assign_stmt
;
7711 first_zero_iter
= i
;
7712 zero_iter_bb
= create_empty_bb (entry_bb
);
7713 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7714 *gsi
= gsi_after_labels (zero_iter_bb
);
7715 if (i
< fd
->collapse
)
7716 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7717 build_zero_cst (type
));
7720 counts
[i
] = create_tmp_reg (type
, ".count");
7722 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7724 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7725 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7728 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7729 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7730 e
->flags
= EDGE_TRUE_VALUE
;
7731 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7732 if (l2_dom_bb
== NULL
)
7733 l2_dom_bb
= entry_bb
;
7735 *gsi
= gsi_last_bb (entry_bb
);
7738 if (POINTER_TYPE_P (itype
))
7739 itype
= signed_type_for (itype
);
7740 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7742 t
= fold_build2 (PLUS_EXPR
, itype
,
7743 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7744 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7745 fold_convert (itype
, fd
->loops
[i
].n2
));
7746 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7747 fold_convert (itype
, fd
->loops
[i
].n1
));
7748 /* ?? We could probably use CEIL_DIV_EXPR instead of
7749 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7750 generate the same code in the end because generically we
7751 don't know that the values involved must be negative for
7753 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7754 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7755 fold_build1 (NEGATE_EXPR
, itype
, t
),
7756 fold_build1 (NEGATE_EXPR
, itype
,
7757 fold_convert (itype
,
7758 fd
->loops
[i
].step
)));
7760 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7761 fold_convert (itype
, fd
->loops
[i
].step
));
7762 t
= fold_convert (type
, t
);
7763 if (TREE_CODE (t
) == INTEGER_CST
)
7767 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7768 counts
[i
] = create_tmp_reg (type
, ".count");
7769 expand_omp_build_assign (gsi
, counts
[i
], t
);
7771 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7776 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7777 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7783 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7785 V3 = N31 + (T % count3) * STEP3;
7787 V2 = N21 + (T % count2) * STEP2;
7789 V1 = N11 + T * STEP1;
7790 if this loop doesn't have an inner loop construct combined with it.
7791 If it does have an inner loop construct combined with it and the
7792 iteration count isn't known constant, store values from counts array
7793 into its _looptemp_ temporaries instead. */
7796 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7797 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7800 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7802 /* If fd->loop.n2 is constant, then no propagation of the counts
7803 is needed, they are constant. */
7804 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7807 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7808 ? gimple_omp_taskreg_clauses (inner_stmt
)
7809 : gimple_omp_for_clauses (inner_stmt
);
7810 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7811 isn't supposed to be handled, as the inner loop doesn't
7813 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7814 gcc_assert (innerc
);
7815 for (i
= 0; i
< fd
->collapse
; i
++)
7817 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7818 OMP_CLAUSE__LOOPTEMP_
);
7819 gcc_assert (innerc
);
7822 tree tem
= OMP_CLAUSE_DECL (innerc
);
7823 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7824 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7825 false, GSI_CONTINUE_LINKING
);
7826 gassign
*stmt
= gimple_build_assign (tem
, t
);
7827 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7833 tree type
= TREE_TYPE (fd
->loop
.v
);
7834 tree tem
= create_tmp_reg (type
, ".tem");
7835 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7836 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7838 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7840 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7842 if (POINTER_TYPE_P (vtype
))
7843 itype
= signed_type_for (vtype
);
7845 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7848 t
= fold_convert (itype
, t
);
7849 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7850 fold_convert (itype
, fd
->loops
[i
].step
));
7851 if (POINTER_TYPE_P (vtype
))
7852 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7854 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7855 t
= force_gimple_operand_gsi (gsi
, t
,
7856 DECL_P (fd
->loops
[i
].v
)
7857 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7859 GSI_CONTINUE_LINKING
);
7860 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7861 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7864 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7865 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7866 false, GSI_CONTINUE_LINKING
);
7867 stmt
= gimple_build_assign (tem
, t
);
7868 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7874 /* Helper function for expand_omp_for_*. Generate code like:
7877 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7881 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7888 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7889 basic_block body_bb
)
7891 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7893 gimple_stmt_iterator gsi
;
7899 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7901 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7903 bb
= create_empty_bb (last_bb
);
7904 add_bb_to_loop (bb
, last_bb
->loop_father
);
7905 gsi
= gsi_start_bb (bb
);
7907 if (i
< fd
->collapse
- 1)
7909 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7910 e
->probability
= REG_BR_PROB_BASE
/ 8;
7912 t
= fd
->loops
[i
+ 1].n1
;
7913 t
= force_gimple_operand_gsi (&gsi
, t
,
7914 DECL_P (fd
->loops
[i
+ 1].v
)
7915 && TREE_ADDRESSABLE (fd
->loops
[i
7918 GSI_CONTINUE_LINKING
);
7919 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7920 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7925 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7927 if (POINTER_TYPE_P (vtype
))
7928 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7930 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7931 t
= force_gimple_operand_gsi (&gsi
, t
,
7932 DECL_P (fd
->loops
[i
].v
)
7933 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7934 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7935 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7936 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7940 t
= fd
->loops
[i
].n2
;
7941 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7942 false, GSI_CONTINUE_LINKING
);
7943 tree v
= fd
->loops
[i
].v
;
7944 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7945 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7946 false, GSI_CONTINUE_LINKING
);
7947 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7948 stmt
= gimple_build_cond_empty (t
);
7949 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7950 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7951 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7954 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7962 /* Expand #pragma omp ordered depend(source). */
7965 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7966 tree
*counts
, location_t loc
)
7968 enum built_in_function source_ix
7969 = fd
->iter_type
== long_integer_type_node
7970 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7972 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7973 build_fold_addr_expr (counts
[fd
->ordered
]));
7974 gimple_set_location (g
, loc
);
7975 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7978 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7981 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7982 tree
*counts
, tree c
, location_t loc
)
7984 auto_vec
<tree
, 10> args
;
7985 enum built_in_function sink_ix
7986 = fd
->iter_type
== long_integer_type_node
7987 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7988 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7990 gimple_stmt_iterator gsi2
= *gsi
;
7991 bool warned_step
= false;
7993 for (i
= 0; i
< fd
->ordered
; i
++)
7995 off
= TREE_PURPOSE (deps
);
7996 if (!integer_zerop (off
))
7998 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7999 || fd
->loops
[i
].cond_code
== GT_EXPR
);
8000 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
8001 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8002 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
8003 "lexically later iteration");
8006 deps
= TREE_CHAIN (deps
);
8008 /* If all offsets corresponding to the collapsed loops are zero,
8009 this depend clause can be ignored. FIXME: but there is still a
8010 flush needed. We need to emit one __sync_synchronize () for it
8011 though (perhaps conditionally)? Solve this together with the
8012 conservative dependence folding optimization.
8013 if (i >= fd->collapse)
8016 deps
= OMP_CLAUSE_DECL (c
);
8018 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
8019 edge e2
= split_block_after_labels (e1
->dest
);
8021 *gsi
= gsi_after_labels (e1
->dest
);
8022 for (i
= 0; i
< fd
->ordered
; i
++)
8024 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
8025 if (POINTER_TYPE_P (itype
))
8028 deps
= TREE_CHAIN (deps
);
8029 off
= TREE_PURPOSE (deps
);
8030 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
8032 if (integer_zerop (off
))
8033 t
= boolean_true_node
;
8037 tree co
= fold_convert_loc (loc
, itype
, off
);
8038 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
8040 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8041 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
8042 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
8043 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
8046 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8047 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8048 fd
->loops
[i
].v
, co
);
8050 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8051 fd
->loops
[i
].v
, co
);
8052 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
8054 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8055 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
8058 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
8061 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8062 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
8065 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
8069 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
8073 off
= fold_convert_loc (loc
, itype
, off
);
8075 if (fd
->loops
[i
].cond_code
== LT_EXPR
8076 ? !integer_onep (fd
->loops
[i
].step
)
8077 : !integer_minus_onep (fd
->loops
[i
].step
))
8079 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8080 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
8081 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8084 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
8085 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
8086 build_int_cst (itype
, 0));
8087 if (integer_zerop (t
) && !warned_step
)
8089 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
8090 "in the iteration space");
8093 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
8097 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8103 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8104 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8105 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
8107 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8108 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
8109 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8112 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
8113 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8114 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
8115 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
8116 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8119 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
8121 if (i
< fd
->collapse
- 1)
8123 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
8128 off
= unshare_expr (off
);
8129 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
8130 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
8131 true, GSI_SAME_STMT
);
8134 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
8135 gimple_set_location (g
, loc
);
8136 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
8138 *gsi
= gsi_last_bb (e1
->src
);
8139 cond
= unshare_expr (cond
);
8140 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
8141 GSI_CONTINUE_LINKING
);
8142 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
8143 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
8144 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8145 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8146 e1
->flags
= EDGE_TRUE_VALUE
;
8147 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
8149 *gsi
= gsi_after_labels (e2
->dest
);
8152 /* Expand all #pragma omp ordered depend(source) and
8153 #pragma omp ordered depend(sink:...) constructs in the current
8154 #pragma omp for ordered(n) region. */
8157 expand_omp_ordered_source_sink (struct omp_region
*region
,
8158 struct omp_for_data
*fd
, tree
*counts
,
8159 basic_block cont_bb
)
8161 struct omp_region
*inner
;
8163 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
8164 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
8165 counts
[i
] = NULL_TREE
;
8166 else if (i
>= fd
->collapse
&& !cont_bb
)
8167 counts
[i
] = build_zero_cst (fd
->iter_type
);
8168 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
8169 && integer_onep (fd
->loops
[i
].step
))
8170 counts
[i
] = NULL_TREE
;
8172 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
8174 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
8175 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
8176 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
8178 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
8179 if (inner
->type
== GIMPLE_OMP_ORDERED
)
8181 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
8182 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
8183 location_t loc
= gimple_location (ord_stmt
);
8185 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8186 c
; c
= OMP_CLAUSE_CHAIN (c
))
8187 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
8190 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
8191 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8192 c
; c
= OMP_CLAUSE_CHAIN (c
))
8193 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8194 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
8195 gsi_remove (&gsi
, true);
8199 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8203 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
8204 basic_block cont_bb
, basic_block body_bb
,
8205 bool ordered_lastprivate
)
8207 if (fd
->ordered
== fd
->collapse
)
8212 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8213 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8215 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8216 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
8217 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
8218 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8219 size_int (i
- fd
->collapse
+ 1),
8220 NULL_TREE
, NULL_TREE
);
8221 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8226 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
8228 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
8229 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8230 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8231 fold_convert (type
, fd
->loops
[i
].n1
));
8233 expand_omp_build_assign (&gsi
, counts
[i
],
8234 build_zero_cst (fd
->iter_type
));
8235 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8236 size_int (i
- fd
->collapse
+ 1),
8237 NULL_TREE
, NULL_TREE
);
8238 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8239 if (!gsi_end_p (gsi
))
8242 gsi
= gsi_last_bb (body_bb
);
8243 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
8244 basic_block new_body
= e1
->dest
;
8245 if (body_bb
== cont_bb
)
8248 basic_block new_header
;
8249 if (EDGE_COUNT (cont_bb
->preds
) > 0)
8251 gsi
= gsi_last_bb (cont_bb
);
8252 if (POINTER_TYPE_P (type
))
8253 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8254 fold_convert (sizetype
,
8255 fd
->loops
[i
].step
));
8257 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8258 fold_convert (type
, fd
->loops
[i
].step
));
8259 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8262 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8263 build_int_cst (fd
->iter_type
, 1));
8264 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8269 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8270 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8271 t
= fold_convert (fd
->iter_type
, t
);
8272 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8273 true, GSI_SAME_STMT
);
8275 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8276 size_int (i
- fd
->collapse
+ 1),
8277 NULL_TREE
, NULL_TREE
);
8278 expand_omp_build_assign (&gsi
, aref
, t
);
8280 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8281 new_header
= e2
->dest
;
8284 new_header
= cont_bb
;
8285 gsi
= gsi_after_labels (new_header
);
8286 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8287 true, GSI_SAME_STMT
);
8289 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8290 true, NULL_TREE
, true, GSI_SAME_STMT
);
8291 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8292 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8293 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8296 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8297 e3
->flags
= EDGE_FALSE_VALUE
;
8298 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8299 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8300 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8302 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8303 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8307 struct loop
*loop
= alloc_loop ();
8308 loop
->header
= new_header
;
8309 loop
->latch
= e2
->src
;
8310 add_loop (loop
, body_bb
->loop_father
);
8314 /* If there are any lastprivate clauses and it is possible some loops
8315 might have zero iterations, ensure all the decls are initialized,
8316 otherwise we could crash evaluating C++ class iterators with lastprivate
8318 bool need_inits
= false;
8319 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8322 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8323 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8324 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8325 fold_convert (type
, fd
->loops
[i
].n1
));
8329 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8330 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8332 fold_convert (type
, fd
->loops
[i
].n1
),
8333 fold_convert (type
, fd
->loops
[i
].n2
));
8334 if (!integer_onep (this_cond
))
8342 /* A subroutine of expand_omp_for. Generate code for a parallel
8343 loop with any schedule. Given parameters:
8345 for (V = N1; V cond N2; V += STEP) BODY;
8347 where COND is "<" or ">", we generate pseudocode
8349 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8350 if (more) goto L0; else goto L3;
8357 if (V cond iend) goto L1; else goto L2;
8359 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8362 If this is a combined omp parallel loop, instead of the call to
8363 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8364 If this is gimple_omp_for_combined_p loop, then instead of assigning
8365 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8366 inner GIMPLE_OMP_FOR and V += STEP; and
8367 if (V cond iend) goto L1; else goto L2; are removed.
8369 For collapsed loops, given parameters:
8371 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8372 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8373 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8376 we generate pseudocode
8378 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8383 count3 = (adj + N32 - N31) / STEP3;
8384 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8389 count2 = (adj + N22 - N21) / STEP2;
8390 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8395 count1 = (adj + N12 - N11) / STEP1;
8396 count = count1 * count2 * count3;
8401 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8402 if (more) goto L0; else goto L3;
8406 V3 = N31 + (T % count3) * STEP3;
8408 V2 = N21 + (T % count2) * STEP2;
8410 V1 = N11 + T * STEP1;
8415 if (V < iend) goto L10; else goto L2;
8418 if (V3 cond3 N32) goto L1; else goto L11;
8422 if (V2 cond2 N22) goto L1; else goto L12;
8428 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8434 expand_omp_for_generic (struct omp_region
*region
,
8435 struct omp_for_data
*fd
,
8436 enum built_in_function start_fn
,
8437 enum built_in_function next_fn
,
8440 tree type
, istart0
, iend0
, iend
;
8441 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8442 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8443 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8444 gimple_stmt_iterator gsi
;
8445 gassign
*assign_stmt
;
8446 bool in_combined_parallel
= is_combined_parallel (region
);
8447 bool broken_loop
= region
->cont
== NULL
;
8449 tree
*counts
= NULL
;
8451 bool ordered_lastprivate
= false;
8453 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8454 gcc_assert (fd
->iter_type
== long_integer_type_node
8455 || !in_combined_parallel
);
8457 entry_bb
= region
->entry
;
8458 cont_bb
= region
->cont
;
8460 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8461 gcc_assert (broken_loop
8462 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8463 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8464 l1_bb
= single_succ (l0_bb
);
8467 l2_bb
= create_empty_bb (cont_bb
);
8468 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8469 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8471 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8475 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8476 exit_bb
= region
->exit
;
8478 gsi
= gsi_last_bb (entry_bb
);
8480 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8482 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8483 OMP_CLAUSE_LASTPRIVATE
))
8484 ordered_lastprivate
= false;
8485 if (fd
->collapse
> 1 || fd
->ordered
)
8487 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8488 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8490 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8491 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8492 zero_iter1_bb
, first_zero_iter1
,
8493 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8497 /* Some counts[i] vars might be uninitialized if
8498 some loop has zero iterations. But the body shouldn't
8499 be executed in that case, so just avoid uninit warnings. */
8500 for (i
= first_zero_iter1
;
8501 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8502 if (SSA_VAR_P (counts
[i
]))
8503 TREE_NO_WARNING (counts
[i
]) = 1;
8505 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8507 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8508 gsi
= gsi_last_bb (entry_bb
);
8509 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8510 get_immediate_dominator (CDI_DOMINATORS
,
8515 /* Some counts[i] vars might be uninitialized if
8516 some loop has zero iterations. But the body shouldn't
8517 be executed in that case, so just avoid uninit warnings. */
8518 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8519 if (SSA_VAR_P (counts
[i
]))
8520 TREE_NO_WARNING (counts
[i
]) = 1;
8522 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8526 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8528 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8529 gsi
= gsi_last_bb (entry_bb
);
8530 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8531 get_immediate_dominator
8532 (CDI_DOMINATORS
, zero_iter2_bb
));
8535 if (fd
->collapse
== 1)
8537 counts
[0] = fd
->loop
.n2
;
8538 fd
->loop
= fd
->loops
[0];
8542 type
= TREE_TYPE (fd
->loop
.v
);
8543 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8544 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8545 TREE_ADDRESSABLE (istart0
) = 1;
8546 TREE_ADDRESSABLE (iend0
) = 1;
8548 /* See if we need to bias by LLONG_MIN. */
8549 if (fd
->iter_type
== long_long_unsigned_type_node
8550 && TREE_CODE (type
) == INTEGER_TYPE
8551 && !TYPE_UNSIGNED (type
)
8552 && fd
->ordered
== 0)
8556 if (fd
->loop
.cond_code
== LT_EXPR
)
8559 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8563 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8566 if (TREE_CODE (n1
) != INTEGER_CST
8567 || TREE_CODE (n2
) != INTEGER_CST
8568 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8569 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8572 gimple_stmt_iterator gsif
= gsi
;
8575 tree arr
= NULL_TREE
;
8576 if (in_combined_parallel
)
8578 gcc_assert (fd
->ordered
== 0);
8579 /* In a combined parallel loop, emit a call to
8580 GOMP_loop_foo_next. */
8581 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8582 build_fold_addr_expr (istart0
),
8583 build_fold_addr_expr (iend0
));
8587 tree t0
, t1
, t2
, t3
, t4
;
8588 /* If this is not a combined parallel loop, emit a call to
8589 GOMP_loop_foo_start in ENTRY_BB. */
8590 t4
= build_fold_addr_expr (iend0
);
8591 t3
= build_fold_addr_expr (istart0
);
8594 t0
= build_int_cst (unsigned_type_node
,
8595 fd
->ordered
- fd
->collapse
+ 1);
8596 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8598 - fd
->collapse
+ 1),
8600 DECL_NAMELESS (arr
) = 1;
8601 TREE_ADDRESSABLE (arr
) = 1;
8602 TREE_STATIC (arr
) = 1;
8603 vec
<constructor_elt
, va_gc
> *v
;
8604 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8607 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8610 if (idx
== 0 && fd
->collapse
> 1)
8613 c
= counts
[idx
+ fd
->collapse
- 1];
8614 tree purpose
= size_int (idx
);
8615 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8616 if (TREE_CODE (c
) != INTEGER_CST
)
8617 TREE_STATIC (arr
) = 0;
8620 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8621 if (!TREE_STATIC (arr
))
8622 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8623 void_type_node
, arr
),
8624 true, NULL_TREE
, true, GSI_SAME_STMT
);
8625 t1
= build_fold_addr_expr (arr
);
8630 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8633 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8636 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8637 OMP_CLAUSE__LOOPTEMP_
);
8638 gcc_assert (innerc
);
8639 t0
= OMP_CLAUSE_DECL (innerc
);
8640 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8641 OMP_CLAUSE__LOOPTEMP_
);
8642 gcc_assert (innerc
);
8643 t1
= OMP_CLAUSE_DECL (innerc
);
8645 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8646 && TYPE_PRECISION (TREE_TYPE (t0
))
8647 != TYPE_PRECISION (fd
->iter_type
))
8649 /* Avoid casting pointers to integer of a different size. */
8650 tree itype
= signed_type_for (type
);
8651 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8652 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8656 t1
= fold_convert (fd
->iter_type
, t1
);
8657 t0
= fold_convert (fd
->iter_type
, t0
);
8661 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8662 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8665 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8669 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8670 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8672 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8673 5, t0
, t1
, t
, t3
, t4
);
8675 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8676 6, t0
, t1
, t2
, t
, t3
, t4
);
8678 else if (fd
->ordered
)
8679 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8682 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8683 5, t0
, t1
, t2
, t3
, t4
);
8691 /* The GOMP_loop_ull_*start functions have additional boolean
8692 argument, true for < loops and false for > loops.
8693 In Fortran, the C bool type can be different from
8694 boolean_type_node. */
8695 bfn_decl
= builtin_decl_explicit (start_fn
);
8696 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8697 t5
= build_int_cst (c_bool_type
,
8698 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8701 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8702 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8703 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8704 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8707 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8708 6, t5
, t0
, t1
, t2
, t3
, t4
);
8711 if (TREE_TYPE (t
) != boolean_type_node
)
8712 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8713 t
, build_int_cst (TREE_TYPE (t
), 0));
8714 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8715 true, GSI_SAME_STMT
);
8716 if (arr
&& !TREE_STATIC (arr
))
8718 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8719 TREE_THIS_VOLATILE (clobber
) = 1;
8720 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8723 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8725 /* Remove the GIMPLE_OMP_FOR statement. */
8726 gsi_remove (&gsi
, true);
8728 if (gsi_end_p (gsif
))
8729 gsif
= gsi_after_labels (gsi_bb (gsif
));
8732 /* Iteration setup for sequential loop goes in L0_BB. */
8733 tree startvar
= fd
->loop
.v
;
8734 tree endvar
= NULL_TREE
;
8736 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8738 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8739 && gimple_omp_for_kind (inner_stmt
)
8740 == GF_OMP_FOR_KIND_SIMD
);
8741 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8742 OMP_CLAUSE__LOOPTEMP_
);
8743 gcc_assert (innerc
);
8744 startvar
= OMP_CLAUSE_DECL (innerc
);
8745 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8746 OMP_CLAUSE__LOOPTEMP_
);
8747 gcc_assert (innerc
);
8748 endvar
= OMP_CLAUSE_DECL (innerc
);
8751 gsi
= gsi_start_bb (l0_bb
);
8753 if (fd
->ordered
&& fd
->collapse
== 1)
8754 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8755 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8757 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8758 if (fd
->ordered
&& fd
->collapse
== 1)
8760 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8761 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8762 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8765 t
= fold_convert (TREE_TYPE (startvar
), t
);
8766 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8772 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8773 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8774 t
= fold_convert (TREE_TYPE (startvar
), t
);
8776 t
= force_gimple_operand_gsi (&gsi
, t
,
8778 && TREE_ADDRESSABLE (startvar
),
8779 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8780 assign_stmt
= gimple_build_assign (startvar
, t
);
8781 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8784 if (fd
->ordered
&& fd
->collapse
== 1)
8785 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8786 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8788 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8789 if (fd
->ordered
&& fd
->collapse
== 1)
8791 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8792 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8793 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8796 t
= fold_convert (TREE_TYPE (startvar
), t
);
8797 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8803 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8804 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8805 t
= fold_convert (TREE_TYPE (startvar
), t
);
8807 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8808 false, GSI_CONTINUE_LINKING
);
8811 assign_stmt
= gimple_build_assign (endvar
, iend
);
8812 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8813 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8814 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8816 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8817 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8819 /* Handle linear clause adjustments. */
8820 tree itercnt
= NULL_TREE
;
8821 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8822 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8823 c
; c
= OMP_CLAUSE_CHAIN (c
))
8824 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8825 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8827 tree d
= OMP_CLAUSE_DECL (c
);
8828 bool is_ref
= is_reference (d
);
8829 tree t
= d
, a
, dest
;
8831 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8832 tree type
= TREE_TYPE (t
);
8833 if (POINTER_TYPE_P (type
))
8835 dest
= unshare_expr (t
);
8836 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8837 expand_omp_build_assign (&gsif
, v
, t
);
8838 if (itercnt
== NULL_TREE
)
8841 tree n1
= fd
->loop
.n1
;
8842 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8845 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8847 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8849 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8851 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8852 itercnt
, fd
->loop
.step
);
8853 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8855 GSI_CONTINUE_LINKING
);
8857 a
= fold_build2 (MULT_EXPR
, type
,
8858 fold_convert (type
, itercnt
),
8859 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8860 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8861 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8862 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8863 false, GSI_CONTINUE_LINKING
);
8864 assign_stmt
= gimple_build_assign (dest
, t
);
8865 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8867 if (fd
->collapse
> 1)
8868 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8872 /* Until now, counts array contained number of iterations or
8873 variable containing it for ith loop. From now on, we need
8874 those counts only for collapsed loops, and only for the 2nd
8875 till the last collapsed one. Move those one element earlier,
8876 we'll use counts[fd->collapse - 1] for the first source/sink
8877 iteration counter and so on and counts[fd->ordered]
8878 as the array holding the current counter values for
8880 if (fd
->collapse
> 1)
8881 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8885 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8887 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8889 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8890 fold_convert (type
, fd
->loops
[i
].n1
),
8891 fold_convert (type
, fd
->loops
[i
].n2
));
8892 if (!integer_onep (this_cond
))
8895 if (i
< fd
->ordered
)
8898 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8899 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8900 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8901 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8902 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8903 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8904 make_edge (cont_bb
, l1_bb
, 0);
8905 l2_bb
= create_empty_bb (cont_bb
);
8906 broken_loop
= false;
8909 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8910 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8911 ordered_lastprivate
);
8912 if (counts
[fd
->collapse
- 1])
8914 gcc_assert (fd
->collapse
== 1);
8915 gsi
= gsi_last_bb (l0_bb
);
8916 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8918 gsi
= gsi_last_bb (cont_bb
);
8919 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8920 build_int_cst (fd
->iter_type
, 1));
8921 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8922 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8923 size_zero_node
, NULL_TREE
, NULL_TREE
);
8924 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8925 t
= counts
[fd
->collapse
- 1];
8927 else if (fd
->collapse
> 1)
8931 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8932 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8933 t
= fold_convert (fd
->iter_type
, t
);
8935 gsi
= gsi_last_bb (l0_bb
);
8936 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8937 size_zero_node
, NULL_TREE
, NULL_TREE
);
8938 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8939 false, GSI_CONTINUE_LINKING
);
8940 expand_omp_build_assign (&gsi
, aref
, t
, true);
8945 /* Code to control the increment and predicate for the sequential
8946 loop goes in the CONT_BB. */
8947 gsi
= gsi_last_bb (cont_bb
);
8948 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8949 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8950 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8951 vback
= gimple_omp_continue_control_def (cont_stmt
);
8953 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8955 if (POINTER_TYPE_P (type
))
8956 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8958 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8959 t
= force_gimple_operand_gsi (&gsi
, t
,
8961 && TREE_ADDRESSABLE (vback
),
8962 NULL_TREE
, true, GSI_SAME_STMT
);
8963 assign_stmt
= gimple_build_assign (vback
, t
);
8964 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8966 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8968 if (fd
->collapse
> 1)
8972 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8973 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8974 t
= fold_convert (fd
->iter_type
, t
);
8976 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8977 counts
[fd
->ordered
], size_zero_node
,
8978 NULL_TREE
, NULL_TREE
);
8979 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8980 true, GSI_SAME_STMT
);
8981 expand_omp_build_assign (&gsi
, aref
, t
);
8984 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8985 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8987 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8988 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8991 /* Remove GIMPLE_OMP_CONTINUE. */
8992 gsi_remove (&gsi
, true);
8994 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8995 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8997 /* Emit code to get the next parallel iteration in L2_BB. */
8998 gsi
= gsi_start_bb (l2_bb
);
9000 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
9001 build_fold_addr_expr (istart0
),
9002 build_fold_addr_expr (iend0
));
9003 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9004 false, GSI_CONTINUE_LINKING
);
9005 if (TREE_TYPE (t
) != boolean_type_node
)
9006 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
9007 t
, build_int_cst (TREE_TYPE (t
), 0));
9008 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9009 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
9012 /* Add the loop cleanup function. */
9013 gsi
= gsi_last_bb (exit_bb
);
9014 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9015 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
9016 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9017 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
9019 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
9020 gcall
*call_stmt
= gimple_build_call (t
, 0);
9021 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
9022 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
9023 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
9026 tree arr
= counts
[fd
->ordered
];
9027 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
9028 TREE_THIS_VOLATILE (clobber
) = 1;
9029 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
9032 gsi_remove (&gsi
, true);
9034 /* Connect the new blocks. */
9035 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
9036 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
9042 e
= find_edge (cont_bb
, l3_bb
);
9043 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
9045 phis
= phi_nodes (l3_bb
);
9046 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9048 gimple
*phi
= gsi_stmt (gsi
);
9049 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
9050 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
9054 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
9055 e
= find_edge (cont_bb
, l1_bb
);
9058 e
= BRANCH_EDGE (cont_bb
);
9059 gcc_assert (single_succ (e
->dest
) == l1_bb
);
9061 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9066 else if (fd
->collapse
> 1)
9069 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9072 e
->flags
= EDGE_TRUE_VALUE
;
9075 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9076 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
9080 e
= find_edge (cont_bb
, l2_bb
);
9081 e
->flags
= EDGE_FALLTHRU
;
9083 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9085 if (gimple_in_ssa_p (cfun
))
9087 /* Add phis to the outer loop that connect to the phis in the inner,
9088 original loop, and move the loop entry value of the inner phi to
9089 the loop entry value of the outer phi. */
9091 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
9093 source_location locus
;
9095 gphi
*exit_phi
= psi
.phi ();
9097 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
9098 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
9100 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
9101 edge latch_to_l1
= find_edge (latch
, l1_bb
);
9103 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
9105 tree t
= gimple_phi_result (exit_phi
);
9106 tree new_res
= copy_ssa_name (t
, NULL
);
9107 nphi
= create_phi_node (new_res
, l0_bb
);
9109 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
9110 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
9111 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
9112 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
9113 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
9115 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
9116 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
9118 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
9122 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
9123 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
9124 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
9125 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
9126 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
9127 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
9128 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
9129 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
9131 /* We enter expand_omp_for_generic with a loop. This original loop may
9132 have its own loop struct, or it may be part of an outer loop struct
9133 (which may be the fake loop). */
9134 struct loop
*outer_loop
= entry_bb
->loop_father
;
9135 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
9137 add_bb_to_loop (l2_bb
, outer_loop
);
9139 /* We've added a new loop around the original loop. Allocate the
9140 corresponding loop struct. */
9141 struct loop
*new_loop
= alloc_loop ();
9142 new_loop
->header
= l0_bb
;
9143 new_loop
->latch
= l2_bb
;
9144 add_loop (new_loop
, outer_loop
);
9146 /* Allocate a loop structure for the original loop unless we already
9148 if (!orig_loop_has_loop_struct
9149 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9151 struct loop
*orig_loop
= alloc_loop ();
9152 orig_loop
->header
= l1_bb
;
9153 /* The loop may have multiple latches. */
9154 add_loop (orig_loop
, new_loop
);
9160 /* A subroutine of expand_omp_for. Generate code for a parallel
9161 loop with static schedule and no specified chunk size. Given
9164 for (V = N1; V cond N2; V += STEP) BODY;
9166 where COND is "<" or ">", we generate pseudocode
9168 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9173 if ((__typeof (V)) -1 > 0 && cond is >)
9174 n = -(adj + N2 - N1) / -STEP;
9176 n = (adj + N2 - N1) / STEP;
9179 if (threadid < tt) goto L3; else goto L4;
9184 s0 = q * threadid + tt;
9187 if (s0 >= e0) goto L2; else goto L0;
9193 if (V cond e) goto L1;
9198 expand_omp_for_static_nochunk (struct omp_region
*region
,
9199 struct omp_for_data
*fd
,
9202 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
9203 tree type
, itype
, vmain
, vback
;
9204 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
9205 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
9207 gimple_stmt_iterator gsi
;
9209 bool broken_loop
= region
->cont
== NULL
;
9210 tree
*counts
= NULL
;
9213 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9214 if (POINTER_TYPE_P (type
))
9215 itype
= signed_type_for (type
);
9217 entry_bb
= region
->entry
;
9218 cont_bb
= region
->cont
;
9219 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9220 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9221 gcc_assert (broken_loop
9222 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
9223 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
9224 body_bb
= single_succ (seq_start_bb
);
9227 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9228 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9229 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9231 exit_bb
= region
->exit
;
9233 /* Iteration space partitioning goes in ENTRY_BB. */
9234 gsi
= gsi_last_bb (entry_bb
);
9235 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9237 if (fd
->collapse
> 1)
9239 int first_zero_iter
= -1, dummy
= -1;
9240 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9242 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9243 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9244 fin_bb
, first_zero_iter
,
9245 dummy_bb
, dummy
, l2_dom_bb
);
9248 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9249 t
= integer_one_node
;
9251 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9252 fold_convert (type
, fd
->loop
.n1
),
9253 fold_convert (type
, fd
->loop
.n2
));
9254 if (fd
->collapse
== 1
9255 && TYPE_UNSIGNED (type
)
9256 && (t
== NULL_TREE
|| !integer_onep (t
)))
9258 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9259 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9260 true, GSI_SAME_STMT
);
9261 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9262 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9263 true, GSI_SAME_STMT
);
9264 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9265 NULL_TREE
, NULL_TREE
);
9266 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9267 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9268 expand_omp_regimplify_p
, NULL
, NULL
)
9269 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9270 expand_omp_regimplify_p
, NULL
, NULL
))
9272 gsi
= gsi_for_stmt (cond_stmt
);
9273 gimple_regimplify_operands (cond_stmt
, &gsi
);
9275 ep
= split_block (entry_bb
, cond_stmt
);
9276 ep
->flags
= EDGE_TRUE_VALUE
;
9277 entry_bb
= ep
->dest
;
9278 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9279 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9280 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9281 if (gimple_in_ssa_p (cfun
))
9283 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9284 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9285 !gsi_end_p (gpi
); gsi_next (&gpi
))
9287 gphi
*phi
= gpi
.phi ();
9288 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9289 ep
, UNKNOWN_LOCATION
);
9292 gsi
= gsi_last_bb (entry_bb
);
9295 switch (gimple_omp_for_kind (fd
->for_stmt
))
9297 case GF_OMP_FOR_KIND_FOR
:
9298 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9299 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9301 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9302 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9303 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9308 nthreads
= build_call_expr (nthreads
, 0);
9309 nthreads
= fold_convert (itype
, nthreads
);
9310 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9311 true, GSI_SAME_STMT
);
9312 threadid
= build_call_expr (threadid
, 0);
9313 threadid
= fold_convert (itype
, threadid
);
9314 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9315 true, GSI_SAME_STMT
);
9319 step
= fd
->loop
.step
;
9320 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9322 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9323 OMP_CLAUSE__LOOPTEMP_
);
9324 gcc_assert (innerc
);
9325 n1
= OMP_CLAUSE_DECL (innerc
);
9326 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9327 OMP_CLAUSE__LOOPTEMP_
);
9328 gcc_assert (innerc
);
9329 n2
= OMP_CLAUSE_DECL (innerc
);
9331 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9332 true, NULL_TREE
, true, GSI_SAME_STMT
);
9333 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9334 true, NULL_TREE
, true, GSI_SAME_STMT
);
9335 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9336 true, NULL_TREE
, true, GSI_SAME_STMT
);
9338 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9339 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9340 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9341 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9342 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9343 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9344 fold_build1 (NEGATE_EXPR
, itype
, t
),
9345 fold_build1 (NEGATE_EXPR
, itype
, step
));
9347 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9348 t
= fold_convert (itype
, t
);
9349 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9351 q
= create_tmp_reg (itype
, "q");
9352 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9353 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9354 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9356 tt
= create_tmp_reg (itype
, "tt");
9357 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9358 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9359 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9361 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9362 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9363 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9365 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9366 gsi
= gsi_last_bb (second_bb
);
9367 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9369 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9371 gassign
*assign_stmt
9372 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9373 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9375 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9376 gsi
= gsi_last_bb (third_bb
);
9377 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9379 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9380 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9381 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9383 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9384 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9386 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9387 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9389 /* Remove the GIMPLE_OMP_FOR statement. */
9390 gsi_remove (&gsi
, true);
9392 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9393 gsi
= gsi_start_bb (seq_start_bb
);
9395 tree startvar
= fd
->loop
.v
;
9396 tree endvar
= NULL_TREE
;
9398 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9400 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9401 ? gimple_omp_parallel_clauses (inner_stmt
)
9402 : gimple_omp_for_clauses (inner_stmt
);
9403 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9404 gcc_assert (innerc
);
9405 startvar
= OMP_CLAUSE_DECL (innerc
);
9406 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9407 OMP_CLAUSE__LOOPTEMP_
);
9408 gcc_assert (innerc
);
9409 endvar
= OMP_CLAUSE_DECL (innerc
);
9410 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9411 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9414 for (i
= 1; i
< fd
->collapse
; i
++)
9416 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9417 OMP_CLAUSE__LOOPTEMP_
);
9418 gcc_assert (innerc
);
9420 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9421 OMP_CLAUSE__LOOPTEMP_
);
9424 /* If needed (distribute parallel for with lastprivate),
9425 propagate down the total number of iterations. */
9426 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9428 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9429 GSI_CONTINUE_LINKING
);
9430 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9431 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9435 t
= fold_convert (itype
, s0
);
9436 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9437 if (POINTER_TYPE_P (type
))
9438 t
= fold_build_pointer_plus (n1
, t
);
9440 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9441 t
= fold_convert (TREE_TYPE (startvar
), t
);
9442 t
= force_gimple_operand_gsi (&gsi
, t
,
9444 && TREE_ADDRESSABLE (startvar
),
9445 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9446 assign_stmt
= gimple_build_assign (startvar
, t
);
9447 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9449 t
= fold_convert (itype
, e0
);
9450 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9451 if (POINTER_TYPE_P (type
))
9452 t
= fold_build_pointer_plus (n1
, t
);
9454 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9455 t
= fold_convert (TREE_TYPE (startvar
), t
);
9456 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9457 false, GSI_CONTINUE_LINKING
);
9460 assign_stmt
= gimple_build_assign (endvar
, e
);
9461 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9462 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9463 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9465 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9466 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9468 /* Handle linear clause adjustments. */
9469 tree itercnt
= NULL_TREE
;
9470 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9471 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9472 c
; c
= OMP_CLAUSE_CHAIN (c
))
9473 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9474 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9476 tree d
= OMP_CLAUSE_DECL (c
);
9477 bool is_ref
= is_reference (d
);
9478 tree t
= d
, a
, dest
;
9480 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9481 if (itercnt
== NULL_TREE
)
9483 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9485 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9486 fold_convert (itype
, n1
),
9487 fold_convert (itype
, fd
->loop
.n1
));
9488 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9489 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9490 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9492 GSI_CONTINUE_LINKING
);
9497 tree type
= TREE_TYPE (t
);
9498 if (POINTER_TYPE_P (type
))
9500 a
= fold_build2 (MULT_EXPR
, type
,
9501 fold_convert (type
, itercnt
),
9502 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9503 dest
= unshare_expr (t
);
9504 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9505 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9506 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9507 false, GSI_CONTINUE_LINKING
);
9508 assign_stmt
= gimple_build_assign (dest
, t
);
9509 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9511 if (fd
->collapse
> 1)
9512 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9516 /* The code controlling the sequential loop replaces the
9517 GIMPLE_OMP_CONTINUE. */
9518 gsi
= gsi_last_bb (cont_bb
);
9519 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9520 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9521 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9522 vback
= gimple_omp_continue_control_def (cont_stmt
);
9524 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9526 if (POINTER_TYPE_P (type
))
9527 t
= fold_build_pointer_plus (vmain
, step
);
9529 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9530 t
= force_gimple_operand_gsi (&gsi
, t
,
9532 && TREE_ADDRESSABLE (vback
),
9533 NULL_TREE
, true, GSI_SAME_STMT
);
9534 assign_stmt
= gimple_build_assign (vback
, t
);
9535 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9537 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9538 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9540 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9543 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9544 gsi_remove (&gsi
, true);
9546 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9547 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9550 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9551 gsi
= gsi_last_bb (exit_bb
);
9552 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9554 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9555 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9557 gsi_remove (&gsi
, true);
9559 /* Connect all the blocks. */
9560 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9561 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9562 ep
= find_edge (entry_bb
, second_bb
);
9563 ep
->flags
= EDGE_TRUE_VALUE
;
9564 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9565 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9566 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9570 ep
= find_edge (cont_bb
, body_bb
);
9573 ep
= BRANCH_EDGE (cont_bb
);
9574 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9576 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9581 else if (fd
->collapse
> 1)
9584 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9587 ep
->flags
= EDGE_TRUE_VALUE
;
9588 find_edge (cont_bb
, fin_bb
)->flags
9589 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9592 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9593 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9594 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9596 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9597 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9598 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9599 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9601 struct loop
*loop
= body_bb
->loop_father
;
9602 if (loop
!= entry_bb
->loop_father
)
9604 gcc_assert (loop
->header
== body_bb
);
9605 gcc_assert (broken_loop
9606 || loop
->latch
== region
->cont
9607 || single_pred (loop
->latch
) == region
->cont
);
9611 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9613 loop
= alloc_loop ();
9614 loop
->header
= body_bb
;
9615 if (collapse_bb
== NULL
)
9616 loop
->latch
= cont_bb
;
9617 add_loop (loop
, body_bb
->loop_father
);
9621 /* Return phi in E->DEST with ARG on edge E. */
9624 find_phi_with_arg_on_edge (tree arg
, edge e
)
9626 basic_block bb
= e
->dest
;
9628 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9632 gphi
*phi
= gpi
.phi ();
9633 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9640 /* A subroutine of expand_omp_for. Generate code for a parallel
9641 loop with static schedule and a specified chunk size. Given
9644 for (V = N1; V cond N2; V += STEP) BODY;
9646 where COND is "<" or ">", we generate pseudocode
9648 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9653 if ((__typeof (V)) -1 > 0 && cond is >)
9654 n = -(adj + N2 - N1) / -STEP;
9656 n = (adj + N2 - N1) / STEP;
9658 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9659 here so that V is defined
9660 if the loop is not entered
9662 s0 = (trip * nthreads + threadid) * CHUNK;
9663 e0 = min(s0 + CHUNK, n);
9664 if (s0 < n) goto L1; else goto L4;
9671 if (V cond e) goto L2; else goto L3;
9679 expand_omp_for_static_chunk (struct omp_region
*region
,
9680 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9682 tree n
, s0
, e0
, e
, t
;
9683 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9684 tree type
, itype
, vmain
, vback
, vextra
;
9685 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9686 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9687 gimple_stmt_iterator gsi
;
9689 bool broken_loop
= region
->cont
== NULL
;
9690 tree
*counts
= NULL
;
9693 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9694 if (POINTER_TYPE_P (type
))
9695 itype
= signed_type_for (type
);
9697 entry_bb
= region
->entry
;
9698 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9700 iter_part_bb
= se
->dest
;
9701 cont_bb
= region
->cont
;
9702 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9703 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9704 gcc_assert (broken_loop
9705 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9706 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9707 body_bb
= single_succ (seq_start_bb
);
9710 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9711 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9712 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9713 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9715 exit_bb
= region
->exit
;
9717 /* Trip and adjustment setup goes in ENTRY_BB. */
9718 gsi
= gsi_last_bb (entry_bb
);
9719 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9721 if (fd
->collapse
> 1)
9723 int first_zero_iter
= -1, dummy
= -1;
9724 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9726 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9727 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9728 fin_bb
, first_zero_iter
,
9729 dummy_bb
, dummy
, l2_dom_bb
);
9732 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9733 t
= integer_one_node
;
9735 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9736 fold_convert (type
, fd
->loop
.n1
),
9737 fold_convert (type
, fd
->loop
.n2
));
9738 if (fd
->collapse
== 1
9739 && TYPE_UNSIGNED (type
)
9740 && (t
== NULL_TREE
|| !integer_onep (t
)))
9742 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9743 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9744 true, GSI_SAME_STMT
);
9745 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9746 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9747 true, GSI_SAME_STMT
);
9748 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9749 NULL_TREE
, NULL_TREE
);
9750 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9751 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9752 expand_omp_regimplify_p
, NULL
, NULL
)
9753 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9754 expand_omp_regimplify_p
, NULL
, NULL
))
9756 gsi
= gsi_for_stmt (cond_stmt
);
9757 gimple_regimplify_operands (cond_stmt
, &gsi
);
9759 se
= split_block (entry_bb
, cond_stmt
);
9760 se
->flags
= EDGE_TRUE_VALUE
;
9761 entry_bb
= se
->dest
;
9762 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9763 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9764 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9765 if (gimple_in_ssa_p (cfun
))
9767 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9768 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9769 !gsi_end_p (gpi
); gsi_next (&gpi
))
9771 gphi
*phi
= gpi
.phi ();
9772 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9773 se
, UNKNOWN_LOCATION
);
9776 gsi
= gsi_last_bb (entry_bb
);
9779 switch (gimple_omp_for_kind (fd
->for_stmt
))
9781 case GF_OMP_FOR_KIND_FOR
:
9782 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9783 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9785 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9786 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9787 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9792 nthreads
= build_call_expr (nthreads
, 0);
9793 nthreads
= fold_convert (itype
, nthreads
);
9794 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9795 true, GSI_SAME_STMT
);
9796 threadid
= build_call_expr (threadid
, 0);
9797 threadid
= fold_convert (itype
, threadid
);
9798 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9799 true, GSI_SAME_STMT
);
9803 step
= fd
->loop
.step
;
9804 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9806 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9807 OMP_CLAUSE__LOOPTEMP_
);
9808 gcc_assert (innerc
);
9809 n1
= OMP_CLAUSE_DECL (innerc
);
9810 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9811 OMP_CLAUSE__LOOPTEMP_
);
9812 gcc_assert (innerc
);
9813 n2
= OMP_CLAUSE_DECL (innerc
);
9815 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9816 true, NULL_TREE
, true, GSI_SAME_STMT
);
9817 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9818 true, NULL_TREE
, true, GSI_SAME_STMT
);
9819 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9820 true, NULL_TREE
, true, GSI_SAME_STMT
);
9821 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9822 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9824 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9827 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9828 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9829 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9830 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9831 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9832 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9833 fold_build1 (NEGATE_EXPR
, itype
, t
),
9834 fold_build1 (NEGATE_EXPR
, itype
, step
));
9836 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9837 t
= fold_convert (itype
, t
);
9838 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9839 true, GSI_SAME_STMT
);
9841 trip_var
= create_tmp_reg (itype
, ".trip");
9842 if (gimple_in_ssa_p (cfun
))
9844 trip_init
= make_ssa_name (trip_var
);
9845 trip_main
= make_ssa_name (trip_var
);
9846 trip_back
= make_ssa_name (trip_var
);
9850 trip_init
= trip_var
;
9851 trip_main
= trip_var
;
9852 trip_back
= trip_var
;
9855 gassign
*assign_stmt
9856 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9857 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9859 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9860 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9861 if (POINTER_TYPE_P (type
))
9862 t
= fold_build_pointer_plus (n1
, t
);
9864 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9865 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9866 true, GSI_SAME_STMT
);
9868 /* Remove the GIMPLE_OMP_FOR. */
9869 gsi_remove (&gsi
, true);
9871 gimple_stmt_iterator gsif
= gsi
;
9873 /* Iteration space partitioning goes in ITER_PART_BB. */
9874 gsi
= gsi_last_bb (iter_part_bb
);
9876 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9877 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9878 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9879 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9880 false, GSI_CONTINUE_LINKING
);
9882 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9883 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9884 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9885 false, GSI_CONTINUE_LINKING
);
9887 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9888 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9890 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9891 gsi
= gsi_start_bb (seq_start_bb
);
9893 tree startvar
= fd
->loop
.v
;
9894 tree endvar
= NULL_TREE
;
9896 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9898 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9899 ? gimple_omp_parallel_clauses (inner_stmt
)
9900 : gimple_omp_for_clauses (inner_stmt
);
9901 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9902 gcc_assert (innerc
);
9903 startvar
= OMP_CLAUSE_DECL (innerc
);
9904 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9905 OMP_CLAUSE__LOOPTEMP_
);
9906 gcc_assert (innerc
);
9907 endvar
= OMP_CLAUSE_DECL (innerc
);
9908 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9909 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9912 for (i
= 1; i
< fd
->collapse
; i
++)
9914 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9915 OMP_CLAUSE__LOOPTEMP_
);
9916 gcc_assert (innerc
);
9918 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9919 OMP_CLAUSE__LOOPTEMP_
);
9922 /* If needed (distribute parallel for with lastprivate),
9923 propagate down the total number of iterations. */
9924 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9926 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9927 GSI_CONTINUE_LINKING
);
9928 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9929 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9934 t
= fold_convert (itype
, s0
);
9935 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9936 if (POINTER_TYPE_P (type
))
9937 t
= fold_build_pointer_plus (n1
, t
);
9939 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9940 t
= fold_convert (TREE_TYPE (startvar
), t
);
9941 t
= force_gimple_operand_gsi (&gsi
, t
,
9943 && TREE_ADDRESSABLE (startvar
),
9944 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9945 assign_stmt
= gimple_build_assign (startvar
, t
);
9946 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9948 t
= fold_convert (itype
, e0
);
9949 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9950 if (POINTER_TYPE_P (type
))
9951 t
= fold_build_pointer_plus (n1
, t
);
9953 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9954 t
= fold_convert (TREE_TYPE (startvar
), t
);
9955 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9956 false, GSI_CONTINUE_LINKING
);
9959 assign_stmt
= gimple_build_assign (endvar
, e
);
9960 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9961 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9962 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9964 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9965 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9967 /* Handle linear clause adjustments. */
9968 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9969 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9970 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9971 c
; c
= OMP_CLAUSE_CHAIN (c
))
9972 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9973 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9975 tree d
= OMP_CLAUSE_DECL (c
);
9976 bool is_ref
= is_reference (d
);
9977 tree t
= d
, a
, dest
;
9979 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9980 tree type
= TREE_TYPE (t
);
9981 if (POINTER_TYPE_P (type
))
9983 dest
= unshare_expr (t
);
9984 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9985 expand_omp_build_assign (&gsif
, v
, t
);
9986 if (itercnt
== NULL_TREE
)
9988 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9991 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9992 fold_convert (itype
, fd
->loop
.n1
));
9993 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9996 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9999 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
10000 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
10002 GSI_CONTINUE_LINKING
);
10007 a
= fold_build2 (MULT_EXPR
, type
,
10008 fold_convert (type
, itercnt
),
10009 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
10010 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
10011 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
10012 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10013 false, GSI_CONTINUE_LINKING
);
10014 assign_stmt
= gimple_build_assign (dest
, t
);
10015 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10017 if (fd
->collapse
> 1)
10018 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10022 /* The code controlling the sequential loop goes in CONT_BB,
10023 replacing the GIMPLE_OMP_CONTINUE. */
10024 gsi
= gsi_last_bb (cont_bb
);
10025 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10026 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10027 vback
= gimple_omp_continue_control_def (cont_stmt
);
10029 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10031 if (POINTER_TYPE_P (type
))
10032 t
= fold_build_pointer_plus (vmain
, step
);
10034 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10035 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
10036 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10037 true, GSI_SAME_STMT
);
10038 assign_stmt
= gimple_build_assign (vback
, t
);
10039 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10041 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
10042 t
= build2 (EQ_EXPR
, boolean_type_node
,
10043 build_int_cst (itype
, 0),
10044 build_int_cst (itype
, 1));
10046 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10047 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10049 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10052 /* Remove GIMPLE_OMP_CONTINUE. */
10053 gsi_remove (&gsi
, true);
10055 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10056 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10058 /* Trip update code goes into TRIP_UPDATE_BB. */
10059 gsi
= gsi_start_bb (trip_update_bb
);
10061 t
= build_int_cst (itype
, 1);
10062 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
10063 assign_stmt
= gimple_build_assign (trip_back
, t
);
10064 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10067 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10068 gsi
= gsi_last_bb (exit_bb
);
10069 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
10071 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
10072 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
10074 gsi_remove (&gsi
, true);
10076 /* Connect the new blocks. */
10077 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
10078 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
10082 se
= find_edge (cont_bb
, body_bb
);
10085 se
= BRANCH_EDGE (cont_bb
);
10086 gcc_assert (single_succ (se
->dest
) == body_bb
);
10088 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10093 else if (fd
->collapse
> 1)
10096 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10099 se
->flags
= EDGE_TRUE_VALUE
;
10100 find_edge (cont_bb
, trip_update_bb
)->flags
10101 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10103 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
10106 if (gimple_in_ssa_p (cfun
))
10114 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
10116 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10117 remove arguments of the phi nodes in fin_bb. We need to create
10118 appropriate phi nodes in iter_part_bb instead. */
10119 se
= find_edge (iter_part_bb
, fin_bb
);
10120 re
= single_succ_edge (trip_update_bb
);
10121 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
10122 ene
= single_succ_edge (entry_bb
);
10124 psi
= gsi_start_phis (fin_bb
);
10125 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
10126 gsi_next (&psi
), ++i
)
10129 source_location locus
;
10132 t
= gimple_phi_result (phi
);
10133 gcc_assert (t
== redirect_edge_var_map_result (vm
));
10135 if (!single_pred_p (fin_bb
))
10136 t
= copy_ssa_name (t
, phi
);
10138 nphi
= create_phi_node (t
, iter_part_bb
);
10140 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
10141 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
10143 /* A special case -- fd->loop.v is not yet computed in
10144 iter_part_bb, we need to use vextra instead. */
10145 if (t
== fd
->loop
.v
)
10147 add_phi_arg (nphi
, t
, ene
, locus
);
10148 locus
= redirect_edge_var_map_location (vm
);
10149 tree back_arg
= redirect_edge_var_map_def (vm
);
10150 add_phi_arg (nphi
, back_arg
, re
, locus
);
10151 edge ce
= find_edge (cont_bb
, body_bb
);
10154 ce
= BRANCH_EDGE (cont_bb
);
10155 gcc_assert (single_succ (ce
->dest
) == body_bb
);
10156 ce
= single_succ_edge (ce
->dest
);
10158 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
10159 gcc_assert (inner_loop_phi
!= NULL
);
10160 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
10161 find_edge (seq_start_bb
, body_bb
), locus
);
10163 if (!single_pred_p (fin_bb
))
10164 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
10166 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
10167 redirect_edge_var_map_clear (re
);
10168 if (single_pred_p (fin_bb
))
10171 psi
= gsi_start_phis (fin_bb
);
10172 if (gsi_end_p (psi
))
10174 remove_phi_node (&psi
, false);
10177 /* Make phi node for trip. */
10178 phi
= create_phi_node (trip_main
, iter_part_bb
);
10179 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
10181 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
10186 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
10187 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
10188 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
10189 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10190 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10191 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
10192 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
10193 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10194 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10198 struct loop
*loop
= body_bb
->loop_father
;
10199 struct loop
*trip_loop
= alloc_loop ();
10200 trip_loop
->header
= iter_part_bb
;
10201 trip_loop
->latch
= trip_update_bb
;
10202 add_loop (trip_loop
, iter_part_bb
->loop_father
);
10204 if (loop
!= entry_bb
->loop_father
)
10206 gcc_assert (loop
->header
== body_bb
);
10207 gcc_assert (loop
->latch
== region
->cont
10208 || single_pred (loop
->latch
) == region
->cont
);
10209 trip_loop
->inner
= loop
;
10213 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10215 loop
= alloc_loop ();
10216 loop
->header
= body_bb
;
10217 if (collapse_bb
== NULL
)
10218 loop
->latch
= cont_bb
;
10219 add_loop (loop
, trip_loop
);
10224 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10226 for (V = N1; V cond N2; V += STEP) BODY;
10228 where COND is "<" or ">" or "!=", we generate pseudocode
10230 for (ind_var = low; ind_var < high; ind_var++)
10232 V = n1 + (ind_var * STEP)
10237 In the above pseudocode, low and high are function parameters of the
10238 child function. In the function below, we are inserting a temp.
10239 variable that will be making a call to two OMP functions that will not be
10240 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10241 with _Cilk_for). These functions are replaced with low and high
10242 by the function that handles taskreg. */
10246 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10248 bool broken_loop
= region
->cont
== NULL
;
10249 basic_block entry_bb
= region
->entry
;
10250 basic_block cont_bb
= region
->cont
;
10252 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10253 gcc_assert (broken_loop
10254 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10255 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10256 basic_block l1_bb
, l2_bb
;
10260 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10261 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10262 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10263 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10267 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10268 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10269 l2_bb
= single_succ (l1_bb
);
10271 basic_block exit_bb
= region
->exit
;
10272 basic_block l2_dom_bb
= NULL
;
10274 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10276 /* Below statements until the "tree high_val = ..." are pseudo statements
10277 used to pass information to be used by expand_omp_taskreg.
10278 low_val and high_val will be replaced by the __low and __high
10279 parameter from the child function.
10281 The call_exprs part is a place-holder, it is mainly used
10282 to distinctly identify to the top-level part that this is
10283 where we should put low and high (reasoning given in header
10287 = gimple_omp_parallel_child_fn (
10288 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10289 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10290 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10292 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10294 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10297 gcc_assert (low_val
&& high_val
);
10299 tree type
= TREE_TYPE (low_val
);
10300 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10301 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10303 /* Not needed in SSA form right now. */
10304 gcc_assert (!gimple_in_ssa_p (cfun
));
10305 if (l2_dom_bb
== NULL
)
10309 tree n2
= high_val
;
10311 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10313 /* Replace the GIMPLE_OMP_FOR statement. */
10314 gsi_replace (&gsi
, stmt
, true);
10318 /* Code to control the increment goes in the CONT_BB. */
10319 gsi
= gsi_last_bb (cont_bb
);
10320 stmt
= gsi_stmt (gsi
);
10321 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10322 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10323 build_one_cst (type
));
10325 /* Replace GIMPLE_OMP_CONTINUE. */
10326 gsi_replace (&gsi
, stmt
, true);
10329 /* Emit the condition in L1_BB. */
10330 gsi
= gsi_after_labels (l1_bb
);
10331 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10332 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10334 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10335 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10336 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10338 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10339 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10340 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10341 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10343 /* The condition is always '<' since the runtime will fill in the low
10344 and high values. */
10345 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10346 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10348 /* Remove GIMPLE_OMP_RETURN. */
10349 gsi
= gsi_last_bb (exit_bb
);
10350 gsi_remove (&gsi
, true);
10352 /* Connect the new blocks. */
10353 remove_edge (FALLTHRU_EDGE (entry_bb
));
10358 remove_edge (BRANCH_EDGE (entry_bb
));
10359 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10361 e
= BRANCH_EDGE (l1_bb
);
10362 ne
= FALLTHRU_EDGE (l1_bb
);
10363 e
->flags
= EDGE_TRUE_VALUE
;
10367 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10369 ne
= single_succ_edge (l1_bb
);
10370 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10373 ne
->flags
= EDGE_FALSE_VALUE
;
10374 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10375 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10377 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10378 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10379 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10383 struct loop
*loop
= alloc_loop ();
10384 loop
->header
= l1_bb
;
10385 loop
->latch
= cont_bb
;
10386 add_loop (loop
, l1_bb
->loop_father
);
10387 loop
->safelen
= INT_MAX
;
10390 /* Pick the correct library function based on the precision of the
10391 induction variable type. */
10392 tree lib_fun
= NULL_TREE
;
10393 if (TYPE_PRECISION (type
) == 32)
10394 lib_fun
= cilk_for_32_fndecl
;
10395 else if (TYPE_PRECISION (type
) == 64)
10396 lib_fun
= cilk_for_64_fndecl
;
10398 gcc_unreachable ();
10400 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10402 /* WS_ARGS contains the library function flavor to call:
10403 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10404 user-defined grain value. If the user does not define one, then zero
10405 is passed in by the parser. */
10406 vec_alloc (region
->ws_args
, 2);
10407 region
->ws_args
->quick_push (lib_fun
);
10408 region
->ws_args
->quick_push (fd
->chunk_size
);
10411 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10412 loop. Given parameters:
10414 for (V = N1; V cond N2; V += STEP) BODY;
10416 where COND is "<" or ">", we generate pseudocode
10424 if (V cond N2) goto L0; else goto L2;
10427 For collapsed loops, given parameters:
10429 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10430 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10431 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10434 we generate pseudocode
10440 count3 = (adj + N32 - N31) / STEP3;
10445 count2 = (adj + N22 - N21) / STEP2;
10450 count1 = (adj + N12 - N11) / STEP1;
10451 count = count1 * count2 * count3;
10461 V2 += (V3 cond3 N32) ? 0 : STEP2;
10462 V3 = (V3 cond3 N32) ? V3 : N31;
10463 V1 += (V2 cond2 N22) ? 0 : STEP1;
10464 V2 = (V2 cond2 N22) ? V2 : N21;
10466 if (V < count) goto L0; else goto L2;
10472 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10475 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10476 gimple_stmt_iterator gsi
;
10479 bool broken_loop
= region
->cont
== NULL
;
10481 tree
*counts
= NULL
;
10483 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10484 OMP_CLAUSE_SAFELEN
);
10485 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10486 OMP_CLAUSE__SIMDUID_
);
10489 type
= TREE_TYPE (fd
->loop
.v
);
10490 entry_bb
= region
->entry
;
10491 cont_bb
= region
->cont
;
10492 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10493 gcc_assert (broken_loop
10494 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10495 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10498 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10499 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10500 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10501 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10505 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10506 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10507 l2_bb
= single_succ (l1_bb
);
10509 exit_bb
= region
->exit
;
10512 gsi
= gsi_last_bb (entry_bb
);
10514 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10515 /* Not needed in SSA form right now. */
10516 gcc_assert (!gimple_in_ssa_p (cfun
));
10517 if (fd
->collapse
> 1)
10519 int first_zero_iter
= -1, dummy
= -1;
10520 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10522 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10523 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10524 zero_iter_bb
, first_zero_iter
,
10525 dummy_bb
, dummy
, l2_dom_bb
);
10527 if (l2_dom_bb
== NULL
)
10532 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10534 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10535 OMP_CLAUSE__LOOPTEMP_
);
10536 gcc_assert (innerc
);
10537 n1
= OMP_CLAUSE_DECL (innerc
);
10538 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10539 OMP_CLAUSE__LOOPTEMP_
);
10540 gcc_assert (innerc
);
10541 n2
= OMP_CLAUSE_DECL (innerc
);
10542 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10543 fold_convert (type
, n1
));
10544 if (fd
->collapse
> 1)
10547 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10553 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10554 fold_convert (type
, fd
->loop
.n1
));
10555 if (fd
->collapse
> 1)
10556 for (i
= 0; i
< fd
->collapse
; i
++)
10558 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10559 if (POINTER_TYPE_P (itype
))
10560 itype
= signed_type_for (itype
);
10561 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10562 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10566 /* Remove the GIMPLE_OMP_FOR statement. */
10567 gsi_remove (&gsi
, true);
10571 /* Code to control the increment goes in the CONT_BB. */
10572 gsi
= gsi_last_bb (cont_bb
);
10573 stmt
= gsi_stmt (gsi
);
10574 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10576 if (POINTER_TYPE_P (type
))
10577 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10579 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10580 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10582 if (fd
->collapse
> 1)
10584 i
= fd
->collapse
- 1;
10585 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10587 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10588 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10592 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10593 fd
->loops
[i
].step
);
10594 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10595 fd
->loops
[i
].v
, t
);
10597 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10599 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10601 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10602 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10603 if (POINTER_TYPE_P (itype2
))
10604 itype2
= signed_type_for (itype2
);
10605 t
= build3 (COND_EXPR
, itype2
,
10606 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10608 fold_convert (itype
, fd
->loops
[i
].n2
)),
10609 build_int_cst (itype2
, 0),
10610 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10611 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10612 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10614 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10615 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10617 t
= build3 (COND_EXPR
, itype
,
10618 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10620 fold_convert (itype
, fd
->loops
[i
].n2
)),
10622 fold_convert (itype
, fd
->loops
[i
].n1
));
10623 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10627 /* Remove GIMPLE_OMP_CONTINUE. */
10628 gsi_remove (&gsi
, true);
10631 /* Emit the condition in L1_BB. */
10632 gsi
= gsi_start_bb (l1_bb
);
10634 t
= fold_convert (type
, n2
);
10635 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10636 false, GSI_CONTINUE_LINKING
);
10637 tree v
= fd
->loop
.v
;
10638 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10639 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10640 false, GSI_CONTINUE_LINKING
);
10641 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10642 cond_stmt
= gimple_build_cond_empty (t
);
10643 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10644 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10646 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10649 gsi
= gsi_for_stmt (cond_stmt
);
10650 gimple_regimplify_operands (cond_stmt
, &gsi
);
10653 /* Remove GIMPLE_OMP_RETURN. */
10654 gsi
= gsi_last_bb (exit_bb
);
10655 gsi_remove (&gsi
, true);
10657 /* Connect the new blocks. */
10658 remove_edge (FALLTHRU_EDGE (entry_bb
));
10662 remove_edge (BRANCH_EDGE (entry_bb
));
10663 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10665 e
= BRANCH_EDGE (l1_bb
);
10666 ne
= FALLTHRU_EDGE (l1_bb
);
10667 e
->flags
= EDGE_TRUE_VALUE
;
10671 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10673 ne
= single_succ_edge (l1_bb
);
10674 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10677 ne
->flags
= EDGE_FALSE_VALUE
;
10678 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10679 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10681 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10682 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10683 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10687 struct loop
*loop
= alloc_loop ();
10688 loop
->header
= l1_bb
;
10689 loop
->latch
= cont_bb
;
10690 add_loop (loop
, l1_bb
->loop_father
);
10691 if (safelen
== NULL_TREE
)
10692 loop
->safelen
= INT_MAX
;
10695 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10696 if (TREE_CODE (safelen
) != INTEGER_CST
)
10698 else if (!tree_fits_uhwi_p (safelen
)
10699 || tree_to_uhwi (safelen
) > INT_MAX
)
10700 loop
->safelen
= INT_MAX
;
10702 loop
->safelen
= tree_to_uhwi (safelen
);
10703 if (loop
->safelen
== 1)
10708 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10709 cfun
->has_simduid_loops
= true;
10711 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10713 if ((flag_tree_loop_vectorize
10714 || (!global_options_set
.x_flag_tree_loop_vectorize
10715 && !global_options_set
.x_flag_tree_vectorize
))
10716 && flag_tree_loop_optimize
10717 && loop
->safelen
> 1)
10719 loop
->force_vectorize
= true;
10720 cfun
->has_force_vectorize_loops
= true;
10724 cfun
->has_simduid_loops
= true;
10727 /* Taskloop construct is represented after gimplification with
10728 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10729 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10730 which should just compute all the needed loop temporaries
10731 for GIMPLE_OMP_TASK. */
10734 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10735 struct omp_for_data
*fd
,
10736 gimple
*inner_stmt
)
10738 tree type
, bias
= NULL_TREE
;
10739 basic_block entry_bb
, cont_bb
, exit_bb
;
10740 gimple_stmt_iterator gsi
;
10741 gassign
*assign_stmt
;
10742 tree
*counts
= NULL
;
10745 gcc_assert (inner_stmt
);
10746 gcc_assert (region
->cont
);
10747 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10748 && gimple_omp_task_taskloop_p (inner_stmt
));
10749 type
= TREE_TYPE (fd
->loop
.v
);
10751 /* See if we need to bias by LLONG_MIN. */
10752 if (fd
->iter_type
== long_long_unsigned_type_node
10753 && TREE_CODE (type
) == INTEGER_TYPE
10754 && !TYPE_UNSIGNED (type
))
10758 if (fd
->loop
.cond_code
== LT_EXPR
)
10761 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10765 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10768 if (TREE_CODE (n1
) != INTEGER_CST
10769 || TREE_CODE (n2
) != INTEGER_CST
10770 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10771 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10774 entry_bb
= region
->entry
;
10775 cont_bb
= region
->cont
;
10776 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10777 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10778 exit_bb
= region
->exit
;
10780 gsi
= gsi_last_bb (entry_bb
);
10781 gimple
*for_stmt
= gsi_stmt (gsi
);
10782 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10783 if (fd
->collapse
> 1)
10785 int first_zero_iter
= -1, dummy
= -1;
10786 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10788 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10789 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10790 zero_iter_bb
, first_zero_iter
,
10791 dummy_bb
, dummy
, l2_dom_bb
);
10795 /* Some counts[i] vars might be uninitialized if
10796 some loop has zero iterations. But the body shouldn't
10797 be executed in that case, so just avoid uninit warnings. */
10798 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10799 if (SSA_VAR_P (counts
[i
]))
10800 TREE_NO_WARNING (counts
[i
]) = 1;
10802 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10803 entry_bb
= e
->dest
;
10804 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10805 gsi
= gsi_last_bb (entry_bb
);
10806 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10807 get_immediate_dominator (CDI_DOMINATORS
,
10815 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10816 && TYPE_PRECISION (TREE_TYPE (t0
))
10817 != TYPE_PRECISION (fd
->iter_type
))
10819 /* Avoid casting pointers to integer of a different size. */
10820 tree itype
= signed_type_for (type
);
10821 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10822 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10826 t1
= fold_convert (fd
->iter_type
, t1
);
10827 t0
= fold_convert (fd
->iter_type
, t0
);
10831 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10832 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10835 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10836 OMP_CLAUSE__LOOPTEMP_
);
10837 gcc_assert (innerc
);
10838 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10839 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10840 gcc_assert (innerc
);
10841 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10842 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10844 gcc_assert (innerc
);
10845 for (i
= 1; i
< fd
->collapse
; i
++)
10847 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10848 OMP_CLAUSE__LOOPTEMP_
);
10849 gcc_assert (innerc
);
10851 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10852 OMP_CLAUSE__LOOPTEMP_
);
10855 /* If needed (inner taskloop has lastprivate clause), propagate
10856 down the total number of iterations. */
10857 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10859 GSI_CONTINUE_LINKING
);
10860 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10861 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10865 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10866 GSI_CONTINUE_LINKING
);
10867 assign_stmt
= gimple_build_assign (startvar
, t0
);
10868 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10870 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10871 GSI_CONTINUE_LINKING
);
10872 assign_stmt
= gimple_build_assign (endvar
, t1
);
10873 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10874 if (fd
->collapse
> 1)
10875 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10877 /* Remove the GIMPLE_OMP_FOR statement. */
10878 gsi
= gsi_for_stmt (for_stmt
);
10879 gsi_remove (&gsi
, true);
10881 gsi
= gsi_last_bb (cont_bb
);
10882 gsi_remove (&gsi
, true);
10884 gsi
= gsi_last_bb (exit_bb
);
10885 gsi_remove (&gsi
, true);
10887 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10888 remove_edge (BRANCH_EDGE (entry_bb
));
10889 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10890 remove_edge (BRANCH_EDGE (cont_bb
));
10891 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10892 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10893 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10896 /* Taskloop construct is represented after gimplification with
10897 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10898 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10899 GOMP_taskloop{,_ull} function arranges for each task to be given just
10900 a single range of iterations. */
10903 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10904 struct omp_for_data
*fd
,
10905 gimple
*inner_stmt
)
10907 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10908 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10909 basic_block fin_bb
;
10910 gimple_stmt_iterator gsi
;
10912 bool broken_loop
= region
->cont
== NULL
;
10913 tree
*counts
= NULL
;
10916 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10917 if (POINTER_TYPE_P (type
))
10918 itype
= signed_type_for (type
);
10920 /* See if we need to bias by LLONG_MIN. */
10921 if (fd
->iter_type
== long_long_unsigned_type_node
10922 && TREE_CODE (type
) == INTEGER_TYPE
10923 && !TYPE_UNSIGNED (type
))
10927 if (fd
->loop
.cond_code
== LT_EXPR
)
10930 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10934 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10937 if (TREE_CODE (n1
) != INTEGER_CST
10938 || TREE_CODE (n2
) != INTEGER_CST
10939 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10940 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10943 entry_bb
= region
->entry
;
10944 cont_bb
= region
->cont
;
10945 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10946 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10947 gcc_assert (broken_loop
10948 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10949 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10952 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10953 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10955 exit_bb
= region
->exit
;
10957 /* Iteration space partitioning goes in ENTRY_BB. */
10958 gsi
= gsi_last_bb (entry_bb
);
10959 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10961 if (fd
->collapse
> 1)
10963 int first_zero_iter
= -1, dummy
= -1;
10964 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10966 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10967 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10968 fin_bb
, first_zero_iter
,
10969 dummy_bb
, dummy
, l2_dom_bb
);
10973 t
= integer_one_node
;
10975 step
= fd
->loop
.step
;
10976 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10977 OMP_CLAUSE__LOOPTEMP_
);
10978 gcc_assert (innerc
);
10979 n1
= OMP_CLAUSE_DECL (innerc
);
10980 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10981 gcc_assert (innerc
);
10982 n2
= OMP_CLAUSE_DECL (innerc
);
10985 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10986 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10988 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10989 true, NULL_TREE
, true, GSI_SAME_STMT
);
10990 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10991 true, NULL_TREE
, true, GSI_SAME_STMT
);
10992 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10993 true, NULL_TREE
, true, GSI_SAME_STMT
);
10995 tree startvar
= fd
->loop
.v
;
10996 tree endvar
= NULL_TREE
;
10998 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11000 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
11001 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
11002 gcc_assert (innerc
);
11003 startvar
= OMP_CLAUSE_DECL (innerc
);
11004 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
11005 OMP_CLAUSE__LOOPTEMP_
);
11006 gcc_assert (innerc
);
11007 endvar
= OMP_CLAUSE_DECL (innerc
);
11009 t
= fold_convert (TREE_TYPE (startvar
), n1
);
11010 t
= force_gimple_operand_gsi (&gsi
, t
,
11012 && TREE_ADDRESSABLE (startvar
),
11013 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
11014 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
11015 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11017 t
= fold_convert (TREE_TYPE (startvar
), n2
);
11018 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
11019 false, GSI_CONTINUE_LINKING
);
11022 assign_stmt
= gimple_build_assign (endvar
, e
);
11023 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11024 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
11025 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
11027 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
11028 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
11030 if (fd
->collapse
> 1)
11031 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
11035 /* The code controlling the sequential loop replaces the
11036 GIMPLE_OMP_CONTINUE. */
11037 gsi
= gsi_last_bb (cont_bb
);
11038 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11039 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
11040 vmain
= gimple_omp_continue_control_use (cont_stmt
);
11041 vback
= gimple_omp_continue_control_def (cont_stmt
);
11043 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
11045 if (POINTER_TYPE_P (type
))
11046 t
= fold_build_pointer_plus (vmain
, step
);
11048 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
11049 t
= force_gimple_operand_gsi (&gsi
, t
,
11051 && TREE_ADDRESSABLE (vback
),
11052 NULL_TREE
, true, GSI_SAME_STMT
);
11053 assign_stmt
= gimple_build_assign (vback
, t
);
11054 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
11056 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
11057 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
11059 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
11062 /* Remove the GIMPLE_OMP_CONTINUE statement. */
11063 gsi_remove (&gsi
, true);
11065 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
11066 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
11069 /* Remove the GIMPLE_OMP_FOR statement. */
11070 gsi
= gsi_for_stmt (fd
->for_stmt
);
11071 gsi_remove (&gsi
, true);
11073 /* Remove the GIMPLE_OMP_RETURN statement. */
11074 gsi
= gsi_last_bb (exit_bb
);
11075 gsi_remove (&gsi
, true);
11077 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
11079 remove_edge (BRANCH_EDGE (entry_bb
));
11082 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
11083 region
->outer
->cont
= NULL
;
11086 /* Connect all the blocks. */
11089 ep
= find_edge (cont_bb
, body_bb
);
11090 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11095 else if (fd
->collapse
> 1)
11098 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
11101 ep
->flags
= EDGE_TRUE_VALUE
;
11102 find_edge (cont_bb
, fin_bb
)->flags
11103 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
11106 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
11107 recompute_dominator (CDI_DOMINATORS
, body_bb
));
11109 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
11110 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
11112 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
11114 struct loop
*loop
= alloc_loop ();
11115 loop
->header
= body_bb
;
11116 if (collapse_bb
== NULL
)
11117 loop
->latch
= cont_bb
;
11118 add_loop (loop
, body_bb
->loop_father
);
11122 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11123 partitioned loop. The lowering here is abstracted, in that the
11124 loop parameters are passed through internal functions, which are
11125 further lowered by oacc_device_lower, once we get to the target
11126 compiler. The loop is of the form:
11128 for (V = B; V LTGT E; V += S) {BODY}
11130 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11131 (constant 0 for no chunking) and we will have a GWV partitioning
11132 mask, specifying dimensions over which the loop is to be
11133 partitioned (see note below). We generate code that looks like:
11135 <entry_bb> [incoming FALL->body, BRANCH->exit]
11136 typedef signedintify (typeof (V)) T; // underlying signed integral type
11139 T DIR = LTGT == '<' ? +1 : -1;
11140 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11141 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11143 <head_bb> [created by splitting end of entry_bb]
11144 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11145 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11146 if (!(offset LTGT bound)) goto bottom_bb;
11148 <body_bb> [incoming]
11152 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11154 if (offset LTGT bound) goto body_bb; [*]
11156 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11158 if (chunk < chunk_max) goto head_bb;
11160 <exit_bb> [incoming]
11161 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11163 [*] Needed if V live at end of loop
11165 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11166 transition, and will be specified by a more general mechanism shortly.
11170 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
11172 tree v
= fd
->loop
.v
;
11173 enum tree_code cond_code
= fd
->loop
.cond_code
;
11174 enum tree_code plus_code
= PLUS_EXPR
;
11176 tree chunk_size
= integer_minus_one_node
;
11177 tree gwv
= integer_zero_node
;
11178 tree iter_type
= TREE_TYPE (v
);
11179 tree diff_type
= iter_type
;
11180 tree plus_type
= iter_type
;
11181 struct oacc_collapse
*counts
= NULL
;
11183 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
11184 == GF_OMP_FOR_KIND_OACC_LOOP
);
11185 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
11186 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
11188 if (POINTER_TYPE_P (iter_type
))
11190 plus_code
= POINTER_PLUS_EXPR
;
11191 plus_type
= sizetype
;
11193 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
11194 diff_type
= signed_type_for (diff_type
);
11196 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
11197 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
11198 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
11199 basic_block bottom_bb
= NULL
;
11201 /* entry_bb has two sucessors; the branch edge is to the exit
11202 block, fallthrough edge to body. */
11203 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
11204 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
11206 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11207 body_bb, or to a block whose only successor is the body_bb. Its
11208 fallthrough successor is the final block (same as the branch
11209 successor of the entry_bb). */
11212 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
11213 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
11215 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
11216 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
11219 gcc_assert (!gimple_in_ssa_p (cfun
));
11221 /* The exit block only has entry_bb and cont_bb as predecessors. */
11222 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
11225 tree chunk_max
= NULL_TREE
;
11226 tree bound
, offset
;
11227 tree step
= create_tmp_var (diff_type
, ".step");
11228 bool up
= cond_code
== LT_EXPR
;
11229 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
11230 bool chunking
= !gimple_in_ssa_p (cfun
);;
11233 /* SSA instances. */
11234 tree offset_incr
= NULL_TREE
;
11235 tree offset_init
= NULL_TREE
;
11237 gimple_stmt_iterator gsi
;
11243 edge split
, be
, fte
;
11245 /* Split the end of entry_bb to create head_bb. */
11246 split
= split_block (entry_bb
, last_stmt (entry_bb
));
11247 basic_block head_bb
= split
->dest
;
11248 entry_bb
= split
->src
;
11250 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11251 gsi
= gsi_last_bb (entry_bb
);
11252 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
11253 loc
= gimple_location (for_stmt
);
11255 if (gimple_in_ssa_p (cfun
))
11257 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11258 gcc_assert (integer_zerop (fd
->loop
.n1
));
11259 /* The SSA parallelizer does gang parallelism. */
11260 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11263 if (fd
->collapse
> 1)
11265 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11266 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11267 TREE_TYPE (fd
->loop
.n2
));
11269 if (SSA_VAR_P (fd
->loop
.n2
))
11271 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11272 true, GSI_SAME_STMT
);
11273 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11274 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11279 tree b
= fd
->loop
.n1
;
11280 tree e
= fd
->loop
.n2
;
11281 tree s
= fd
->loop
.step
;
11283 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11284 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11286 /* Convert the step, avoiding possible unsigned->signed overflow. */
11287 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11289 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11290 s
= fold_convert (diff_type
, s
);
11292 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11293 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11296 chunk_size
= integer_zero_node
;
11297 expr
= fold_convert (diff_type
, chunk_size
);
11298 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11299 NULL_TREE
, true, GSI_SAME_STMT
);
11300 /* Determine the range, avoiding possible unsigned->signed overflow. */
11301 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11302 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11303 fold_convert (plus_type
, negating
? b
: e
),
11304 fold_convert (plus_type
, negating
? e
: b
));
11305 expr
= fold_convert (diff_type
, expr
);
11307 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11308 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11309 NULL_TREE
, true, GSI_SAME_STMT
);
11311 chunk_no
= build_int_cst (diff_type
, 0);
11314 gcc_assert (!gimple_in_ssa_p (cfun
));
11317 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11318 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11320 ass
= gimple_build_assign (chunk_no
, expr
);
11321 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11323 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11324 build_int_cst (integer_type_node
,
11325 IFN_GOACC_LOOP_CHUNKS
),
11326 dir
, range
, s
, chunk_size
, gwv
);
11327 gimple_call_set_lhs (call
, chunk_max
);
11328 gimple_set_location (call
, loc
);
11329 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11332 chunk_size
= chunk_no
;
11334 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11335 build_int_cst (integer_type_node
,
11336 IFN_GOACC_LOOP_STEP
),
11337 dir
, range
, s
, chunk_size
, gwv
);
11338 gimple_call_set_lhs (call
, step
);
11339 gimple_set_location (call
, loc
);
11340 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11342 /* Remove the GIMPLE_OMP_FOR. */
11343 gsi_remove (&gsi
, true);
11345 /* Fixup edges from head_bb */
11346 be
= BRANCH_EDGE (head_bb
);
11347 fte
= FALLTHRU_EDGE (head_bb
);
11348 be
->flags
|= EDGE_FALSE_VALUE
;
11349 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11351 basic_block body_bb
= fte
->dest
;
11353 if (gimple_in_ssa_p (cfun
))
11355 gsi
= gsi_last_bb (cont_bb
);
11356 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11358 offset
= gimple_omp_continue_control_use (cont_stmt
);
11359 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11363 offset
= create_tmp_var (diff_type
, ".offset");
11364 offset_init
= offset_incr
= offset
;
11366 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11368 /* Loop offset & bound go into head_bb. */
11369 gsi
= gsi_start_bb (head_bb
);
11371 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11372 build_int_cst (integer_type_node
,
11373 IFN_GOACC_LOOP_OFFSET
),
11375 chunk_size
, gwv
, chunk_no
);
11376 gimple_call_set_lhs (call
, offset_init
);
11377 gimple_set_location (call
, loc
);
11378 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11380 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11381 build_int_cst (integer_type_node
,
11382 IFN_GOACC_LOOP_BOUND
),
11384 chunk_size
, gwv
, offset_init
);
11385 gimple_call_set_lhs (call
, bound
);
11386 gimple_set_location (call
, loc
);
11387 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11389 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11390 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11391 GSI_CONTINUE_LINKING
);
11393 /* V assignment goes into body_bb. */
11394 if (!gimple_in_ssa_p (cfun
))
11396 gsi
= gsi_start_bb (body_bb
);
11398 expr
= build2 (plus_code
, iter_type
, b
,
11399 fold_convert (plus_type
, offset
));
11400 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11401 true, GSI_SAME_STMT
);
11402 ass
= gimple_build_assign (v
, expr
);
11403 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11404 if (fd
->collapse
> 1)
11405 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11408 /* Loop increment goes into cont_bb. If this is not a loop, we
11409 will have spawned threads as if it was, and each one will
11410 execute one iteration. The specification is not explicit about
11411 whether such constructs are ill-formed or not, and they can
11412 occur, especially when noreturn routines are involved. */
11415 gsi
= gsi_last_bb (cont_bb
);
11416 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11417 loc
= gimple_location (cont_stmt
);
11419 /* Increment offset. */
11420 if (gimple_in_ssa_p (cfun
))
11421 expr
= build2 (plus_code
, iter_type
, offset
,
11422 fold_convert (plus_type
, step
));
11424 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11425 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11426 true, GSI_SAME_STMT
);
11427 ass
= gimple_build_assign (offset_incr
, expr
);
11428 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11429 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11430 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11432 /* Remove the GIMPLE_OMP_CONTINUE. */
11433 gsi_remove (&gsi
, true);
11435 /* Fixup edges from cont_bb */
11436 be
= BRANCH_EDGE (cont_bb
);
11437 fte
= FALLTHRU_EDGE (cont_bb
);
11438 be
->flags
|= EDGE_TRUE_VALUE
;
11439 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11443 /* Split the beginning of exit_bb to make bottom_bb. We
11444 need to insert a nop at the start, because splitting is
11445 after a stmt, not before. */
11446 gsi
= gsi_start_bb (exit_bb
);
11447 stmt
= gimple_build_nop ();
11448 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11449 split
= split_block (exit_bb
, stmt
);
11450 bottom_bb
= split
->src
;
11451 exit_bb
= split
->dest
;
11452 gsi
= gsi_last_bb (bottom_bb
);
11454 /* Chunk increment and test goes into bottom_bb. */
11455 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11456 build_int_cst (diff_type
, 1));
11457 ass
= gimple_build_assign (chunk_no
, expr
);
11458 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11460 /* Chunk test at end of bottom_bb. */
11461 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11462 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11463 GSI_CONTINUE_LINKING
);
11465 /* Fixup edges from bottom_bb. */
11466 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11467 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11471 gsi
= gsi_last_bb (exit_bb
);
11472 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11473 loc
= gimple_location (gsi_stmt (gsi
));
11475 if (!gimple_in_ssa_p (cfun
))
11477 /* Insert the final value of V, in case it is live. This is the
11478 value for the only thread that survives past the join. */
11479 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11480 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11481 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11482 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11483 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11484 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11485 true, GSI_SAME_STMT
);
11486 ass
= gimple_build_assign (v
, expr
);
11487 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11490 /* Remove the OMP_RETURN. */
11491 gsi_remove (&gsi
, true);
11495 /* We now have one or two nested loops. Update the loop
11497 struct loop
*parent
= entry_bb
->loop_father
;
11498 struct loop
*body
= body_bb
->loop_father
;
11502 struct loop
*chunk_loop
= alloc_loop ();
11503 chunk_loop
->header
= head_bb
;
11504 chunk_loop
->latch
= bottom_bb
;
11505 add_loop (chunk_loop
, parent
);
11506 parent
= chunk_loop
;
11508 else if (parent
!= body
)
11510 gcc_assert (body
->header
== body_bb
);
11511 gcc_assert (body
->latch
== cont_bb
11512 || single_pred (body
->latch
) == cont_bb
);
11518 struct loop
*body_loop
= alloc_loop ();
11519 body_loop
->header
= body_bb
;
11520 body_loop
->latch
= cont_bb
;
11521 add_loop (body_loop
, parent
);
11526 /* Expand the OMP loop defined by REGION. */
11529 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11531 struct omp_for_data fd
;
11532 struct omp_for_data_loop
*loops
;
11535 = (struct omp_for_data_loop
*)
11536 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11537 * sizeof (struct omp_for_data_loop
));
11538 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11540 region
->sched_kind
= fd
.sched_kind
;
11541 region
->sched_modifiers
= fd
.sched_modifiers
;
11543 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11544 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11545 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11548 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11549 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11550 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11553 /* If there isn't a continue then this is a degerate case where
11554 the introduction of abnormal edges during lowering will prevent
11555 original loops from being detected. Fix that up. */
11556 loops_state_set (LOOPS_NEED_FIXUP
);
11558 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11559 expand_omp_simd (region
, &fd
);
11560 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11561 expand_cilk_for (region
, &fd
);
11562 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11564 gcc_assert (!inner_stmt
);
11565 expand_oacc_for (region
, &fd
);
11567 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11569 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11570 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11572 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11574 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11575 && !fd
.have_ordered
)
11577 if (fd
.chunk_size
== NULL
)
11578 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11580 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11584 int fn_index
, start_ix
, next_ix
;
11586 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11587 == GF_OMP_FOR_KIND_FOR
);
11588 if (fd
.chunk_size
== NULL
11589 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11590 fd
.chunk_size
= integer_zero_node
;
11591 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11592 switch (fd
.sched_kind
)
11594 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11597 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11598 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11599 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11601 && !fd
.have_ordered
)
11603 fn_index
= 3 + fd
.sched_kind
;
11608 fn_index
= fd
.sched_kind
;
11612 fn_index
+= fd
.have_ordered
* 6;
11614 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11616 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11617 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11618 if (fd
.iter_type
== long_long_unsigned_type_node
)
11620 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11621 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11622 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11623 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11625 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11626 (enum built_in_function
) next_ix
, inner_stmt
);
11629 if (gimple_in_ssa_p (cfun
))
11630 update_ssa (TODO_update_ssa_only_virtuals
);
11634 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11636 v = GOMP_sections_start (n);
11653 v = GOMP_sections_next ();
11658 If this is a combined parallel sections, replace the call to
11659 GOMP_sections_start with call to GOMP_sections_next. */
11662 expand_omp_sections (struct omp_region
*region
)
11664 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11666 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11667 gimple_stmt_iterator si
, switch_si
;
11668 gomp_sections
*sections_stmt
;
11670 gomp_continue
*cont
;
11673 struct omp_region
*inner
;
11675 bool exit_reachable
= region
->cont
!= NULL
;
11677 gcc_assert (region
->exit
!= NULL
);
11678 entry_bb
= region
->entry
;
11679 l0_bb
= single_succ (entry_bb
);
11680 l1_bb
= region
->cont
;
11681 l2_bb
= region
->exit
;
11682 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11683 l2
= gimple_block_label (l2_bb
);
11686 /* This can happen if there are reductions. */
11687 len
= EDGE_COUNT (l0_bb
->succs
);
11688 gcc_assert (len
> 0);
11689 e
= EDGE_SUCC (l0_bb
, len
- 1);
11690 si
= gsi_last_bb (e
->dest
);
11693 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11694 l2
= gimple_block_label (e
->dest
);
11696 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11698 si
= gsi_last_bb (e
->dest
);
11700 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11702 l2
= gimple_block_label (e
->dest
);
11707 if (exit_reachable
)
11708 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11710 default_bb
= create_empty_bb (l0_bb
);
11712 /* We will build a switch() with enough cases for all the
11713 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11714 and a default case to abort if something goes wrong. */
11715 len
= EDGE_COUNT (l0_bb
->succs
);
11717 /* Use vec::quick_push on label_vec throughout, since we know the size
11719 auto_vec
<tree
> label_vec (len
);
11721 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11722 GIMPLE_OMP_SECTIONS statement. */
11723 si
= gsi_last_bb (entry_bb
);
11724 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11725 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11726 vin
= gimple_omp_sections_control (sections_stmt
);
11727 if (!is_combined_parallel (region
))
11729 /* If we are not inside a combined parallel+sections region,
11730 call GOMP_sections_start. */
11731 t
= build_int_cst (unsigned_type_node
, len
- 1);
11732 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11733 stmt
= gimple_build_call (u
, 1, t
);
11737 /* Otherwise, call GOMP_sections_next. */
11738 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11739 stmt
= gimple_build_call (u
, 0);
11741 gimple_call_set_lhs (stmt
, vin
);
11742 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11743 gsi_remove (&si
, true);
11745 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11747 switch_si
= gsi_last_bb (l0_bb
);
11748 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11749 if (exit_reachable
)
11751 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11752 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11753 vmain
= gimple_omp_continue_control_use (cont
);
11754 vnext
= gimple_omp_continue_control_def (cont
);
11762 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11763 label_vec
.quick_push (t
);
11766 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11767 for (inner
= region
->inner
, casei
= 1;
11769 inner
= inner
->next
, i
++, casei
++)
11771 basic_block s_entry_bb
, s_exit_bb
;
11773 /* Skip optional reduction region. */
11774 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11781 s_entry_bb
= inner
->entry
;
11782 s_exit_bb
= inner
->exit
;
11784 t
= gimple_block_label (s_entry_bb
);
11785 u
= build_int_cst (unsigned_type_node
, casei
);
11786 u
= build_case_label (u
, NULL
, t
);
11787 label_vec
.quick_push (u
);
11789 si
= gsi_last_bb (s_entry_bb
);
11790 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11791 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11792 gsi_remove (&si
, true);
11793 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11795 if (s_exit_bb
== NULL
)
11798 si
= gsi_last_bb (s_exit_bb
);
11799 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11800 gsi_remove (&si
, true);
11802 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11805 /* Error handling code goes in DEFAULT_BB. */
11806 t
= gimple_block_label (default_bb
);
11807 u
= build_case_label (NULL
, NULL
, t
);
11808 make_edge (l0_bb
, default_bb
, 0);
11809 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11811 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11812 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11813 gsi_remove (&switch_si
, true);
11815 si
= gsi_start_bb (default_bb
);
11816 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11817 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11819 if (exit_reachable
)
11823 /* Code to get the next section goes in L1_BB. */
11824 si
= gsi_last_bb (l1_bb
);
11825 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11827 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11828 stmt
= gimple_build_call (bfn_decl
, 0);
11829 gimple_call_set_lhs (stmt
, vnext
);
11830 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11831 gsi_remove (&si
, true);
11833 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11836 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11837 si
= gsi_last_bb (l2_bb
);
11838 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11839 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11840 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11841 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11843 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11844 stmt
= gimple_build_call (t
, 0);
11845 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11846 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11847 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11848 gsi_remove (&si
, true);
11850 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11854 /* Expand code for an OpenMP single directive. We've already expanded
11855 much of the code, here we simply place the GOMP_barrier call. */
11858 expand_omp_single (struct omp_region
*region
)
11860 basic_block entry_bb
, exit_bb
;
11861 gimple_stmt_iterator si
;
11863 entry_bb
= region
->entry
;
11864 exit_bb
= region
->exit
;
11866 si
= gsi_last_bb (entry_bb
);
11867 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11868 gsi_remove (&si
, true);
11869 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11871 si
= gsi_last_bb (exit_bb
);
11872 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11874 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11875 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11877 gsi_remove (&si
, true);
11878 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11882 /* Generic expansion for OpenMP synchronization directives: master,
11883 ordered and critical. All we need to do here is remove the entry
11884 and exit markers for REGION. */
11887 expand_omp_synch (struct omp_region
*region
)
11889 basic_block entry_bb
, exit_bb
;
11890 gimple_stmt_iterator si
;
11892 entry_bb
= region
->entry
;
11893 exit_bb
= region
->exit
;
11895 si
= gsi_last_bb (entry_bb
);
11896 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11897 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11898 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11899 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11900 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11901 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11902 gsi_remove (&si
, true);
11903 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11907 si
= gsi_last_bb (exit_bb
);
11908 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11909 gsi_remove (&si
, true);
11910 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11914 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11915 operation as a normal volatile load. */
11918 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11919 tree loaded_val
, int index
)
11921 enum built_in_function tmpbase
;
11922 gimple_stmt_iterator gsi
;
11923 basic_block store_bb
;
11926 tree decl
, call
, type
, itype
;
11928 gsi
= gsi_last_bb (load_bb
);
11929 stmt
= gsi_stmt (gsi
);
11930 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11931 loc
= gimple_location (stmt
);
11933 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11934 is smaller than word size, then expand_atomic_load assumes that the load
11935 is atomic. We could avoid the builtin entirely in this case. */
11937 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11938 decl
= builtin_decl_explicit (tmpbase
);
11939 if (decl
== NULL_TREE
)
11942 type
= TREE_TYPE (loaded_val
);
11943 itype
= TREE_TYPE (TREE_TYPE (decl
));
11945 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11946 build_int_cst (NULL
,
11947 gimple_omp_atomic_seq_cst_p (stmt
)
11949 : MEMMODEL_RELAXED
));
11950 if (!useless_type_conversion_p (type
, itype
))
11951 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11952 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11954 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11955 gsi_remove (&gsi
, true);
11957 store_bb
= single_succ (load_bb
);
11958 gsi
= gsi_last_bb (store_bb
);
11959 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11960 gsi_remove (&gsi
, true);
11962 if (gimple_in_ssa_p (cfun
))
11963 update_ssa (TODO_update_ssa_no_phi
);
11968 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11969 operation as a normal volatile store. */
11972 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11973 tree loaded_val
, tree stored_val
, int index
)
11975 enum built_in_function tmpbase
;
11976 gimple_stmt_iterator gsi
;
11977 basic_block store_bb
= single_succ (load_bb
);
11980 tree decl
, call
, type
, itype
;
11981 machine_mode imode
;
11984 gsi
= gsi_last_bb (load_bb
);
11985 stmt
= gsi_stmt (gsi
);
11986 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11988 /* If the load value is needed, then this isn't a store but an exchange. */
11989 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11991 gsi
= gsi_last_bb (store_bb
);
11992 stmt
= gsi_stmt (gsi
);
11993 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11994 loc
= gimple_location (stmt
);
11996 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11997 is smaller than word size, then expand_atomic_store assumes that the store
11998 is atomic. We could avoid the builtin entirely in this case. */
12000 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
12001 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
12002 decl
= builtin_decl_explicit (tmpbase
);
12003 if (decl
== NULL_TREE
)
12006 type
= TREE_TYPE (stored_val
);
12008 /* Dig out the type of the function's second argument. */
12009 itype
= TREE_TYPE (decl
);
12010 itype
= TYPE_ARG_TYPES (itype
);
12011 itype
= TREE_CHAIN (itype
);
12012 itype
= TREE_VALUE (itype
);
12013 imode
= TYPE_MODE (itype
);
12015 if (exchange
&& !can_atomic_exchange_p (imode
, true))
12018 if (!useless_type_conversion_p (itype
, type
))
12019 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
12020 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
12021 build_int_cst (NULL
,
12022 gimple_omp_atomic_seq_cst_p (stmt
)
12024 : MEMMODEL_RELAXED
));
12027 if (!useless_type_conversion_p (type
, itype
))
12028 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
12029 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
12032 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12033 gsi_remove (&gsi
, true);
12035 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
12036 gsi
= gsi_last_bb (load_bb
);
12037 gsi_remove (&gsi
, true);
12039 if (gimple_in_ssa_p (cfun
))
12040 update_ssa (TODO_update_ssa_no_phi
);
12045 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
12046 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
12047 size of the data type, and thus usable to find the index of the builtin
12048 decl. Returns false if the expression is not of the proper form. */
12051 expand_omp_atomic_fetch_op (basic_block load_bb
,
12052 tree addr
, tree loaded_val
,
12053 tree stored_val
, int index
)
12055 enum built_in_function oldbase
, newbase
, tmpbase
;
12056 tree decl
, itype
, call
;
12058 basic_block store_bb
= single_succ (load_bb
);
12059 gimple_stmt_iterator gsi
;
12062 enum tree_code code
;
12063 bool need_old
, need_new
;
12064 machine_mode imode
;
12067 /* We expect to find the following sequences:
12070 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12073 val = tmp OP something; (or: something OP tmp)
12074 GIMPLE_OMP_STORE (val)
12076 ???FIXME: Allow a more flexible sequence.
12077 Perhaps use data flow to pick the statements.
12081 gsi
= gsi_after_labels (store_bb
);
12082 stmt
= gsi_stmt (gsi
);
12083 loc
= gimple_location (stmt
);
12084 if (!is_gimple_assign (stmt
))
12087 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
12089 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
12090 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
12091 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
12092 gcc_checking_assert (!need_old
|| !need_new
);
12094 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
12097 /* Check for one of the supported fetch-op operations. */
12098 code
= gimple_assign_rhs_code (stmt
);
12102 case POINTER_PLUS_EXPR
:
12103 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
12104 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
12107 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
12108 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
12111 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
12112 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
12115 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
12116 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
12119 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
12120 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
12126 /* Make sure the expression is of the proper form. */
12127 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
12128 rhs
= gimple_assign_rhs2 (stmt
);
12129 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
12130 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
12131 rhs
= gimple_assign_rhs1 (stmt
);
12135 tmpbase
= ((enum built_in_function
)
12136 ((need_new
? newbase
: oldbase
) + index
+ 1));
12137 decl
= builtin_decl_explicit (tmpbase
);
12138 if (decl
== NULL_TREE
)
12140 itype
= TREE_TYPE (TREE_TYPE (decl
));
12141 imode
= TYPE_MODE (itype
);
12143 /* We could test all of the various optabs involved, but the fact of the
12144 matter is that (with the exception of i486 vs i586 and xadd) all targets
12145 that support any atomic operaton optab also implements compare-and-swap.
12146 Let optabs.c take care of expanding any compare-and-swap loop. */
12147 if (!can_compare_and_swap_p (imode
, true))
12150 gsi
= gsi_last_bb (load_bb
);
12151 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12153 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12154 It only requires that the operation happen atomically. Thus we can
12155 use the RELAXED memory model. */
12156 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
12157 fold_convert_loc (loc
, itype
, rhs
),
12158 build_int_cst (NULL
,
12159 seq_cst
? MEMMODEL_SEQ_CST
12160 : MEMMODEL_RELAXED
));
12162 if (need_old
|| need_new
)
12164 lhs
= need_old
? loaded_val
: stored_val
;
12165 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
12166 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
12169 call
= fold_convert_loc (loc
, void_type_node
, call
);
12170 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12171 gsi_remove (&gsi
, true);
12173 gsi
= gsi_last_bb (store_bb
);
12174 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
12175 gsi_remove (&gsi
, true);
12176 gsi
= gsi_last_bb (store_bb
);
12177 stmt
= gsi_stmt (gsi
);
12178 gsi_remove (&gsi
, true);
12180 if (gimple_in_ssa_p (cfun
))
12182 release_defs (stmt
);
12183 update_ssa (TODO_update_ssa_no_phi
);
12189 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12193 newval = rhs; // with oldval replacing *addr in rhs
12194 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12195 if (oldval != newval)
12198 INDEX is log2 of the size of the data type, and thus usable to find the
12199 index of the builtin decl. */
12202 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
12203 tree addr
, tree loaded_val
, tree stored_val
,
12206 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
12207 tree type
, itype
, cmpxchg
, iaddr
;
12208 gimple_stmt_iterator si
;
12209 basic_block loop_header
= single_succ (load_bb
);
12210 gimple
*phi
, *stmt
;
12212 enum built_in_function fncode
;
12214 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12215 order to use the RELAXED memory model effectively. */
12216 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12218 cmpxchg
= builtin_decl_explicit (fncode
);
12219 if (cmpxchg
== NULL_TREE
)
12221 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12222 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
12224 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
12227 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12228 si
= gsi_last_bb (load_bb
);
12229 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12231 /* For floating-point values, we'll need to view-convert them to integers
12232 so that we can perform the atomic compare and swap. Simplify the
12233 following code by always setting up the "i"ntegral variables. */
12234 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
12238 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
12241 = force_gimple_operand_gsi (&si
,
12242 fold_convert (TREE_TYPE (iaddr
), addr
),
12243 false, NULL_TREE
, true, GSI_SAME_STMT
);
12244 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
12245 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12246 loadedi
= create_tmp_var (itype
);
12247 if (gimple_in_ssa_p (cfun
))
12248 loadedi
= make_ssa_name (loadedi
);
12253 loadedi
= loaded_val
;
12256 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
12257 tree loaddecl
= builtin_decl_explicit (fncode
);
12260 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12261 build_call_expr (loaddecl
, 2, iaddr
,
12262 build_int_cst (NULL_TREE
,
12263 MEMMODEL_RELAXED
)));
12265 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12266 build_int_cst (TREE_TYPE (iaddr
), 0));
12269 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12272 /* Move the value to the LOADEDI temporary. */
12273 if (gimple_in_ssa_p (cfun
))
12275 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12276 phi
= create_phi_node (loadedi
, loop_header
);
12277 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12281 gsi_insert_before (&si
,
12282 gimple_build_assign (loadedi
, initial
),
12284 if (loadedi
!= loaded_val
)
12286 gimple_stmt_iterator gsi2
;
12289 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12290 gsi2
= gsi_start_bb (loop_header
);
12291 if (gimple_in_ssa_p (cfun
))
12294 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12295 true, GSI_SAME_STMT
);
12296 stmt
= gimple_build_assign (loaded_val
, x
);
12297 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12301 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12302 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12303 true, GSI_SAME_STMT
);
12306 gsi_remove (&si
, true);
12308 si
= gsi_last_bb (store_bb
);
12309 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12312 storedi
= stored_val
;
12315 force_gimple_operand_gsi (&si
,
12316 build1 (VIEW_CONVERT_EXPR
, itype
,
12317 stored_val
), true, NULL_TREE
, true,
12320 /* Build the compare&swap statement. */
12321 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12322 new_storedi
= force_gimple_operand_gsi (&si
,
12323 fold_convert (TREE_TYPE (loadedi
),
12326 true, GSI_SAME_STMT
);
12328 if (gimple_in_ssa_p (cfun
))
12329 old_vali
= loadedi
;
12332 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12333 stmt
= gimple_build_assign (old_vali
, loadedi
);
12334 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12336 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12337 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12340 /* Note that we always perform the comparison as an integer, even for
12341 floating point. This allows the atomic operation to properly
12342 succeed even with NaNs and -0.0. */
12343 stmt
= gimple_build_cond_empty
12344 (build2 (NE_EXPR
, boolean_type_node
,
12345 new_storedi
, old_vali
));
12346 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12349 e
= single_succ_edge (store_bb
);
12350 e
->flags
&= ~EDGE_FALLTHRU
;
12351 e
->flags
|= EDGE_FALSE_VALUE
;
12353 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12355 /* Copy the new value to loadedi (we already did that before the condition
12356 if we are not in SSA). */
12357 if (gimple_in_ssa_p (cfun
))
12359 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12360 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12363 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12364 gsi_remove (&si
, true);
12366 struct loop
*loop
= alloc_loop ();
12367 loop
->header
= loop_header
;
12368 loop
->latch
= store_bb
;
12369 add_loop (loop
, loop_header
->loop_father
);
12371 if (gimple_in_ssa_p (cfun
))
12372 update_ssa (TODO_update_ssa_no_phi
);
12377 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12379 GOMP_atomic_start ();
12381 GOMP_atomic_end ();
12383 The result is not globally atomic, but works so long as all parallel
12384 references are within #pragma omp atomic directives. According to
12385 responses received from omp@openmp.org, appears to be within spec.
12386 Which makes sense, since that's how several other compilers handle
12387 this situation as well.
12388 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12389 expanding. STORED_VAL is the operand of the matching
12390 GIMPLE_OMP_ATOMIC_STORE.
12393 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12394 loaded_val = *addr;
12397 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12398 *addr = stored_val;
12402 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12403 tree addr
, tree loaded_val
, tree stored_val
)
12405 gimple_stmt_iterator si
;
12409 si
= gsi_last_bb (load_bb
);
12410 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12412 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12413 t
= build_call_expr (t
, 0);
12414 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12416 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12417 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12418 gsi_remove (&si
, true);
12420 si
= gsi_last_bb (store_bb
);
12421 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12423 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12425 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12427 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12428 t
= build_call_expr (t
, 0);
12429 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12430 gsi_remove (&si
, true);
12432 if (gimple_in_ssa_p (cfun
))
12433 update_ssa (TODO_update_ssa_no_phi
);
12437 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12438 using expand_omp_atomic_fetch_op. If it failed, we try to
12439 call expand_omp_atomic_pipeline, and if it fails too, the
12440 ultimate fallback is wrapping the operation in a mutex
12441 (expand_omp_atomic_mutex). REGION is the atomic region built
12442 by build_omp_regions_1(). */
12445 expand_omp_atomic (struct omp_region
*region
)
12447 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12448 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12449 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12450 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12451 tree addr
= gimple_omp_atomic_load_rhs (load
);
12452 tree stored_val
= gimple_omp_atomic_store_val (store
);
12453 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12454 HOST_WIDE_INT index
;
12456 /* Make sure the type is one of the supported sizes. */
12457 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12458 index
= exact_log2 (index
);
12459 if (index
>= 0 && index
<= 4)
12461 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12463 /* __sync builtins require strict data alignment. */
12464 if (exact_log2 (align
) >= index
)
12467 if (loaded_val
== stored_val
12468 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12469 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12470 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12471 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12474 /* Atomic store. */
12475 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12476 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12477 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12478 && store_bb
== single_succ (load_bb
)
12479 && first_stmt (store_bb
) == store
12480 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12481 stored_val
, index
))
12484 /* When possible, use specialized atomic update functions. */
12485 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12486 && store_bb
== single_succ (load_bb
)
12487 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12488 loaded_val
, stored_val
, index
))
12491 /* If we don't have specialized __sync builtins, try and implement
12492 as a compare and swap loop. */
12493 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12494 loaded_val
, stored_val
, index
))
12499 /* The ultimate fallback is wrapping the operation in a mutex. */
12500 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12504 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12505 macro on gomp-constants.h. We do not check for overflow. */
12508 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12512 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12515 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12516 device
, build_int_cst (unsigned_type_node
,
12517 GOMP_LAUNCH_DEVICE_SHIFT
));
12518 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12523 /* Look for compute grid dimension clauses and convert to an attribute
12524 attached to FN. This permits the target-side code to (a) massage
12525 the dimensions, (b) emit that data and (c) optimize. Non-constant
12526 dimensions are pushed onto ARGS.
12528 The attribute value is a TREE_LIST. A set of dimensions is
12529 represented as a list of INTEGER_CST. Those that are runtime
12530 exprs are represented as an INTEGER_CST of zero.
12532 TOOO. Normally the attribute will just contain a single such list. If
12533 however it contains a list of lists, this will represent the use of
12534 device_type. Each member of the outer list is an assoc list of
12535 dimensions, keyed by the device type. The first entry will be the
12536 default. Well, that's the plan. */
12538 #define OACC_FN_ATTRIB "oacc function"
12540 /* Replace any existing oacc fn attribute with updated dimensions. */
12543 replace_oacc_fn_attrib (tree fn
, tree dims
)
12545 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12546 tree attribs
= DECL_ATTRIBUTES (fn
);
12548 /* If we happen to be present as the first attrib, drop it. */
12549 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12550 attribs
= TREE_CHAIN (attribs
);
12551 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12554 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12555 function attribute. Push any that are non-constant onto the ARGS
12556 list, along with an appropriate GOMP_LAUNCH_DIM tag. IS_KERNEL is
12557 true, if these are for a kernels region offload function. */
12560 set_oacc_fn_attrib (tree fn
, tree clauses
, bool is_kernel
, vec
<tree
> *args
)
12562 /* Must match GOMP_DIM ordering. */
12563 static const omp_clause_code ids
[]
12564 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12565 OMP_CLAUSE_VECTOR_LENGTH
};
12567 tree dims
[GOMP_DIM_MAX
];
12568 tree attr
= NULL_TREE
;
12569 unsigned non_const
= 0;
12571 for (ix
= GOMP_DIM_MAX
; ix
--;)
12573 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12574 tree dim
= NULL_TREE
;
12577 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12579 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12581 dim
= integer_zero_node
;
12582 non_const
|= GOMP_DIM_MASK (ix
);
12584 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12585 /* Note kernelness with TREE_PUBLIC. */
12587 TREE_PUBLIC (attr
) = 1;
12590 replace_oacc_fn_attrib (fn
, attr
);
12594 /* Push a dynamic argument set. */
12595 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12596 NULL_TREE
, non_const
));
12597 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12598 if (non_const
& GOMP_DIM_MASK (ix
))
12599 args
->safe_push (dims
[ix
]);
12603 /* Process the routine's dimension clauess to generate an attribute
12604 value. Issue diagnostics as appropriate. We default to SEQ
12605 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12606 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12607 can have a loop partitioned on it. non-zero indicates
12608 yes, zero indicates no. By construction once a non-zero has been
12609 reached, further inner dimensions must also be non-zero. We set
12610 TREE_VALUE to zero for the dimensions that may be partitioned and
12611 1 for the other ones -- if a loop is (erroneously) spawned at
12612 an outer level, we don't want to try and partition it. */
12615 build_oacc_routine_dims (tree clauses
)
12617 /* Must match GOMP_DIM ordering. */
12618 static const omp_clause_code ids
[] =
12619 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12623 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12624 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12625 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12628 error_at (OMP_CLAUSE_LOCATION (clauses
),
12629 "multiple loop axes specified for routine");
12634 /* Default to SEQ. */
12636 level
= GOMP_DIM_MAX
;
12638 tree dims
= NULL_TREE
;
12640 for (ix
= GOMP_DIM_MAX
; ix
--;)
12641 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12642 build_int_cst (integer_type_node
, ix
< level
), dims
);
12647 /* Retrieve the oacc function attrib and return it. Non-oacc
12648 functions will return NULL. */
12651 get_oacc_fn_attrib (tree fn
)
12653 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12656 /* Return true if this oacc fn attrib is for a kernels offload
12657 region. We use the TREE_PUBLIC flag of each dimension -- only
12658 need to check the first one. */
12661 oacc_fn_attrib_kernels_p (tree attr
)
12663 return TREE_PUBLIC (TREE_VALUE (attr
));
12666 /* Return level at which oacc routine may spawn a partitioned loop, or
12667 -1 if it is not a routine (i.e. is an offload fn). */
12670 oacc_fn_attrib_level (tree attr
)
12672 tree pos
= TREE_VALUE (attr
);
12674 if (!TREE_PURPOSE (pos
))
12678 for (ix
= 0; ix
!= GOMP_DIM_MAX
;
12679 ix
++, pos
= TREE_CHAIN (pos
))
12680 if (!integer_zerop (TREE_PURPOSE (pos
)))
12686 /* Extract an oacc execution dimension from FN. FN must be an
12687 offloaded function or routine that has already had its execution
12688 dimensions lowered to the target-specific values. */
12691 get_oacc_fn_dim_size (tree fn
, int axis
)
12693 tree attrs
= get_oacc_fn_attrib (fn
);
12695 gcc_assert (axis
< GOMP_DIM_MAX
);
12697 tree dims
= TREE_VALUE (attrs
);
12699 dims
= TREE_CHAIN (dims
);
12701 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12706 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12707 IFN_GOACC_DIM_SIZE call. */
12710 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12712 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12713 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12714 tree arg
= gimple_call_arg (stmt
, 0);
12715 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12717 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12721 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12725 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
12726 basic_block region_exit
)
12728 struct loop
*outer
= region_entry
->loop_father
;
12729 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
12731 /* Don't parallelize the kernels region if it contains more than one outer
12733 unsigned int nr_outer_loops
= 0;
12734 struct loop
*single_outer
= NULL
;
12735 for (struct loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
12737 gcc_assert (loop_outer (loop
) == outer
);
12739 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
12742 if (region_exit
!= NULL
12743 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
12747 single_outer
= loop
;
12749 if (nr_outer_loops
!= 1)
12752 for (struct loop
*loop
= single_outer
->inner
; loop
!= NULL
; loop
= loop
->inner
)
12756 /* Mark the loops in the region. */
12757 for (struct loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
12758 loop
->in_oacc_kernels_region
= true;
12761 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12763 struct GTY(()) grid_launch_attributes_trees
12765 tree kernel_dim_array_type
;
12766 tree kernel_lattrs_dimnum_decl
;
12767 tree kernel_lattrs_grid_decl
;
12768 tree kernel_lattrs_group_decl
;
12769 tree kernel_launch_attributes_type
;
12772 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
12774 /* Create types used to pass kernel launch attributes to target. */
12777 grid_create_kernel_launch_attr_types (void)
12779 if (grid_attr_trees
)
12781 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
12783 tree dim_arr_index_type
12784 = build_index_type (build_int_cst (integer_type_node
, 2));
12785 grid_attr_trees
->kernel_dim_array_type
12786 = build_array_type (uint32_type_node
, dim_arr_index_type
);
12788 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
12789 grid_attr_trees
->kernel_lattrs_dimnum_decl
12790 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
12792 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
12794 grid_attr_trees
->kernel_lattrs_grid_decl
12795 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
12796 grid_attr_trees
->kernel_dim_array_type
);
12797 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
12798 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
12799 grid_attr_trees
->kernel_lattrs_group_decl
12800 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
12801 grid_attr_trees
->kernel_dim_array_type
);
12802 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
12803 = grid_attr_trees
->kernel_lattrs_grid_decl
;
12804 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
12805 "__gomp_kernel_launch_attributes",
12806 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
12809 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12810 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12811 of type uint32_type_node. */
12814 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
12815 tree fld_decl
, int index
, tree value
)
12817 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
12818 build3 (COMPONENT_REF
,
12819 grid_attr_trees
->kernel_dim_array_type
,
12820 range_var
, fld_decl
, NULL_TREE
),
12821 build_int_cst (integer_type_node
, index
),
12822 NULL_TREE
, NULL_TREE
);
12823 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
12826 /* Return a tree representation of a pointer to a structure with grid and
12827 work-group size information. Statements filling that information will be
12828 inserted before GSI, TGT_STMT is the target statement which has the
12829 necessary information in it. */
12832 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
12833 gomp_target
*tgt_stmt
)
12835 grid_create_kernel_launch_attr_types ();
12836 tree u32_one
= build_one_cst (uint32_type_node
);
12837 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
12838 "__kernel_launch_attrs");
12840 unsigned max_dim
= 0;
12841 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
12843 clause
= OMP_CLAUSE_CHAIN (clause
))
12845 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
12848 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
12849 max_dim
= MAX (dim
, max_dim
);
12851 grid_insert_store_range_dim (gsi
, lattrs
,
12852 grid_attr_trees
->kernel_lattrs_grid_decl
,
12853 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
12854 grid_insert_store_range_dim (gsi
, lattrs
,
12855 grid_attr_trees
->kernel_lattrs_group_decl
,
12856 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
12859 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
12860 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
12861 /* At this moment we cannot gridify a loop with a collapse clause. */
12862 /* TODO: Adjust when we support bigger collapse. */
12863 gcc_assert (max_dim
== 0);
12864 gsi_insert_before (gsi
, gimple_build_assign (dimref
, u32_one
), GSI_SAME_STMT
);
12865 TREE_ADDRESSABLE (lattrs
) = 1;
12866 return build_fold_addr_expr (lattrs
);
12869 /* Build target argument identifier from the DEVICE identifier, value
12870 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12873 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
12875 tree t
= build_int_cst (integer_type_node
, device
);
12876 if (subseqent_param
)
12877 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12878 build_int_cst (integer_type_node
,
12879 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
12880 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12881 build_int_cst (integer_type_node
, id
));
12885 /* Like above but return it in type that can be directly stored as an element
12886 of the argument array. */
12889 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
12891 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
12892 return fold_convert (ptr_type_node
, t
);
12895 /* Return a target argument consisting of DEVICE identifier, value identifier
12896 ID, and the actual VALUE. */
12899 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
12902 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
12903 fold_convert (integer_type_node
, value
),
12904 build_int_cst (unsigned_type_node
,
12905 GOMP_TARGET_ARG_VALUE_SHIFT
));
12906 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12907 get_target_argument_identifier_1 (device
, false, id
));
12908 t
= fold_convert (ptr_type_node
, t
);
12909 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
12912 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12913 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12914 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12918 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
12919 int id
, tree value
, vec
<tree
> *args
)
12921 if (tree_fits_shwi_p (value
)
12922 && tree_to_shwi (value
) > -(1 << 15)
12923 && tree_to_shwi (value
) < (1 << 15))
12924 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
12927 args
->quick_push (get_target_argument_identifier (device
, true, id
));
12928 value
= fold_convert (ptr_type_node
, value
);
12929 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
12931 args
->quick_push (value
);
12935 /* Create an array of arguments that is then passed to GOMP_target. */
12938 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
12940 auto_vec
<tree
, 6> args
;
12941 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
12942 tree t
, c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12944 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
12946 t
= integer_minus_one_node
;
12947 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12948 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
12950 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12952 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
12954 t
= integer_minus_one_node
;
12955 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12956 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
12959 /* Add HSA-specific grid sizes, if available. */
12960 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
12961 OMP_CLAUSE__GRIDDIM_
))
12963 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true,
12964 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
);
12965 args
.quick_push (t
);
12966 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
12969 /* Produce more, perhaps device specific, arguments here. */
12971 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
12972 args
.length () + 1),
12973 ".omp_target_args");
12974 for (unsigned i
= 0; i
< args
.length (); i
++)
12976 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12977 build_int_cst (integer_type_node
, i
),
12978 NULL_TREE
, NULL_TREE
);
12979 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
12982 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12983 build_int_cst (integer_type_node
, args
.length ()),
12984 NULL_TREE
, NULL_TREE
);
12985 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
12987 TREE_ADDRESSABLE (argarray
) = 1;
12988 return build_fold_addr_expr (argarray
);
12991 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12994 expand_omp_target (struct omp_region
*region
)
12996 basic_block entry_bb
, exit_bb
, new_bb
;
12997 struct function
*child_cfun
;
12998 tree child_fn
, block
, t
;
12999 gimple_stmt_iterator gsi
;
13000 gomp_target
*entry_stmt
;
13003 bool offloaded
, data_region
;
13005 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
13006 new_bb
= region
->entry
;
13008 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
13009 switch (gimple_omp_target_kind (entry_stmt
))
13011 case GF_OMP_TARGET_KIND_REGION
:
13012 case GF_OMP_TARGET_KIND_UPDATE
:
13013 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13014 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13015 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13016 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13017 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13018 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13019 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13020 data_region
= false;
13022 case GF_OMP_TARGET_KIND_DATA
:
13023 case GF_OMP_TARGET_KIND_OACC_DATA
:
13024 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13025 data_region
= true;
13028 gcc_unreachable ();
13031 child_fn
= NULL_TREE
;
13035 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
13036 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
13039 /* Supported by expand_omp_taskreg, but not here. */
13040 if (child_cfun
!= NULL
)
13041 gcc_checking_assert (!child_cfun
->cfg
);
13042 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
13044 entry_bb
= region
->entry
;
13045 exit_bb
= region
->exit
;
13047 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
13048 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
13052 unsigned srcidx
, dstidx
, num
;
13054 /* If the offloading region needs data sent from the parent
13055 function, then the very first statement (except possible
13056 tree profile counter updates) of the offloading body
13057 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
13058 &.OMP_DATA_O is passed as an argument to the child function,
13059 we need to replace it with the argument as seen by the child
13062 In most cases, this will end up being the identity assignment
13063 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
13064 a function call that has been inlined, the original PARM_DECL
13065 .OMP_DATA_I may have been converted into a different local
13066 variable. In which case, we need to keep the assignment. */
13067 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
13070 basic_block entry_succ_bb
= single_succ (entry_bb
);
13071 gimple_stmt_iterator gsi
;
13073 gimple
*tgtcopy_stmt
= NULL
;
13074 tree sender
= TREE_VEC_ELT (data_arg
, 0);
13076 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
13078 gcc_assert (!gsi_end_p (gsi
));
13079 stmt
= gsi_stmt (gsi
);
13080 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
13083 if (gimple_num_ops (stmt
) == 2)
13085 tree arg
= gimple_assign_rhs1 (stmt
);
13087 /* We're ignoring the subcode because we're
13088 effectively doing a STRIP_NOPS. */
13090 if (TREE_CODE (arg
) == ADDR_EXPR
13091 && TREE_OPERAND (arg
, 0) == sender
)
13093 tgtcopy_stmt
= stmt
;
13099 gcc_assert (tgtcopy_stmt
!= NULL
);
13100 arg
= DECL_ARGUMENTS (child_fn
);
13102 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
13103 gsi_remove (&gsi
, true);
13106 /* Declare local variables needed in CHILD_CFUN. */
13107 block
= DECL_INITIAL (child_fn
);
13108 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
13109 /* The gimplifier could record temporaries in the offloading block
13110 rather than in containing function's local_decls chain,
13111 which would mean cgraph missed finalizing them. Do it now. */
13112 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
13113 if (TREE_CODE (t
) == VAR_DECL
13115 && !DECL_EXTERNAL (t
))
13116 varpool_node::finalize_decl (t
);
13117 DECL_SAVED_TREE (child_fn
) = NULL
;
13118 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13119 gimple_set_body (child_fn
, NULL
);
13120 TREE_USED (block
) = 1;
13122 /* Reset DECL_CONTEXT on function arguments. */
13123 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13124 DECL_CONTEXT (t
) = child_fn
;
13126 /* Split ENTRY_BB at GIMPLE_*,
13127 so that it can be moved to the child function. */
13128 gsi
= gsi_last_bb (entry_bb
);
13129 stmt
= gsi_stmt (gsi
);
13131 && gimple_code (stmt
) == gimple_code (entry_stmt
));
13132 e
= split_block (entry_bb
, stmt
);
13133 gsi_remove (&gsi
, true);
13134 entry_bb
= e
->dest
;
13135 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
13137 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13140 gsi
= gsi_last_bb (exit_bb
);
13141 gcc_assert (!gsi_end_p (gsi
)
13142 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13143 stmt
= gimple_build_return (NULL
);
13144 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
13145 gsi_remove (&gsi
, true);
13148 /* Move the offloading region into CHILD_CFUN. */
13150 block
= gimple_block (entry_stmt
);
13152 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
13154 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
13155 /* When the OMP expansion process cannot guarantee an up-to-date
13156 loop tree arrange for the child function to fixup loops. */
13157 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13158 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
13160 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13161 num
= vec_safe_length (child_cfun
->local_decls
);
13162 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
13164 t
= (*child_cfun
->local_decls
)[srcidx
];
13165 if (DECL_CONTEXT (t
) == cfun
->decl
)
13167 if (srcidx
!= dstidx
)
13168 (*child_cfun
->local_decls
)[dstidx
] = t
;
13172 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
13174 /* Inform the callgraph about the new function. */
13175 child_cfun
->curr_properties
= cfun
->curr_properties
;
13176 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
13177 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
13178 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
13179 node
->parallelized_function
= 1;
13180 cgraph_node::add_new_function (child_fn
, true);
13182 /* Add the new function to the offload table. */
13183 if (ENABLE_OFFLOADING
)
13184 vec_safe_push (offload_funcs
, child_fn
);
13186 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
13187 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
13189 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13190 fixed in a following pass. */
13191 push_cfun (child_cfun
);
13193 assign_assembler_name_if_neeeded (child_fn
);
13194 cgraph_edge::rebuild_edges ();
13196 /* Some EH regions might become dead, see PR34608. If
13197 pass_cleanup_cfg isn't the first pass to happen with the
13198 new child, these dead EH edges might cause problems.
13199 Clean them up now. */
13200 if (flag_exceptions
)
13203 bool changed
= false;
13205 FOR_EACH_BB_FN (bb
, cfun
)
13206 changed
|= gimple_purge_dead_eh_edges (bb
);
13208 cleanup_tree_cfg ();
13210 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13211 verify_loop_structure ();
13214 if (dump_file
&& !gimple_in_ssa_p (cfun
))
13216 omp_any_child_fn_dumped
= true;
13217 dump_function_header (dump_file
, child_fn
, dump_flags
);
13218 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
13222 /* Emit a library call to launch the offloading region, or do data
13224 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
13225 enum built_in_function start_ix
;
13226 location_t clause_loc
;
13227 unsigned int flags_i
= 0;
13228 bool oacc_kernels_p
= false;
13230 switch (gimple_omp_target_kind (entry_stmt
))
13232 case GF_OMP_TARGET_KIND_REGION
:
13233 start_ix
= BUILT_IN_GOMP_TARGET
;
13235 case GF_OMP_TARGET_KIND_DATA
:
13236 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
13238 case GF_OMP_TARGET_KIND_UPDATE
:
13239 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
13241 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13242 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13244 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13245 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13246 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
13248 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13249 oacc_kernels_p
= true;
13251 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13252 start_ix
= BUILT_IN_GOACC_PARALLEL
;
13254 case GF_OMP_TARGET_KIND_OACC_DATA
:
13255 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13256 start_ix
= BUILT_IN_GOACC_DATA_START
;
13258 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13259 start_ix
= BUILT_IN_GOACC_UPDATE
;
13261 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13262 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
13264 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13265 start_ix
= BUILT_IN_GOACC_DECLARE
;
13268 gcc_unreachable ();
13271 clauses
= gimple_omp_target_clauses (entry_stmt
);
13273 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13274 library choose) and there is no conditional. */
13276 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
13278 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
13280 cond
= OMP_CLAUSE_IF_EXPR (c
);
13282 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
13285 /* Even if we pass it to all library function calls, it is currently only
13286 defined/used for the OpenMP target ones. */
13287 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
13288 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
13289 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
13290 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
13292 device
= OMP_CLAUSE_DEVICE_ID (c
);
13293 clause_loc
= OMP_CLAUSE_LOCATION (c
);
13296 clause_loc
= gimple_location (entry_stmt
);
13298 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
13300 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
13302 /* Ensure 'device' is of the correct type. */
13303 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
13305 /* If we found the clause 'if (cond)', build
13306 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13309 cond
= gimple_boolify (cond
);
13311 basic_block cond_bb
, then_bb
, else_bb
;
13315 tmp_var
= create_tmp_var (TREE_TYPE (device
));
13317 e
= split_block_after_labels (new_bb
);
13320 gsi
= gsi_last_bb (new_bb
);
13322 e
= split_block (new_bb
, gsi_stmt (gsi
));
13328 then_bb
= create_empty_bb (cond_bb
);
13329 else_bb
= create_empty_bb (then_bb
);
13330 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
13331 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
13333 stmt
= gimple_build_cond_empty (cond
);
13334 gsi
= gsi_last_bb (cond_bb
);
13335 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13337 gsi
= gsi_start_bb (then_bb
);
13338 stmt
= gimple_build_assign (tmp_var
, device
);
13339 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13341 gsi
= gsi_start_bb (else_bb
);
13342 stmt
= gimple_build_assign (tmp_var
,
13343 build_int_cst (integer_type_node
,
13344 GOMP_DEVICE_HOST_FALLBACK
));
13345 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13347 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
13348 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
13349 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
13350 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
13351 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
13352 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
13355 gsi
= gsi_last_bb (new_bb
);
13359 gsi
= gsi_last_bb (new_bb
);
13360 device
= force_gimple_operand_gsi (&gsi
, device
, true, NULL_TREE
,
13361 true, GSI_SAME_STMT
);
13364 t
= gimple_omp_target_data_arg (entry_stmt
);
13367 t1
= size_zero_node
;
13368 t2
= build_zero_cst (ptr_type_node
);
13374 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
13375 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
13376 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
13377 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
13378 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
13382 bool tagging
= false;
13383 /* The maximum number used by any start_ix, without varargs. */
13384 auto_vec
<tree
, 11> args
;
13385 args
.quick_push (device
);
13387 args
.quick_push (build_fold_addr_expr (child_fn
));
13388 args
.quick_push (t1
);
13389 args
.quick_push (t2
);
13390 args
.quick_push (t3
);
13391 args
.quick_push (t4
);
13394 case BUILT_IN_GOACC_DATA_START
:
13395 case BUILT_IN_GOACC_DECLARE
:
13396 case BUILT_IN_GOMP_TARGET_DATA
:
13398 case BUILT_IN_GOMP_TARGET
:
13399 case BUILT_IN_GOMP_TARGET_UPDATE
:
13400 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
13401 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
13402 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
13404 depend
= OMP_CLAUSE_DECL (c
);
13406 depend
= build_int_cst (ptr_type_node
, 0);
13407 args
.quick_push (depend
);
13408 if (start_ix
== BUILT_IN_GOMP_TARGET
)
13409 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
13411 case BUILT_IN_GOACC_PARALLEL
:
13413 set_oacc_fn_attrib (child_fn
, clauses
, oacc_kernels_p
, &args
);
13417 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
13418 case BUILT_IN_GOACC_UPDATE
:
13420 tree t_async
= NULL_TREE
;
13422 /* If present, use the value specified by the respective
13423 clause, making sure that is of the correct type. */
13424 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
13426 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13428 OMP_CLAUSE_ASYNC_EXPR (c
));
13430 /* Default values for t_async. */
13431 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
13433 build_int_cst (integer_type_node
,
13435 if (tagging
&& t_async
)
13437 unsigned HOST_WIDE_INT i_async
= GOMP_LAUNCH_OP_MAX
;
13439 if (TREE_CODE (t_async
) == INTEGER_CST
)
13441 /* See if we can pack the async arg in to the tag's
13443 i_async
= TREE_INT_CST_LOW (t_async
);
13444 if (i_async
< GOMP_LAUNCH_OP_MAX
)
13445 t_async
= NULL_TREE
;
13447 i_async
= GOMP_LAUNCH_OP_MAX
;
13449 args
.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC
, NULL_TREE
,
13453 args
.safe_push (t_async
);
13455 /* Save the argument index, and ... */
13456 unsigned t_wait_idx
= args
.length ();
13457 unsigned num_waits
= 0;
13458 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
13460 /* ... push a placeholder. */
13461 args
.safe_push (integer_zero_node
);
13463 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
13464 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
13466 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13468 OMP_CLAUSE_WAIT_EXPR (c
)));
13472 if (!tagging
|| num_waits
)
13476 /* Now that we know the number, update the placeholder. */
13478 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
13480 len
= build_int_cst (integer_type_node
, num_waits
);
13481 len
= fold_convert_loc (gimple_location (entry_stmt
),
13482 unsigned_type_node
, len
);
13483 args
[t_wait_idx
] = len
;
13488 gcc_unreachable ();
13491 /* Push terminal marker - zero. */
13492 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
13494 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
13495 gimple_set_location (g
, gimple_location (entry_stmt
));
13496 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13499 g
= gsi_stmt (gsi
);
13500 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
13501 gsi_remove (&gsi
, true);
13503 if (data_region
&& region
->exit
)
13505 gsi
= gsi_last_bb (region
->exit
);
13506 g
= gsi_stmt (gsi
);
13507 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
13508 gsi_remove (&gsi
, true);
13512 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13513 variable derived from the thread number. */
13516 grid_expand_omp_for_loop (struct omp_region
*kfor
)
13520 gimple_stmt_iterator gsi
;
13522 struct omp_for_data fd
;
13524 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
13525 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
13526 == GF_OMP_FOR_KIND_GRID_LOOP
);
13527 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
13529 gcc_assert (gimple_omp_for_collapse (for_stmt
) == 1);
13530 gcc_assert (kfor
->cont
);
13531 extract_omp_for_data (for_stmt
, &fd
, NULL
);
13533 itype
= type
= TREE_TYPE (fd
.loop
.v
);
13534 if (POINTER_TYPE_P (type
))
13535 itype
= signed_type_for (type
);
13537 gsi
= gsi_start_bb (body_bb
);
13540 step
= fd
.loop
.step
;
13541 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
13542 true, NULL_TREE
, true, GSI_SAME_STMT
);
13543 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
13544 true, NULL_TREE
, true, GSI_SAME_STMT
);
13545 threadid
= build_call_expr (builtin_decl_explicit
13546 (BUILT_IN_OMP_GET_THREAD_NUM
), 0);
13547 threadid
= fold_convert (itype
, threadid
);
13548 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
13549 true, GSI_SAME_STMT
);
13551 tree startvar
= fd
.loop
.v
;
13552 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
13553 if (POINTER_TYPE_P (type
))
13554 t
= fold_build_pointer_plus (n1
, t
);
13556 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
13557 t
= fold_convert (type
, t
);
13558 t
= force_gimple_operand_gsi (&gsi
, t
,
13560 && TREE_ADDRESSABLE (startvar
),
13561 NULL_TREE
, true, GSI_SAME_STMT
);
13562 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
13563 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
13565 /* Remove the omp for statement */
13566 gsi
= gsi_last_bb (kfor
->entry
);
13567 gsi_remove (&gsi
, true);
13569 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13570 gsi
= gsi_last_bb (kfor
->cont
);
13571 gcc_assert (!gsi_end_p (gsi
)
13572 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
13573 gsi_remove (&gsi
, true);
13575 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13576 gsi
= gsi_last_bb (kfor
->exit
);
13577 gcc_assert (!gsi_end_p (gsi
)
13578 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13579 gsi_remove (&gsi
, true);
13581 /* Fixup the much simpler CFG. */
13582 remove_edge (find_edge (kfor
->cont
, body_bb
));
13584 if (kfor
->cont
!= body_bb
)
13585 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
13586 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
13589 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13592 struct grid_arg_decl_map
13598 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13599 pertaining to kernel function. */
13602 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
13604 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13605 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
13608 if (t
== adm
->old_arg
)
13609 *tp
= adm
->new_arg
;
13610 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
13614 static void expand_omp (struct omp_region
*region
);
13616 /* If TARGET region contains a kernel body for loop, remove its region from the
13617 TARGET and expand it in GPGPU kernel fashion. */
13620 grid_expand_target_grid_body (struct omp_region
*target
)
13622 if (!hsa_gen_requested_p ())
13625 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
13626 struct omp_region
**pp
;
13628 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
13629 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
13632 struct omp_region
*gpukernel
= *pp
;
13634 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
13637 /* HSA cannot handle OACC stuff. */
13638 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
13640 gcc_checking_assert (orig_child_fndecl
);
13641 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13642 OMP_CLAUSE__GRIDDIM_
));
13643 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
13645 hsa_register_kernel (n
);
13649 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13650 OMP_CLAUSE__GRIDDIM_
));
13651 tree inside_block
= gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
13652 *pp
= gpukernel
->next
;
13653 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
13654 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
13657 struct omp_region
*kfor
= *pp
;
13659 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor
)->entry
))
13660 == GF_OMP_FOR_KIND_GRID_LOOP
);
13663 expand_omp (kfor
->inner
);
13664 if (gpukernel
->inner
)
13665 expand_omp (gpukernel
->inner
);
13667 tree kern_fndecl
= copy_node (orig_child_fndecl
);
13668 DECL_NAME (kern_fndecl
) = clone_function_name (kern_fndecl
, "kernel");
13669 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
13670 tree tgtblock
= gimple_block (tgt_stmt
);
13671 tree fniniblock
= make_node (BLOCK
);
13672 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = tgtblock
;
13673 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
13674 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
13675 DECL_INITIAL (kern_fndecl
) = fniniblock
;
13676 push_struct_function (kern_fndecl
);
13677 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
13680 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
13681 gcc_assert (!DECL_CHAIN (old_parm_decl
));
13682 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
13683 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
13684 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
13685 gcc_assert (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (kern_fndecl
))));
13686 DECL_RESULT (kern_fndecl
) = copy_node (DECL_RESULT (kern_fndecl
));
13687 DECL_CONTEXT (DECL_RESULT (kern_fndecl
)) = kern_fndecl
;
13688 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
13689 kern_cfun
->curr_properties
= cfun
->curr_properties
;
13691 remove_edge (BRANCH_EDGE (kfor
->entry
));
13692 grid_expand_omp_for_loop (kfor
);
13694 /* Remove the omp for statement */
13695 gimple_stmt_iterator gsi
= gsi_last_bb (gpukernel
->entry
);
13696 gsi_remove (&gsi
, true);
13697 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13699 gsi
= gsi_last_bb (gpukernel
->exit
);
13700 gcc_assert (!gsi_end_p (gsi
)
13701 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13702 gimple
*ret_stmt
= gimple_build_return (NULL
);
13703 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
13704 gsi_remove (&gsi
, true);
13706 /* Statements in the first BB in the target construct have been produced by
13707 target lowering and must be copied inside the GPUKERNEL, with the two
13708 exceptions of the first OMP statement and the OMP_DATA assignment
13710 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
13711 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
13712 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
13713 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
13714 !gsi_end_p (tsi
); gsi_next (&tsi
))
13716 gimple
*stmt
= gsi_stmt (tsi
);
13717 if (is_gimple_omp (stmt
))
13720 && is_gimple_assign (stmt
)
13721 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
13722 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
13724 gimple
*copy
= gimple_copy (stmt
);
13725 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
13726 gimple_set_block (copy
, fniniblock
);
13729 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
13730 gpukernel
->exit
, inside_block
);
13732 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
13733 kcn
->mark_force_output ();
13734 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
13736 hsa_register_kernel (kcn
, orig_child
);
13738 cgraph_node::add_new_function (kern_fndecl
, true);
13739 push_cfun (kern_cfun
);
13740 cgraph_edge::rebuild_edges ();
13742 /* Re-map any mention of the PARM_DECL of the original function to the
13743 PARM_DECL of the new one.
13745 TODO: It would be great if lowering produced references into the GPU
13746 kernel decl straight away and we did not have to do this. */
13747 struct grid_arg_decl_map adm
;
13748 adm
.old_arg
= old_parm_decl
;
13749 adm
.new_arg
= new_parm_decl
;
13751 FOR_EACH_BB_FN (bb
, kern_cfun
)
13753 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13755 gimple
*stmt
= gsi_stmt (gsi
);
13756 struct walk_stmt_info wi
;
13757 memset (&wi
, 0, sizeof (wi
));
13759 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
13767 /* Expand the parallel region tree rooted at REGION. Expansion
13768 proceeds in depth-first order. Innermost regions are expanded
13769 first. This way, parallel regions that require a new function to
13770 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13771 internal dependencies in their body. */
13774 expand_omp (struct omp_region
*region
)
13776 omp_any_child_fn_dumped
= false;
13779 location_t saved_location
;
13780 gimple
*inner_stmt
= NULL
;
13782 /* First, determine whether this is a combined parallel+workshare
13784 if (region
->type
== GIMPLE_OMP_PARALLEL
)
13785 determine_parallel_type (region
);
13786 else if (region
->type
== GIMPLE_OMP_TARGET
)
13787 grid_expand_target_grid_body (region
);
13789 if (region
->type
== GIMPLE_OMP_FOR
13790 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
13791 inner_stmt
= last_stmt (region
->inner
->entry
);
13794 expand_omp (region
->inner
);
13796 saved_location
= input_location
;
13797 if (gimple_has_location (last_stmt (region
->entry
)))
13798 input_location
= gimple_location (last_stmt (region
->entry
));
13800 switch (region
->type
)
13802 case GIMPLE_OMP_PARALLEL
:
13803 case GIMPLE_OMP_TASK
:
13804 expand_omp_taskreg (region
);
13807 case GIMPLE_OMP_FOR
:
13808 expand_omp_for (region
, inner_stmt
);
13811 case GIMPLE_OMP_SECTIONS
:
13812 expand_omp_sections (region
);
13815 case GIMPLE_OMP_SECTION
:
13816 /* Individual omp sections are handled together with their
13817 parent GIMPLE_OMP_SECTIONS region. */
13820 case GIMPLE_OMP_SINGLE
:
13821 expand_omp_single (region
);
13824 case GIMPLE_OMP_ORDERED
:
13826 gomp_ordered
*ord_stmt
13827 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13828 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13829 OMP_CLAUSE_DEPEND
))
13831 /* We'll expand these when expanding corresponding
13832 worksharing region with ordered(n) clause. */
13833 gcc_assert (region
->outer
13834 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13835 region
->ord_stmt
= ord_stmt
;
13840 case GIMPLE_OMP_MASTER
:
13841 case GIMPLE_OMP_TASKGROUP
:
13842 case GIMPLE_OMP_CRITICAL
:
13843 case GIMPLE_OMP_TEAMS
:
13844 expand_omp_synch (region
);
13847 case GIMPLE_OMP_ATOMIC_LOAD
:
13848 expand_omp_atomic (region
);
13851 case GIMPLE_OMP_TARGET
:
13852 expand_omp_target (region
);
13856 gcc_unreachable ();
13859 input_location
= saved_location
;
13860 region
= region
->next
;
13862 if (omp_any_child_fn_dumped
)
13865 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13866 omp_any_child_fn_dumped
= false;
13871 /* Helper for build_omp_regions. Scan the dominator tree starting at
13872 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13873 true, the function ends once a single tree is built (otherwise, whole
13874 forest of OMP constructs may be built). */
13877 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13880 gimple_stmt_iterator gsi
;
13884 gsi
= gsi_last_bb (bb
);
13885 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13887 struct omp_region
*region
;
13888 enum gimple_code code
;
13890 stmt
= gsi_stmt (gsi
);
13891 code
= gimple_code (stmt
);
13892 if (code
== GIMPLE_OMP_RETURN
)
13894 /* STMT is the return point out of region PARENT. Mark it
13895 as the exit point and make PARENT the immediately
13896 enclosing region. */
13897 gcc_assert (parent
);
13900 parent
= parent
->outer
;
13902 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13904 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13905 GIMPLE_OMP_RETURN, but matches with
13906 GIMPLE_OMP_ATOMIC_LOAD. */
13907 gcc_assert (parent
);
13908 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13911 parent
= parent
->outer
;
13913 else if (code
== GIMPLE_OMP_CONTINUE
)
13915 gcc_assert (parent
);
13918 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13920 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13921 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13925 region
= new_omp_region (bb
, code
, parent
);
13927 if (code
== GIMPLE_OMP_TARGET
)
13929 switch (gimple_omp_target_kind (stmt
))
13931 case GF_OMP_TARGET_KIND_REGION
:
13932 case GF_OMP_TARGET_KIND_DATA
:
13933 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13934 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13935 case GF_OMP_TARGET_KIND_OACC_DATA
:
13936 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13938 case GF_OMP_TARGET_KIND_UPDATE
:
13939 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13940 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13941 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13942 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13943 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13944 /* ..., other than for those stand-alone directives... */
13948 gcc_unreachable ();
13951 else if (code
== GIMPLE_OMP_ORDERED
13952 && find_omp_clause (gimple_omp_ordered_clauses
13953 (as_a
<gomp_ordered
*> (stmt
)),
13954 OMP_CLAUSE_DEPEND
))
13955 /* #pragma omp ordered depend is also just a stand-alone
13958 /* ..., this directive becomes the parent for a new region. */
13964 if (single_tree
&& !parent
)
13967 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13969 son
= next_dom_son (CDI_DOMINATORS
, son
))
13970 build_omp_regions_1 (son
, parent
, single_tree
);
13973 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13974 root_omp_region. */
13977 build_omp_regions_root (basic_block root
)
13979 gcc_assert (root_omp_region
== NULL
);
13980 build_omp_regions_1 (root
, NULL
, true);
13981 gcc_assert (root_omp_region
!= NULL
);
13984 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13987 omp_expand_local (basic_block head
)
13989 build_omp_regions_root (head
);
13990 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13992 fprintf (dump_file
, "\nOMP region tree\n\n");
13993 dump_omp_region (dump_file
, root_omp_region
, 0);
13994 fprintf (dump_file
, "\n");
13997 remove_exit_barriers (root_omp_region
);
13998 expand_omp (root_omp_region
);
14000 free_omp_regions ();
14003 /* Scan the CFG and build a tree of OMP regions. Return the root of
14004 the OMP region tree. */
14007 build_omp_regions (void)
14009 gcc_assert (root_omp_region
== NULL
);
14010 calculate_dominance_info (CDI_DOMINATORS
);
14011 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
14014 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
14016 static unsigned int
14017 execute_expand_omp (void)
14019 build_omp_regions ();
14021 if (!root_omp_region
)
14026 fprintf (dump_file
, "\nOMP region tree\n\n");
14027 dump_omp_region (dump_file
, root_omp_region
, 0);
14028 fprintf (dump_file
, "\n");
14031 remove_exit_barriers (root_omp_region
);
14033 expand_omp (root_omp_region
);
14035 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
14036 verify_loop_structure ();
14037 cleanup_tree_cfg ();
14039 free_omp_regions ();
14044 /* OMP expansion -- the default pass, run before creation of SSA form. */
14048 const pass_data pass_data_expand_omp
=
14050 GIMPLE_PASS
, /* type */
14051 "ompexp", /* name */
14052 OPTGROUP_NONE
, /* optinfo_flags */
14053 TV_NONE
, /* tv_id */
14054 PROP_gimple_any
, /* properties_required */
14055 PROP_gimple_eomp
, /* properties_provided */
14056 0, /* properties_destroyed */
14057 0, /* todo_flags_start */
14058 0, /* todo_flags_finish */
14061 class pass_expand_omp
: public gimple_opt_pass
14064 pass_expand_omp (gcc::context
*ctxt
)
14065 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
14068 /* opt_pass methods: */
14069 virtual unsigned int execute (function
*)
14071 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
14072 || flag_openmp_simd
!= 0)
14073 && !seen_error ());
14075 /* This pass always runs, to provide PROP_gimple_eomp.
14076 But often, there is nothing to do. */
14080 return execute_expand_omp ();
14083 }; // class pass_expand_omp
14085 } // anon namespace
14088 make_pass_expand_omp (gcc::context
*ctxt
)
14090 return new pass_expand_omp (ctxt
);
14095 const pass_data pass_data_expand_omp_ssa
=
14097 GIMPLE_PASS
, /* type */
14098 "ompexpssa", /* name */
14099 OPTGROUP_NONE
, /* optinfo_flags */
14100 TV_NONE
, /* tv_id */
14101 PROP_cfg
| PROP_ssa
, /* properties_required */
14102 PROP_gimple_eomp
, /* properties_provided */
14103 0, /* properties_destroyed */
14104 0, /* todo_flags_start */
14105 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
14108 class pass_expand_omp_ssa
: public gimple_opt_pass
14111 pass_expand_omp_ssa (gcc::context
*ctxt
)
14112 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
14115 /* opt_pass methods: */
14116 virtual bool gate (function
*fun
)
14118 return !(fun
->curr_properties
& PROP_gimple_eomp
);
14120 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
14121 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
14123 }; // class pass_expand_omp_ssa
14125 } // anon namespace
14128 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
14130 return new pass_expand_omp_ssa (ctxt
);
14133 /* Routines to lower OMP directives into OMP-GIMPLE. */
14135 /* If ctx is a worksharing context inside of a cancellable parallel
14136 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14137 and conditional branch to parallel's cancel_label to handle
14138 cancellation in the implicit barrier. */
14141 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
14143 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
14144 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
14145 if (gimple_omp_return_nowait_p (omp_return
))
14148 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
14149 && ctx
->outer
->cancellable
)
14151 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
14152 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
14153 tree lhs
= create_tmp_var (c_bool_type
);
14154 gimple_omp_return_set_lhs (omp_return
, lhs
);
14155 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
14156 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
14157 fold_convert (c_bool_type
,
14158 boolean_false_node
),
14159 ctx
->outer
->cancel_label
, fallthru_label
);
14160 gimple_seq_add_stmt (body
, g
);
14161 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
14165 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14166 CTX is the enclosing OMP context for the current statement. */
14169 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14171 tree block
, control
;
14172 gimple_stmt_iterator tgsi
;
14173 gomp_sections
*stmt
;
14175 gbind
*new_stmt
, *bind
;
14176 gimple_seq ilist
, dlist
, olist
, new_body
;
14178 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
14180 push_gimplify_context ();
14184 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
14185 &ilist
, &dlist
, ctx
, NULL
);
14187 new_body
= gimple_omp_body (stmt
);
14188 gimple_omp_set_body (stmt
, NULL
);
14189 tgsi
= gsi_start (new_body
);
14190 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
14195 sec_start
= gsi_stmt (tgsi
);
14196 sctx
= maybe_lookup_ctx (sec_start
);
14199 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
14200 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
14201 GSI_CONTINUE_LINKING
);
14202 gimple_omp_set_body (sec_start
, NULL
);
14204 if (gsi_one_before_end_p (tgsi
))
14206 gimple_seq l
= NULL
;
14207 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
14209 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
14210 gimple_omp_section_set_last (sec_start
);
14213 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
14214 GSI_CONTINUE_LINKING
);
14217 block
= make_node (BLOCK
);
14218 bind
= gimple_build_bind (NULL
, new_body
, block
);
14221 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
14223 block
= make_node (BLOCK
);
14224 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14225 gsi_replace (gsi_p
, new_stmt
, true);
14227 pop_gimplify_context (new_stmt
);
14228 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14229 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14230 if (BLOCK_VARS (block
))
14231 TREE_USED (block
) = 1;
14234 gimple_seq_add_seq (&new_body
, ilist
);
14235 gimple_seq_add_stmt (&new_body
, stmt
);
14236 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
14237 gimple_seq_add_stmt (&new_body
, bind
);
14239 control
= create_tmp_var (unsigned_type_node
, ".section");
14240 t
= gimple_build_omp_continue (control
, control
);
14241 gimple_omp_sections_set_control (stmt
, control
);
14242 gimple_seq_add_stmt (&new_body
, t
);
14244 gimple_seq_add_seq (&new_body
, olist
);
14245 if (ctx
->cancellable
)
14246 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14247 gimple_seq_add_seq (&new_body
, dlist
);
14249 new_body
= maybe_catch_exception (new_body
);
14251 t
= gimple_build_omp_return
14252 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
14253 OMP_CLAUSE_NOWAIT
));
14254 gimple_seq_add_stmt (&new_body
, t
);
14255 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
14257 gimple_bind_set_body (new_stmt
, new_body
);
14261 /* A subroutine of lower_omp_single. Expand the simple form of
14262 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14264 if (GOMP_single_start ())
14266 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14268 FIXME. It may be better to delay expanding the logic of this until
14269 pass_expand_omp. The expanded logic may make the job more difficult
14270 to a synchronization analysis pass. */
14273 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
14275 location_t loc
= gimple_location (single_stmt
);
14276 tree tlabel
= create_artificial_label (loc
);
14277 tree flabel
= create_artificial_label (loc
);
14278 gimple
*call
, *cond
;
14281 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
14282 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
14283 call
= gimple_build_call (decl
, 0);
14284 gimple_call_set_lhs (call
, lhs
);
14285 gimple_seq_add_stmt (pre_p
, call
);
14287 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
14288 fold_convert_loc (loc
, TREE_TYPE (lhs
),
14289 boolean_true_node
),
14291 gimple_seq_add_stmt (pre_p
, cond
);
14292 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
14293 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14294 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
14298 /* A subroutine of lower_omp_single. Expand the simple form of
14299 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14301 #pragma omp single copyprivate (a, b, c)
14303 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14306 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14312 GOMP_single_copy_end (©out);
14323 FIXME. It may be better to delay expanding the logic of this until
14324 pass_expand_omp. The expanded logic may make the job more difficult
14325 to a synchronization analysis pass. */
14328 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
14331 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
14332 gimple_seq copyin_seq
;
14333 location_t loc
= gimple_location (single_stmt
);
14335 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
14337 ptr_type
= build_pointer_type (ctx
->record_type
);
14338 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
14340 l0
= create_artificial_label (loc
);
14341 l1
= create_artificial_label (loc
);
14342 l2
= create_artificial_label (loc
);
14344 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
14345 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
14346 t
= fold_convert_loc (loc
, ptr_type
, t
);
14347 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
14349 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
14350 build_int_cst (ptr_type
, 0));
14351 t
= build3 (COND_EXPR
, void_type_node
, t
,
14352 build_and_jump (&l0
), build_and_jump (&l1
));
14353 gimplify_and_add (t
, pre_p
);
14355 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
14357 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14360 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
14363 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14364 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
14365 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
14366 gimplify_and_add (t
, pre_p
);
14368 t
= build_and_jump (&l2
);
14369 gimplify_and_add (t
, pre_p
);
14371 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
14373 gimple_seq_add_seq (pre_p
, copyin_seq
);
14375 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
14379 /* Expand code for an OpenMP single directive. */
14382 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14386 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
14388 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
14390 push_gimplify_context ();
14392 block
= make_node (BLOCK
);
14393 bind
= gimple_build_bind (NULL
, NULL
, block
);
14394 gsi_replace (gsi_p
, bind
, true);
14397 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
14398 &bind_body
, &dlist
, ctx
, NULL
);
14399 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
14401 gimple_seq_add_stmt (&bind_body
, single_stmt
);
14403 if (ctx
->record_type
)
14404 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
14406 lower_omp_single_simple (single_stmt
, &bind_body
);
14408 gimple_omp_set_body (single_stmt
, NULL
);
14410 gimple_seq_add_seq (&bind_body
, dlist
);
14412 bind_body
= maybe_catch_exception (bind_body
);
14414 t
= gimple_build_omp_return
14415 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
14416 OMP_CLAUSE_NOWAIT
));
14417 gimple_seq_add_stmt (&bind_body_tail
, t
);
14418 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
14419 if (ctx
->record_type
)
14421 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
14422 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14423 TREE_THIS_VOLATILE (clobber
) = 1;
14424 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
14425 clobber
), GSI_SAME_STMT
);
14427 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
14428 gimple_bind_set_body (bind
, bind_body
);
14430 pop_gimplify_context (bind
);
14432 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14433 BLOCK_VARS (block
) = ctx
->block_vars
;
14434 if (BLOCK_VARS (block
))
14435 TREE_USED (block
) = 1;
14439 /* Expand code for an OpenMP master directive. */
14442 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14444 tree block
, lab
= NULL
, x
, bfn_decl
;
14445 gimple
*stmt
= gsi_stmt (*gsi_p
);
14447 location_t loc
= gimple_location (stmt
);
14450 push_gimplify_context ();
14452 block
= make_node (BLOCK
);
14453 bind
= gimple_build_bind (NULL
, NULL
, block
);
14454 gsi_replace (gsi_p
, bind
, true);
14455 gimple_bind_add_stmt (bind
, stmt
);
14457 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
14458 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
14459 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
14460 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
14462 gimplify_and_add (x
, &tseq
);
14463 gimple_bind_add_seq (bind
, tseq
);
14465 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14466 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14467 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14468 gimple_omp_set_body (stmt
, NULL
);
14470 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
14472 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14474 pop_gimplify_context (bind
);
14476 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14477 BLOCK_VARS (block
) = ctx
->block_vars
;
14481 /* Expand code for an OpenMP taskgroup directive. */
14484 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14486 gimple
*stmt
= gsi_stmt (*gsi_p
);
14489 tree block
= make_node (BLOCK
);
14491 bind
= gimple_build_bind (NULL
, NULL
, block
);
14492 gsi_replace (gsi_p
, bind
, true);
14493 gimple_bind_add_stmt (bind
, stmt
);
14495 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
14497 gimple_bind_add_stmt (bind
, x
);
14499 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14500 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14501 gimple_omp_set_body (stmt
, NULL
);
14503 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14505 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14506 BLOCK_VARS (block
) = ctx
->block_vars
;
14510 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14513 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
14516 struct omp_for_data fd
;
14517 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
14520 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
14521 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
14522 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
14526 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14527 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
14528 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14529 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
14531 /* Merge depend clauses from multiple adjacent
14532 #pragma omp ordered depend(sink:...) constructs
14533 into one #pragma omp ordered depend(sink:...), so that
14534 we can optimize them together. */
14535 gimple_stmt_iterator gsi
= *gsi_p
;
14537 while (!gsi_end_p (gsi
))
14539 gimple
*stmt
= gsi_stmt (gsi
);
14540 if (is_gimple_debug (stmt
)
14541 || gimple_code (stmt
) == GIMPLE_NOP
)
14546 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
14548 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
14549 c
= gimple_omp_ordered_clauses (ord_stmt2
);
14551 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
14552 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14555 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
14557 gsi_remove (&gsi
, true);
14561 /* Canonicalize sink dependence clauses into one folded clause if
14564 The basic algorithm is to create a sink vector whose first
14565 element is the GCD of all the first elements, and whose remaining
14566 elements are the minimum of the subsequent columns.
14568 We ignore dependence vectors whose first element is zero because
14569 such dependencies are known to be executed by the same thread.
14571 We take into account the direction of the loop, so a minimum
14572 becomes a maximum if the loop is iterating forwards. We also
14573 ignore sink clauses where the loop direction is unknown, or where
14574 the offsets are clearly invalid because they are not a multiple
14575 of the loop increment.
14579 #pragma omp for ordered(2)
14580 for (i=0; i < N; ++i)
14581 for (j=0; j < M; ++j)
14583 #pragma omp ordered \
14584 depend(sink:i-8,j-2) \
14585 depend(sink:i,j-1) \ // Completely ignored because i+0.
14586 depend(sink:i-4,j-3) \
14587 depend(sink:i-6,j-4)
14588 #pragma omp ordered depend(source)
14593 depend(sink:-gcd(8,4,6),-min(2,3,4))
14598 /* FIXME: Computing GCD's where the first element is zero is
14599 non-trivial in the presence of collapsed loops. Do this later. */
14600 if (fd
.collapse
> 1)
14603 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
14604 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
14605 tree folded_dep
= NULL_TREE
;
14606 /* TRUE if the first dimension's offset is negative. */
14607 bool neg_offset_p
= false;
14609 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14611 while ((c
= *list_p
) != NULL
)
14613 bool remove
= false;
14615 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
14616 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14617 goto next_ordered_clause
;
14620 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
14621 vec
&& TREE_CODE (vec
) == TREE_LIST
;
14622 vec
= TREE_CHAIN (vec
), ++i
)
14624 gcc_assert (i
< len
);
14626 /* extract_omp_for_data has canonicalized the condition. */
14627 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
14628 || fd
.loops
[i
].cond_code
== GT_EXPR
);
14629 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
14630 bool maybe_lexically_later
= true;
14632 /* While the committee makes up its mind, bail if we have any
14633 non-constant steps. */
14634 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
14635 goto lower_omp_ordered_ret
;
14637 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
14638 if (POINTER_TYPE_P (itype
))
14640 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
14641 TYPE_PRECISION (itype
),
14642 TYPE_SIGN (itype
));
14644 /* Ignore invalid offsets that are not multiples of the step. */
14645 if (!wi::multiple_of_p
14646 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
14649 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
14650 "ignoring sink clause with offset that is not "
14651 "a multiple of the loop step");
14653 goto next_ordered_clause
;
14656 /* Calculate the first dimension. The first dimension of
14657 the folded dependency vector is the GCD of the first
14658 elements, while ignoring any first elements whose offset
14662 /* Ignore dependence vectors whose first dimension is 0. */
14666 goto next_ordered_clause
;
14670 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
14672 error_at (OMP_CLAUSE_LOCATION (c
),
14673 "first offset must be in opposite direction "
14674 "of loop iterations");
14675 goto lower_omp_ordered_ret
;
14679 neg_offset_p
= forward
;
14680 /* Initialize the first time around. */
14681 if (folded_dep
== NULL_TREE
)
14684 folded_deps
[0] = offset
;
14687 folded_deps
[0] = wi::gcd (folded_deps
[0],
14691 /* Calculate minimum for the remaining dimensions. */
14694 folded_deps
[len
+ i
- 1] = offset
;
14695 if (folded_dep
== c
)
14696 folded_deps
[i
] = offset
;
14697 else if (maybe_lexically_later
14698 && !wi::eq_p (folded_deps
[i
], offset
))
14700 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
14704 for (j
= 1; j
<= i
; j
++)
14705 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
14708 maybe_lexically_later
= false;
14712 gcc_assert (i
== len
);
14716 next_ordered_clause
:
14718 *list_p
= OMP_CLAUSE_CHAIN (c
);
14720 list_p
= &OMP_CLAUSE_CHAIN (c
);
14726 folded_deps
[0] = -folded_deps
[0];
14728 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
14729 if (POINTER_TYPE_P (itype
))
14732 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14733 = wide_int_to_tree (itype
, folded_deps
[0]);
14734 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14735 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14738 lower_omp_ordered_ret
:
14740 /* Ordered without clauses is #pragma omp threads, while we want
14741 a nop instead if we remove all clauses. */
14742 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14743 gsi_replace (gsi_p
, gimple_build_nop (), true);
14747 /* Expand code for an OpenMP ordered directive. */
14750 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14753 gimple
*stmt
= gsi_stmt (*gsi_p
);
14754 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14757 bool simd
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14759 bool threads
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14760 OMP_CLAUSE_THREADS
);
14762 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14763 OMP_CLAUSE_DEPEND
))
14765 /* FIXME: This is needs to be moved to the expansion to verify various
14766 conditions only testable on cfg with dominators computed, and also
14767 all the depend clauses to be merged still might need to be available
14768 for the runtime checks. */
14770 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14774 push_gimplify_context ();
14776 block
= make_node (BLOCK
);
14777 bind
= gimple_build_bind (NULL
, NULL
, block
);
14778 gsi_replace (gsi_p
, bind
, true);
14779 gimple_bind_add_stmt (bind
, stmt
);
14783 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
14784 build_int_cst (NULL_TREE
, threads
));
14785 cfun
->has_simduid_loops
= true;
14788 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14790 gimple_bind_add_stmt (bind
, x
);
14792 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14793 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14794 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14795 gimple_omp_set_body (stmt
, NULL
);
14798 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
14799 build_int_cst (NULL_TREE
, threads
));
14801 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14803 gimple_bind_add_stmt (bind
, x
);
14805 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14807 pop_gimplify_context (bind
);
14809 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14810 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14814 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14815 substitution of a couple of function calls. But in the NAMED case,
14816 requires that languages coordinate a symbol name. It is therefore
14817 best put here in common code. */
14819 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14822 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14825 tree name
, lock
, unlock
;
14826 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14828 location_t loc
= gimple_location (stmt
);
14831 name
= gimple_omp_critical_name (stmt
);
14836 if (!critical_name_mutexes
)
14837 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14839 tree
*n
= critical_name_mutexes
->get (name
);
14844 decl
= create_tmp_var_raw (ptr_type_node
);
14846 new_str
= ACONCAT ((".gomp_critical_user_",
14847 IDENTIFIER_POINTER (name
), NULL
));
14848 DECL_NAME (decl
) = get_identifier (new_str
);
14849 TREE_PUBLIC (decl
) = 1;
14850 TREE_STATIC (decl
) = 1;
14851 DECL_COMMON (decl
) = 1;
14852 DECL_ARTIFICIAL (decl
) = 1;
14853 DECL_IGNORED_P (decl
) = 1;
14855 varpool_node::finalize_decl (decl
);
14857 critical_name_mutexes
->put (name
, decl
);
14862 /* If '#pragma omp critical' is inside offloaded region or
14863 inside function marked as offloadable, the symbol must be
14864 marked as offloadable too. */
14866 if (cgraph_node::get (current_function_decl
)->offloadable
)
14867 varpool_node::get_create (decl
)->offloadable
= 1;
14869 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14870 if (is_gimple_omp_offloaded (octx
->stmt
))
14872 varpool_node::get_create (decl
)->offloadable
= 1;
14876 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14877 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14879 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14880 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14881 build_fold_addr_expr_loc (loc
, decl
));
14885 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14886 lock
= build_call_expr_loc (loc
, lock
, 0);
14888 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14889 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14892 push_gimplify_context ();
14894 block
= make_node (BLOCK
);
14895 bind
= gimple_build_bind (NULL
, NULL
, block
);
14896 gsi_replace (gsi_p
, bind
, true);
14897 gimple_bind_add_stmt (bind
, stmt
);
14899 tbody
= gimple_bind_body (bind
);
14900 gimplify_and_add (lock
, &tbody
);
14901 gimple_bind_set_body (bind
, tbody
);
14903 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14904 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14905 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14906 gimple_omp_set_body (stmt
, NULL
);
14908 tbody
= gimple_bind_body (bind
);
14909 gimplify_and_add (unlock
, &tbody
);
14910 gimple_bind_set_body (bind
, tbody
);
14912 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14914 pop_gimplify_context (bind
);
14915 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14916 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14920 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14921 for a lastprivate clause. Given a loop control predicate of (V
14922 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14923 is appended to *DLIST, iterator initialization is appended to
14927 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14928 gimple_seq
*dlist
, struct omp_context
*ctx
)
14930 tree clauses
, cond
, vinit
;
14931 enum tree_code cond_code
;
14934 cond_code
= fd
->loop
.cond_code
;
14935 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14937 /* When possible, use a strict equality expression. This can let VRP
14938 type optimizations deduce the value and remove a copy. */
14939 if (tree_fits_shwi_p (fd
->loop
.step
))
14941 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14942 if (step
== 1 || step
== -1)
14943 cond_code
= EQ_EXPR
;
14946 tree n2
= fd
->loop
.n2
;
14947 if (fd
->collapse
> 1
14948 && TREE_CODE (n2
) != INTEGER_CST
14949 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14951 struct omp_context
*taskreg_ctx
= NULL
;
14952 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14954 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14955 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14956 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14958 if (gimple_omp_for_combined_into_p (gfor
))
14960 gcc_assert (ctx
->outer
->outer
14961 && is_parallel_ctx (ctx
->outer
->outer
));
14962 taskreg_ctx
= ctx
->outer
->outer
;
14966 struct omp_for_data outer_fd
;
14967 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14968 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14971 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14972 taskreg_ctx
= ctx
->outer
->outer
;
14974 else if (is_taskreg_ctx (ctx
->outer
))
14975 taskreg_ctx
= ctx
->outer
;
14980 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14981 OMP_CLAUSE__LOOPTEMP_
);
14982 gcc_assert (innerc
);
14983 for (i
= 0; i
< fd
->collapse
; i
++)
14985 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14986 OMP_CLAUSE__LOOPTEMP_
);
14987 gcc_assert (innerc
);
14989 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14990 OMP_CLAUSE__LOOPTEMP_
);
14992 n2
= fold_convert (TREE_TYPE (n2
),
14993 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14997 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14999 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
15001 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
15002 if (!gimple_seq_empty_p (stmts
))
15004 gimple_seq_add_seq (&stmts
, *dlist
);
15007 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
15008 vinit
= fd
->loop
.n1
;
15009 if (cond_code
== EQ_EXPR
15010 && tree_fits_shwi_p (fd
->loop
.n2
)
15011 && ! integer_zerop (fd
->loop
.n2
))
15012 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
15014 vinit
= unshare_expr (vinit
);
15016 /* Initialize the iterator variable, so that threads that don't execute
15017 any iterations don't execute the lastprivate clauses by accident. */
15018 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
15023 /* Lower code for an OMP loop directive. */
15026 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15028 tree
*rhs_p
, block
;
15029 struct omp_for_data fd
, *fdp
= NULL
;
15030 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
15032 gimple_seq omp_for_body
, body
, dlist
;
15033 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
15036 push_gimplify_context ();
15038 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
15040 block
= make_node (BLOCK
);
15041 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
15042 /* Replace at gsi right away, so that 'stmt' is no member
15043 of a sequence anymore as we're going to add to a different
15045 gsi_replace (gsi_p
, new_stmt
, true);
15047 /* Move declaration of temporaries in the loop body before we make
15049 omp_for_body
= gimple_omp_body (stmt
);
15050 if (!gimple_seq_empty_p (omp_for_body
)
15051 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
15054 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
15055 tree vars
= gimple_bind_vars (inner_bind
);
15056 gimple_bind_append_vars (new_stmt
, vars
);
15057 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
15058 keep them on the inner_bind and it's block. */
15059 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
15060 if (gimple_bind_block (inner_bind
))
15061 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
15064 if (gimple_omp_for_combined_into_p (stmt
))
15066 extract_omp_for_data (stmt
, &fd
, NULL
);
15069 /* We need two temporaries with fd.loop.v type (istart/iend)
15070 and then (fd.collapse - 1) temporaries with the same
15071 type for count2 ... countN-1 vars if not constant. */
15073 tree type
= fd
.iter_type
;
15074 if (fd
.collapse
> 1
15075 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
15076 count
+= fd
.collapse
- 1;
15078 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
15079 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
15080 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
15081 tree clauses
= *pc
;
15084 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
15085 OMP_CLAUSE__LOOPTEMP_
);
15086 for (i
= 0; i
< count
; i
++)
15091 gcc_assert (outerc
);
15092 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
15093 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
15094 OMP_CLAUSE__LOOPTEMP_
);
15098 temp
= create_tmp_var (type
);
15099 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
15101 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
15102 OMP_CLAUSE_DECL (*pc
) = temp
;
15103 pc
= &OMP_CLAUSE_CHAIN (*pc
);
15108 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15111 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
15113 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
15115 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
15117 /* Lower the header expressions. At this point, we can assume that
15118 the header is of the form:
15120 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15122 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15123 using the .omp_data_s mapping, if needed. */
15124 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
15126 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
15127 if (!is_gimple_min_invariant (*rhs_p
))
15128 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15130 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
15131 if (!is_gimple_min_invariant (*rhs_p
))
15132 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15134 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
15135 if (!is_gimple_min_invariant (*rhs_p
))
15136 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15139 /* Once lowered, extract the bounds and clauses. */
15140 extract_omp_for_data (stmt
, &fd
, NULL
);
15142 if (is_gimple_omp_oacc (ctx
->stmt
)
15143 && !ctx_in_oacc_kernels_region (ctx
))
15144 lower_oacc_head_tail (gimple_location (stmt
),
15145 gimple_omp_for_clauses (stmt
),
15146 &oacc_head
, &oacc_tail
, ctx
);
15148 /* Add OpenACC partitioning and reduction markers just before the loop */
15150 gimple_seq_add_seq (&body
, oacc_head
);
15152 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
15154 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
15155 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15156 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
15157 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
15159 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
15160 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
15161 OMP_CLAUSE_LINEAR_STEP (c
)
15162 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
15166 if (!gimple_omp_for_grid_phony (stmt
))
15167 gimple_seq_add_stmt (&body
, stmt
);
15168 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
15170 if (!gimple_omp_for_grid_phony (stmt
))
15171 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
15174 /* After the loop, add exit clauses. */
15175 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
15177 if (ctx
->cancellable
)
15178 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
15180 gimple_seq_add_seq (&body
, dlist
);
15182 body
= maybe_catch_exception (body
);
15184 if (!gimple_omp_for_grid_phony (stmt
))
15186 /* Region exit marker goes at the end of the loop body. */
15187 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
15188 maybe_add_implicit_barrier_cancel (ctx
, &body
);
15191 /* Add OpenACC joining and reduction markers just after the loop. */
15193 gimple_seq_add_seq (&body
, oacc_tail
);
15195 pop_gimplify_context (new_stmt
);
15197 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
15198 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
15199 if (BLOCK_VARS (block
))
15200 TREE_USED (block
) = 1;
15202 gimple_bind_set_body (new_stmt
, body
);
15203 gimple_omp_set_body (stmt
, NULL
);
15204 gimple_omp_for_set_pre_body (stmt
, NULL
);
15207 /* Callback for walk_stmts. Check if the current statement only contains
15208 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15211 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
15212 bool *handled_ops_p
,
15213 struct walk_stmt_info
*wi
)
15215 int *info
= (int *) wi
->info
;
15216 gimple
*stmt
= gsi_stmt (*gsi_p
);
15218 *handled_ops_p
= true;
15219 switch (gimple_code (stmt
))
15223 case GIMPLE_OMP_FOR
:
15224 case GIMPLE_OMP_SECTIONS
:
15225 *info
= *info
== 0 ? 1 : -1;
15234 struct omp_taskcopy_context
15236 /* This field must be at the beginning, as we do "inheritance": Some
15237 callback functions for tree-inline.c (e.g., omp_copy_decl)
15238 receive a copy_body_data pointer that is up-casted to an
15239 omp_context pointer. */
15245 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
15247 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
15249 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
15250 return create_tmp_var (TREE_TYPE (var
));
15256 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
15258 tree name
, new_fields
= NULL
, type
, f
;
15260 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
15261 name
= DECL_NAME (TYPE_NAME (orig_type
));
15262 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
15263 TYPE_DECL
, name
, type
);
15264 TYPE_NAME (type
) = name
;
15266 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
15268 tree new_f
= copy_node (f
);
15269 DECL_CONTEXT (new_f
) = type
;
15270 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
15271 TREE_CHAIN (new_f
) = new_fields
;
15272 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15273 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15274 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
15276 new_fields
= new_f
;
15277 tcctx
->cb
.decl_map
->put (f
, new_f
);
15279 TYPE_FIELDS (type
) = nreverse (new_fields
);
15280 layout_type (type
);
15284 /* Create task copyfn. */
15287 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
15289 struct function
*child_cfun
;
15290 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
15291 tree record_type
, srecord_type
, bind
, list
;
15292 bool record_needs_remap
= false, srecord_needs_remap
= false;
15294 struct omp_taskcopy_context tcctx
;
15295 location_t loc
= gimple_location (task_stmt
);
15297 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
15298 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
15299 gcc_assert (child_cfun
->cfg
== NULL
);
15300 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
15302 /* Reset DECL_CONTEXT on function arguments. */
15303 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
15304 DECL_CONTEXT (t
) = child_fn
;
15306 /* Populate the function. */
15307 push_gimplify_context ();
15308 push_cfun (child_cfun
);
15310 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
15311 TREE_SIDE_EFFECTS (bind
) = 1;
15313 DECL_SAVED_TREE (child_fn
) = bind
;
15314 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
15316 /* Remap src and dst argument types if needed. */
15317 record_type
= ctx
->record_type
;
15318 srecord_type
= ctx
->srecord_type
;
15319 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
15320 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15322 record_needs_remap
= true;
15325 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
15326 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15328 srecord_needs_remap
= true;
15332 if (record_needs_remap
|| srecord_needs_remap
)
15334 memset (&tcctx
, '\0', sizeof (tcctx
));
15335 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
15336 tcctx
.cb
.dst_fn
= child_fn
;
15337 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
15338 gcc_checking_assert (tcctx
.cb
.src_node
);
15339 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
15340 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
15341 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
15342 tcctx
.cb
.eh_lp_nr
= 0;
15343 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
15344 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
15347 if (record_needs_remap
)
15348 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
15349 if (srecord_needs_remap
)
15350 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
15353 tcctx
.cb
.decl_map
= NULL
;
15355 arg
= DECL_ARGUMENTS (child_fn
);
15356 TREE_TYPE (arg
) = build_pointer_type (record_type
);
15357 sarg
= DECL_CHAIN (arg
);
15358 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
15360 /* First pass: initialize temporaries used in record_type and srecord_type
15361 sizes and field offsets. */
15362 if (tcctx
.cb
.decl_map
)
15363 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15364 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15368 decl
= OMP_CLAUSE_DECL (c
);
15369 p
= tcctx
.cb
.decl_map
->get (decl
);
15372 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15373 sf
= (tree
) n
->value
;
15374 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15375 src
= build_simple_mem_ref_loc (loc
, sarg
);
15376 src
= omp_build_component_ref (src
, sf
);
15377 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
15378 append_to_statement_list (t
, &list
);
15381 /* Second pass: copy shared var pointers and copy construct non-VLA
15382 firstprivate vars. */
15383 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15384 switch (OMP_CLAUSE_CODE (c
))
15386 splay_tree_key key
;
15387 case OMP_CLAUSE_SHARED
:
15388 decl
= OMP_CLAUSE_DECL (c
);
15389 key
= (splay_tree_key
) decl
;
15390 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
15391 key
= (splay_tree_key
) &DECL_UID (decl
);
15392 n
= splay_tree_lookup (ctx
->field_map
, key
);
15395 f
= (tree
) n
->value
;
15396 if (tcctx
.cb
.decl_map
)
15397 f
= *tcctx
.cb
.decl_map
->get (f
);
15398 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
15399 sf
= (tree
) n
->value
;
15400 if (tcctx
.cb
.decl_map
)
15401 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15402 src
= build_simple_mem_ref_loc (loc
, sarg
);
15403 src
= omp_build_component_ref (src
, sf
);
15404 dst
= build_simple_mem_ref_loc (loc
, arg
);
15405 dst
= omp_build_component_ref (dst
, f
);
15406 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15407 append_to_statement_list (t
, &list
);
15409 case OMP_CLAUSE_FIRSTPRIVATE
:
15410 decl
= OMP_CLAUSE_DECL (c
);
15411 if (is_variable_sized (decl
))
15413 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15416 f
= (tree
) n
->value
;
15417 if (tcctx
.cb
.decl_map
)
15418 f
= *tcctx
.cb
.decl_map
->get (f
);
15419 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15422 sf
= (tree
) n
->value
;
15423 if (tcctx
.cb
.decl_map
)
15424 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15425 src
= build_simple_mem_ref_loc (loc
, sarg
);
15426 src
= omp_build_component_ref (src
, sf
);
15427 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
15428 src
= build_simple_mem_ref_loc (loc
, src
);
15432 dst
= build_simple_mem_ref_loc (loc
, arg
);
15433 dst
= omp_build_component_ref (dst
, f
);
15434 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15435 append_to_statement_list (t
, &list
);
15437 case OMP_CLAUSE_PRIVATE
:
15438 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
15440 decl
= OMP_CLAUSE_DECL (c
);
15441 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15442 f
= (tree
) n
->value
;
15443 if (tcctx
.cb
.decl_map
)
15444 f
= *tcctx
.cb
.decl_map
->get (f
);
15445 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15448 sf
= (tree
) n
->value
;
15449 if (tcctx
.cb
.decl_map
)
15450 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15451 src
= build_simple_mem_ref_loc (loc
, sarg
);
15452 src
= omp_build_component_ref (src
, sf
);
15453 if (use_pointer_for_field (decl
, NULL
))
15454 src
= build_simple_mem_ref_loc (loc
, src
);
15458 dst
= build_simple_mem_ref_loc (loc
, arg
);
15459 dst
= omp_build_component_ref (dst
, f
);
15460 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15461 append_to_statement_list (t
, &list
);
15467 /* Last pass: handle VLA firstprivates. */
15468 if (tcctx
.cb
.decl_map
)
15469 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15470 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15474 decl
= OMP_CLAUSE_DECL (c
);
15475 if (!is_variable_sized (decl
))
15477 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15480 f
= (tree
) n
->value
;
15481 f
= *tcctx
.cb
.decl_map
->get (f
);
15482 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
15483 ind
= DECL_VALUE_EXPR (decl
);
15484 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
15485 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
15486 n
= splay_tree_lookup (ctx
->sfield_map
,
15487 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15488 sf
= (tree
) n
->value
;
15489 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15490 src
= build_simple_mem_ref_loc (loc
, sarg
);
15491 src
= omp_build_component_ref (src
, sf
);
15492 src
= build_simple_mem_ref_loc (loc
, src
);
15493 dst
= build_simple_mem_ref_loc (loc
, arg
);
15494 dst
= omp_build_component_ref (dst
, f
);
15495 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15496 append_to_statement_list (t
, &list
);
15497 n
= splay_tree_lookup (ctx
->field_map
,
15498 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15499 df
= (tree
) n
->value
;
15500 df
= *tcctx
.cb
.decl_map
->get (df
);
15501 ptr
= build_simple_mem_ref_loc (loc
, arg
);
15502 ptr
= omp_build_component_ref (ptr
, df
);
15503 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
15504 build_fold_addr_expr_loc (loc
, dst
));
15505 append_to_statement_list (t
, &list
);
15508 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
15509 append_to_statement_list (t
, &list
);
15511 if (tcctx
.cb
.decl_map
)
15512 delete tcctx
.cb
.decl_map
;
15513 pop_gimplify_context (NULL
);
15514 BIND_EXPR_BODY (bind
) = list
;
15519 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
15523 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
15525 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
15526 gcc_assert (clauses
);
15527 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15528 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
15529 switch (OMP_CLAUSE_DEPEND_KIND (c
))
15531 case OMP_CLAUSE_DEPEND_IN
:
15534 case OMP_CLAUSE_DEPEND_OUT
:
15535 case OMP_CLAUSE_DEPEND_INOUT
:
15538 case OMP_CLAUSE_DEPEND_SOURCE
:
15539 case OMP_CLAUSE_DEPEND_SINK
:
15542 gcc_unreachable ();
15544 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
15545 tree array
= create_tmp_var (type
);
15546 TREE_ADDRESSABLE (array
) = 1;
15547 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
15549 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
15550 gimple_seq_add_stmt (iseq
, g
);
15551 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
15553 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
15554 gimple_seq_add_stmt (iseq
, g
);
15555 for (i
= 0; i
< 2; i
++)
15557 if ((i
? n_in
: n_out
) == 0)
15559 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15560 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
15561 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
15563 tree t
= OMP_CLAUSE_DECL (c
);
15564 t
= fold_convert (ptr_type_node
, t
);
15565 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
15566 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
15567 NULL_TREE
, NULL_TREE
);
15568 g
= gimple_build_assign (r
, t
);
15569 gimple_seq_add_stmt (iseq
, g
);
15572 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
15573 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
15574 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
15576 tree clobber
= build_constructor (type
, NULL
);
15577 TREE_THIS_VOLATILE (clobber
) = 1;
15578 g
= gimple_build_assign (array
, clobber
);
15579 gimple_seq_add_stmt (oseq
, g
);
15582 /* Lower the OpenMP parallel or task directive in the current statement
15583 in GSI_P. CTX holds context information for the directive. */
15586 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15590 gimple
*stmt
= gsi_stmt (*gsi_p
);
15591 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
15592 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
15593 location_t loc
= gimple_location (stmt
);
15595 clauses
= gimple_omp_taskreg_clauses (stmt
);
15597 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
15598 par_body
= gimple_bind_body (par_bind
);
15599 child_fn
= ctx
->cb
.dst_fn
;
15600 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15601 && !gimple_omp_parallel_combined_p (stmt
))
15603 struct walk_stmt_info wi
;
15606 memset (&wi
, 0, sizeof (wi
));
15608 wi
.val_only
= true;
15609 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
15611 gimple_omp_parallel_set_combined_p (stmt
, true);
15613 gimple_seq dep_ilist
= NULL
;
15614 gimple_seq dep_olist
= NULL
;
15615 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
15616 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15618 push_gimplify_context ();
15619 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15620 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
15621 &dep_ilist
, &dep_olist
);
15624 if (ctx
->srecord_type
)
15625 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
15627 push_gimplify_context ();
15632 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15633 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
15634 if (phony_construct
&& ctx
->record_type
)
15636 gcc_checking_assert (!ctx
->receiver_decl
);
15637 ctx
->receiver_decl
= create_tmp_var
15638 (build_reference_type (ctx
->record_type
), ".omp_rec");
15640 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
15641 lower_omp (&par_body
, ctx
);
15642 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
15643 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
15645 /* Declare all the variables created by mapping and the variables
15646 declared in the scope of the parallel body. */
15647 record_vars_into (ctx
->block_vars
, child_fn
);
15648 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
15650 if (ctx
->record_type
)
15653 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
15654 : ctx
->record_type
, ".omp_data_o");
15655 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15656 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15657 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
15662 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
15663 lower_send_shared_vars (&ilist
, &olist
, ctx
);
15665 if (ctx
->record_type
)
15667 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
15668 TREE_THIS_VOLATILE (clobber
) = 1;
15669 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15673 /* Once all the expansions are done, sequence all the different
15674 fragments inside gimple_omp_body. */
15678 if (ctx
->record_type
)
15680 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15681 /* fixup_child_record_type might have changed receiver_decl's type. */
15682 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15683 gimple_seq_add_stmt (&new_body
,
15684 gimple_build_assign (ctx
->receiver_decl
, t
));
15687 gimple_seq_add_seq (&new_body
, par_ilist
);
15688 gimple_seq_add_seq (&new_body
, par_body
);
15689 gimple_seq_add_seq (&new_body
, par_rlist
);
15690 if (ctx
->cancellable
)
15691 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
15692 gimple_seq_add_seq (&new_body
, par_olist
);
15693 new_body
= maybe_catch_exception (new_body
);
15694 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
15695 gimple_seq_add_stmt (&new_body
,
15696 gimple_build_omp_continue (integer_zero_node
,
15697 integer_zero_node
));
15698 if (!phony_construct
)
15700 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15701 gimple_omp_set_body (stmt
, new_body
);
15704 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
15705 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15706 gimple_bind_add_seq (bind
, ilist
);
15707 if (!phony_construct
)
15708 gimple_bind_add_stmt (bind
, stmt
);
15710 gimple_bind_add_seq (bind
, new_body
);
15711 gimple_bind_add_seq (bind
, olist
);
15713 pop_gimplify_context (NULL
);
15717 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15718 gimple_bind_add_stmt (dep_bind
, bind
);
15719 gimple_bind_add_seq (dep_bind
, dep_olist
);
15720 pop_gimplify_context (dep_bind
);
15724 /* Lower the GIMPLE_OMP_TARGET in the current statement
15725 in GSI_P. CTX holds context information for the directive. */
15728 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15731 tree child_fn
, t
, c
;
15732 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
15733 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
15734 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
15735 location_t loc
= gimple_location (stmt
);
15736 bool offloaded
, data_region
;
15737 unsigned int map_cnt
= 0;
15739 offloaded
= is_gimple_omp_offloaded (stmt
);
15740 switch (gimple_omp_target_kind (stmt
))
15742 case GF_OMP_TARGET_KIND_REGION
:
15743 case GF_OMP_TARGET_KIND_UPDATE
:
15744 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15745 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15746 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15747 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15748 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15749 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15750 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
15751 data_region
= false;
15753 case GF_OMP_TARGET_KIND_DATA
:
15754 case GF_OMP_TARGET_KIND_OACC_DATA
:
15755 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
15756 data_region
= true;
15759 gcc_unreachable ();
15762 clauses
= gimple_omp_target_clauses (stmt
);
15764 gimple_seq dep_ilist
= NULL
;
15765 gimple_seq dep_olist
= NULL
;
15766 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15768 push_gimplify_context ();
15769 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15770 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15771 &dep_ilist
, &dep_olist
);
15778 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15779 tgt_body
= gimple_bind_body (tgt_bind
);
15781 else if (data_region
)
15782 tgt_body
= gimple_omp_body (stmt
);
15783 child_fn
= ctx
->cb
.dst_fn
;
15785 push_gimplify_context ();
15788 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15789 switch (OMP_CLAUSE_CODE (c
))
15795 case OMP_CLAUSE_MAP
:
15797 /* First check what we're prepared to handle in the following. */
15798 switch (OMP_CLAUSE_MAP_KIND (c
))
15800 case GOMP_MAP_ALLOC
:
15802 case GOMP_MAP_FROM
:
15803 case GOMP_MAP_TOFROM
:
15804 case GOMP_MAP_POINTER
:
15805 case GOMP_MAP_TO_PSET
:
15806 case GOMP_MAP_DELETE
:
15807 case GOMP_MAP_RELEASE
:
15808 case GOMP_MAP_ALWAYS_TO
:
15809 case GOMP_MAP_ALWAYS_FROM
:
15810 case GOMP_MAP_ALWAYS_TOFROM
:
15811 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15812 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
15813 case GOMP_MAP_STRUCT
:
15814 case GOMP_MAP_ALWAYS_POINTER
:
15816 case GOMP_MAP_FORCE_ALLOC
:
15817 case GOMP_MAP_FORCE_TO
:
15818 case GOMP_MAP_FORCE_FROM
:
15819 case GOMP_MAP_FORCE_TOFROM
:
15820 case GOMP_MAP_FORCE_PRESENT
:
15821 case GOMP_MAP_FORCE_DEVICEPTR
:
15822 case GOMP_MAP_DEVICE_RESIDENT
:
15823 case GOMP_MAP_LINK
:
15824 gcc_assert (is_gimple_omp_oacc (stmt
));
15827 gcc_unreachable ();
15831 case OMP_CLAUSE_TO
:
15832 case OMP_CLAUSE_FROM
:
15834 var
= OMP_CLAUSE_DECL (c
);
15837 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15838 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15839 && (OMP_CLAUSE_MAP_KIND (c
)
15840 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15845 if (DECL_SIZE (var
)
15846 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15848 tree var2
= DECL_VALUE_EXPR (var
);
15849 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15850 var2
= TREE_OPERAND (var2
, 0);
15851 gcc_assert (DECL_P (var2
));
15856 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15857 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15858 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15860 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15862 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15863 && varpool_node::get_create (var
)->offloadable
)
15866 tree type
= build_pointer_type (TREE_TYPE (var
));
15867 tree new_var
= lookup_decl (var
, ctx
);
15868 x
= create_tmp_var_raw (type
, get_name (new_var
));
15869 gimple_add_tmp_var (x
);
15870 x
= build_simple_mem_ref (x
);
15871 SET_DECL_VALUE_EXPR (new_var
, x
);
15872 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15877 if (!maybe_lookup_field (var
, ctx
))
15880 /* Don't remap oacc parallel reduction variables, because the
15881 intermediate result must be local to each gang. */
15882 if (offloaded
&& !(OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15883 && OMP_CLAUSE_MAP_IN_REDUCTION (c
)))
15885 x
= build_receiver_ref (var
, true, ctx
);
15886 tree new_var
= lookup_decl (var
, ctx
);
15888 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15889 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15890 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15891 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15892 x
= build_simple_mem_ref (x
);
15893 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15895 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15896 if (is_reference (new_var
))
15898 /* Create a local object to hold the instance
15900 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15901 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15902 tree inst
= create_tmp_var (type
, id
);
15903 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15904 x
= build_fold_addr_expr (inst
);
15906 gimplify_assign (new_var
, x
, &fplist
);
15908 else if (DECL_P (new_var
))
15910 SET_DECL_VALUE_EXPR (new_var
, x
);
15911 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15914 gcc_unreachable ();
15919 case OMP_CLAUSE_FIRSTPRIVATE
:
15920 if (is_oacc_parallel (ctx
))
15921 goto oacc_firstprivate
;
15923 var
= OMP_CLAUSE_DECL (c
);
15924 if (!is_reference (var
)
15925 && !is_gimple_reg_type (TREE_TYPE (var
)))
15927 tree new_var
= lookup_decl (var
, ctx
);
15928 if (is_variable_sized (var
))
15930 tree pvar
= DECL_VALUE_EXPR (var
);
15931 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15932 pvar
= TREE_OPERAND (pvar
, 0);
15933 gcc_assert (DECL_P (pvar
));
15934 tree new_pvar
= lookup_decl (pvar
, ctx
);
15935 x
= build_fold_indirect_ref (new_pvar
);
15936 TREE_THIS_NOTRAP (x
) = 1;
15939 x
= build_receiver_ref (var
, true, ctx
);
15940 SET_DECL_VALUE_EXPR (new_var
, x
);
15941 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15945 case OMP_CLAUSE_PRIVATE
:
15946 if (is_gimple_omp_oacc (ctx
->stmt
))
15948 var
= OMP_CLAUSE_DECL (c
);
15949 if (is_variable_sized (var
))
15951 tree new_var
= lookup_decl (var
, ctx
);
15952 tree pvar
= DECL_VALUE_EXPR (var
);
15953 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15954 pvar
= TREE_OPERAND (pvar
, 0);
15955 gcc_assert (DECL_P (pvar
));
15956 tree new_pvar
= lookup_decl (pvar
, ctx
);
15957 x
= build_fold_indirect_ref (new_pvar
);
15958 TREE_THIS_NOTRAP (x
) = 1;
15959 SET_DECL_VALUE_EXPR (new_var
, x
);
15960 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15964 case OMP_CLAUSE_USE_DEVICE_PTR
:
15965 case OMP_CLAUSE_IS_DEVICE_PTR
:
15966 var
= OMP_CLAUSE_DECL (c
);
15968 if (is_variable_sized (var
))
15970 tree new_var
= lookup_decl (var
, ctx
);
15971 tree pvar
= DECL_VALUE_EXPR (var
);
15972 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15973 pvar
= TREE_OPERAND (pvar
, 0);
15974 gcc_assert (DECL_P (pvar
));
15975 tree new_pvar
= lookup_decl (pvar
, ctx
);
15976 x
= build_fold_indirect_ref (new_pvar
);
15977 TREE_THIS_NOTRAP (x
) = 1;
15978 SET_DECL_VALUE_EXPR (new_var
, x
);
15979 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15981 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15983 tree new_var
= lookup_decl (var
, ctx
);
15984 tree type
= build_pointer_type (TREE_TYPE (var
));
15985 x
= create_tmp_var_raw (type
, get_name (new_var
));
15986 gimple_add_tmp_var (x
);
15987 x
= build_simple_mem_ref (x
);
15988 SET_DECL_VALUE_EXPR (new_var
, x
);
15989 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15993 tree new_var
= lookup_decl (var
, ctx
);
15994 x
= create_tmp_var_raw (TREE_TYPE (new_var
), get_name (new_var
));
15995 gimple_add_tmp_var (x
);
15996 SET_DECL_VALUE_EXPR (new_var
, x
);
15997 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
16004 target_nesting_level
++;
16005 lower_omp (&tgt_body
, ctx
);
16006 target_nesting_level
--;
16008 else if (data_region
)
16009 lower_omp (&tgt_body
, ctx
);
16013 /* Declare all the variables created by mapping and the variables
16014 declared in the scope of the target body. */
16015 record_vars_into (ctx
->block_vars
, child_fn
);
16016 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
16021 if (ctx
->record_type
)
16024 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
16025 DECL_NAMELESS (ctx
->sender_decl
) = 1;
16026 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
16027 t
= make_tree_vec (3);
16028 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
16029 TREE_VEC_ELT (t
, 1)
16030 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
16031 ".omp_data_sizes");
16032 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
16033 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
16034 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
16035 tree tkind_type
= short_unsigned_type_node
;
16036 int talign_shift
= 8;
16037 TREE_VEC_ELT (t
, 2)
16038 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
16039 ".omp_data_kinds");
16040 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
16041 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
16042 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
16043 gimple_omp_target_set_data_arg (stmt
, t
);
16045 vec
<constructor_elt
, va_gc
> *vsize
;
16046 vec
<constructor_elt
, va_gc
> *vkind
;
16047 vec_alloc (vsize
, map_cnt
);
16048 vec_alloc (vkind
, map_cnt
);
16049 unsigned int map_idx
= 0;
16051 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16052 switch (OMP_CLAUSE_CODE (c
))
16054 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
16055 unsigned int talign
;
16060 case OMP_CLAUSE_MAP
:
16061 case OMP_CLAUSE_TO
:
16062 case OMP_CLAUSE_FROM
:
16063 oacc_firstprivate_map
:
16065 ovar
= OMP_CLAUSE_DECL (c
);
16066 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16067 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16068 || (OMP_CLAUSE_MAP_KIND (c
)
16069 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16071 if (!DECL_P (ovar
))
16073 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16074 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
16076 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
16077 == get_base_address (ovar
));
16078 nc
= OMP_CLAUSE_CHAIN (c
);
16079 ovar
= OMP_CLAUSE_DECL (nc
);
16083 tree x
= build_sender_ref (ovar
, ctx
);
16085 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
16086 gimplify_assign (x
, v
, &ilist
);
16092 if (DECL_SIZE (ovar
)
16093 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
16095 tree ovar2
= DECL_VALUE_EXPR (ovar
);
16096 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
16097 ovar2
= TREE_OPERAND (ovar2
, 0);
16098 gcc_assert (DECL_P (ovar2
));
16101 if (!maybe_lookup_field (ovar
, ctx
))
16105 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
16106 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
16107 talign
= DECL_ALIGN_UNIT (ovar
);
16110 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16111 x
= build_sender_ref (ovar
, ctx
);
16113 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
16114 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
16115 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
16116 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
16118 gcc_assert (offloaded
);
16120 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
16121 mark_addressable (avar
);
16122 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
16123 talign
= DECL_ALIGN_UNIT (avar
);
16124 avar
= build_fold_addr_expr (avar
);
16125 gimplify_assign (x
, avar
, &ilist
);
16127 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16129 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
16130 if (!is_reference (var
))
16132 if (is_gimple_reg (var
)
16133 && OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16134 TREE_NO_WARNING (var
) = 1;
16135 var
= build_fold_addr_expr (var
);
16138 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16139 gimplify_assign (x
, var
, &ilist
);
16141 else if (is_gimple_reg (var
))
16143 gcc_assert (offloaded
);
16144 tree avar
= create_tmp_var (TREE_TYPE (var
));
16145 mark_addressable (avar
);
16146 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
16147 if (GOMP_MAP_COPY_TO_P (map_kind
)
16148 || map_kind
== GOMP_MAP_POINTER
16149 || map_kind
== GOMP_MAP_TO_PSET
16150 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16152 /* If we need to initialize a temporary
16153 with VAR because it is not addressable, and
16154 the variable hasn't been initialized yet, then
16155 we'll get a warning for the store to avar.
16156 Don't warn in that case, the mapping might
16158 TREE_NO_WARNING (var
) = 1;
16159 gimplify_assign (avar
, var
, &ilist
);
16161 avar
= build_fold_addr_expr (avar
);
16162 gimplify_assign (x
, avar
, &ilist
);
16163 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
16164 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16165 && !TYPE_READONLY (TREE_TYPE (var
)))
16167 x
= unshare_expr (x
);
16168 x
= build_simple_mem_ref (x
);
16169 gimplify_assign (var
, x
, &olist
);
16174 var
= build_fold_addr_expr (var
);
16175 gimplify_assign (x
, var
, &ilist
);
16179 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16181 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16182 s
= TREE_TYPE (ovar
);
16183 if (TREE_CODE (s
) == REFERENCE_TYPE
)
16185 s
= TYPE_SIZE_UNIT (s
);
16188 s
= OMP_CLAUSE_SIZE (c
);
16189 if (s
== NULL_TREE
)
16190 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16191 s
= fold_convert (size_type_node
, s
);
16192 purpose
= size_int (map_idx
++);
16193 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16194 if (TREE_CODE (s
) != INTEGER_CST
)
16195 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16197 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
16198 switch (OMP_CLAUSE_CODE (c
))
16200 case OMP_CLAUSE_MAP
:
16201 tkind
= OMP_CLAUSE_MAP_KIND (c
);
16202 tkind_zero
= tkind
;
16203 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
16206 case GOMP_MAP_ALLOC
:
16208 case GOMP_MAP_FROM
:
16209 case GOMP_MAP_TOFROM
:
16210 case GOMP_MAP_ALWAYS_TO
:
16211 case GOMP_MAP_ALWAYS_FROM
:
16212 case GOMP_MAP_ALWAYS_TOFROM
:
16213 case GOMP_MAP_RELEASE
:
16214 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
16216 case GOMP_MAP_DELETE
:
16217 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
16221 if (tkind_zero
!= tkind
)
16223 if (integer_zerop (s
))
16224 tkind
= tkind_zero
;
16225 else if (integer_nonzerop (s
))
16226 tkind_zero
= tkind
;
16229 case OMP_CLAUSE_FIRSTPRIVATE
:
16230 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16231 tkind
= GOMP_MAP_TO
;
16232 tkind_zero
= tkind
;
16234 case OMP_CLAUSE_TO
:
16235 tkind
= GOMP_MAP_TO
;
16236 tkind_zero
= tkind
;
16238 case OMP_CLAUSE_FROM
:
16239 tkind
= GOMP_MAP_FROM
;
16240 tkind_zero
= tkind
;
16243 gcc_unreachable ();
16245 gcc_checking_assert (tkind
16246 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16247 gcc_checking_assert (tkind_zero
16248 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16249 talign
= ceil_log2 (talign
);
16250 tkind
|= talign
<< talign_shift
;
16251 tkind_zero
|= talign
<< talign_shift
;
16252 gcc_checking_assert (tkind
16253 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16254 gcc_checking_assert (tkind_zero
16255 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16256 if (tkind
== tkind_zero
)
16257 x
= build_int_cstu (tkind_type
, tkind
);
16260 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
16261 x
= build3 (COND_EXPR
, tkind_type
,
16262 fold_build2 (EQ_EXPR
, boolean_type_node
,
16263 unshare_expr (s
), size_zero_node
),
16264 build_int_cstu (tkind_type
, tkind_zero
),
16265 build_int_cstu (tkind_type
, tkind
));
16267 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
16272 case OMP_CLAUSE_FIRSTPRIVATE
:
16273 if (is_oacc_parallel (ctx
))
16274 goto oacc_firstprivate_map
;
16275 ovar
= OMP_CLAUSE_DECL (c
);
16276 if (is_reference (ovar
))
16277 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16279 talign
= DECL_ALIGN_UNIT (ovar
);
16280 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16281 x
= build_sender_ref (ovar
, ctx
);
16282 tkind
= GOMP_MAP_FIRSTPRIVATE
;
16283 type
= TREE_TYPE (ovar
);
16284 if (is_reference (ovar
))
16285 type
= TREE_TYPE (type
);
16286 if ((INTEGRAL_TYPE_P (type
)
16287 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16288 || TREE_CODE (type
) == POINTER_TYPE
)
16290 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16292 if (is_reference (var
))
16293 t
= build_simple_mem_ref (var
);
16294 else if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16295 TREE_NO_WARNING (var
) = 1;
16296 if (TREE_CODE (type
) != POINTER_TYPE
)
16297 t
= fold_convert (pointer_sized_int_node
, t
);
16298 t
= fold_convert (TREE_TYPE (x
), t
);
16299 gimplify_assign (x
, t
, &ilist
);
16301 else if (is_reference (var
))
16302 gimplify_assign (x
, var
, &ilist
);
16303 else if (is_gimple_reg (var
))
16305 tree avar
= create_tmp_var (TREE_TYPE (var
));
16306 mark_addressable (avar
);
16307 if (OMP_CLAUSE_FIRSTPRIVATE_IMPLICIT (c
))
16308 TREE_NO_WARNING (var
) = 1;
16309 gimplify_assign (avar
, var
, &ilist
);
16310 avar
= build_fold_addr_expr (avar
);
16311 gimplify_assign (x
, avar
, &ilist
);
16315 var
= build_fold_addr_expr (var
);
16316 gimplify_assign (x
, var
, &ilist
);
16318 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
16320 else if (is_reference (var
))
16321 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16323 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16324 s
= fold_convert (size_type_node
, s
);
16325 purpose
= size_int (map_idx
++);
16326 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16327 if (TREE_CODE (s
) != INTEGER_CST
)
16328 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16330 gcc_checking_assert (tkind
16331 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16332 talign
= ceil_log2 (talign
);
16333 tkind
|= talign
<< talign_shift
;
16334 gcc_checking_assert (tkind
16335 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16336 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16337 build_int_cstu (tkind_type
, tkind
));
16340 case OMP_CLAUSE_USE_DEVICE_PTR
:
16341 case OMP_CLAUSE_IS_DEVICE_PTR
:
16342 ovar
= OMP_CLAUSE_DECL (c
);
16343 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16344 x
= build_sender_ref (ovar
, ctx
);
16345 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16346 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
16348 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16349 type
= TREE_TYPE (ovar
);
16350 if (TREE_CODE (type
) == ARRAY_TYPE
)
16351 var
= build_fold_addr_expr (var
);
16354 if (is_reference (ovar
))
16356 type
= TREE_TYPE (type
);
16357 if (TREE_CODE (type
) != ARRAY_TYPE
)
16358 var
= build_simple_mem_ref (var
);
16359 var
= fold_convert (TREE_TYPE (x
), var
);
16362 gimplify_assign (x
, var
, &ilist
);
16364 purpose
= size_int (map_idx
++);
16365 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16366 gcc_checking_assert (tkind
16367 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16368 gcc_checking_assert (tkind
16369 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16370 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16371 build_int_cstu (tkind_type
, tkind
));
16375 gcc_assert (map_idx
== map_cnt
);
16377 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
16378 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
16379 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
16380 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
16381 for (int i
= 1; i
<= 2; i
++)
16382 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
16384 gimple_seq initlist
= NULL
;
16385 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
16386 TREE_VEC_ELT (t
, i
)),
16387 &initlist
, true, NULL_TREE
);
16388 gimple_seq_add_seq (&ilist
, initlist
);
16390 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
16392 TREE_THIS_VOLATILE (clobber
) = 1;
16393 gimple_seq_add_stmt (&olist
,
16394 gimple_build_assign (TREE_VEC_ELT (t
, i
),
16398 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
16399 TREE_THIS_VOLATILE (clobber
) = 1;
16400 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
16404 /* Once all the expansions are done, sequence all the different
16405 fragments inside gimple_omp_body. */
16410 && ctx
->record_type
)
16412 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
16413 /* fixup_child_record_type might have changed receiver_decl's type. */
16414 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
16415 gimple_seq_add_stmt (&new_body
,
16416 gimple_build_assign (ctx
->receiver_decl
, t
));
16418 gimple_seq_add_seq (&new_body
, fplist
);
16420 if (offloaded
|| data_region
)
16422 tree prev
= NULL_TREE
;
16423 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16424 switch (OMP_CLAUSE_CODE (c
))
16429 case OMP_CLAUSE_FIRSTPRIVATE
:
16430 if (is_gimple_omp_oacc (ctx
->stmt
))
16432 var
= OMP_CLAUSE_DECL (c
);
16433 if (is_reference (var
)
16434 || is_gimple_reg_type (TREE_TYPE (var
)))
16436 tree new_var
= lookup_decl (var
, ctx
);
16438 type
= TREE_TYPE (var
);
16439 if (is_reference (var
))
16440 type
= TREE_TYPE (type
);
16441 if ((INTEGRAL_TYPE_P (type
)
16442 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16443 || TREE_CODE (type
) == POINTER_TYPE
)
16445 x
= build_receiver_ref (var
, false, ctx
);
16446 if (TREE_CODE (type
) != POINTER_TYPE
)
16447 x
= fold_convert (pointer_sized_int_node
, x
);
16448 x
= fold_convert (type
, x
);
16449 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16451 if (is_reference (var
))
16453 tree v
= create_tmp_var_raw (type
, get_name (var
));
16454 gimple_add_tmp_var (v
);
16455 TREE_ADDRESSABLE (v
) = 1;
16456 gimple_seq_add_stmt (&new_body
,
16457 gimple_build_assign (v
, x
));
16458 x
= build_fold_addr_expr (v
);
16460 gimple_seq_add_stmt (&new_body
,
16461 gimple_build_assign (new_var
, x
));
16465 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
16466 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16468 gimple_seq_add_stmt (&new_body
,
16469 gimple_build_assign (new_var
, x
));
16472 else if (is_variable_sized (var
))
16474 tree pvar
= DECL_VALUE_EXPR (var
);
16475 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16476 pvar
= TREE_OPERAND (pvar
, 0);
16477 gcc_assert (DECL_P (pvar
));
16478 tree new_var
= lookup_decl (pvar
, ctx
);
16479 x
= build_receiver_ref (var
, false, ctx
);
16480 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16481 gimple_seq_add_stmt (&new_body
,
16482 gimple_build_assign (new_var
, x
));
16485 case OMP_CLAUSE_PRIVATE
:
16486 if (is_gimple_omp_oacc (ctx
->stmt
))
16488 var
= OMP_CLAUSE_DECL (c
);
16489 if (is_reference (var
))
16491 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16492 tree new_var
= lookup_decl (var
, ctx
);
16493 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16494 if (TREE_CONSTANT (x
))
16496 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
16498 gimple_add_tmp_var (x
);
16499 TREE_ADDRESSABLE (x
) = 1;
16500 x
= build_fold_addr_expr_loc (clause_loc
, x
);
16505 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16506 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16507 gimple_seq_add_stmt (&new_body
,
16508 gimple_build_assign (new_var
, x
));
16511 case OMP_CLAUSE_USE_DEVICE_PTR
:
16512 case OMP_CLAUSE_IS_DEVICE_PTR
:
16513 var
= OMP_CLAUSE_DECL (c
);
16514 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16515 x
= build_sender_ref (var
, ctx
);
16517 x
= build_receiver_ref (var
, false, ctx
);
16518 if (is_variable_sized (var
))
16520 tree pvar
= DECL_VALUE_EXPR (var
);
16521 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16522 pvar
= TREE_OPERAND (pvar
, 0);
16523 gcc_assert (DECL_P (pvar
));
16524 tree new_var
= lookup_decl (pvar
, ctx
);
16525 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16526 gimple_seq_add_stmt (&new_body
,
16527 gimple_build_assign (new_var
, x
));
16529 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16531 tree new_var
= lookup_decl (var
, ctx
);
16532 new_var
= DECL_VALUE_EXPR (new_var
);
16533 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
16534 new_var
= TREE_OPERAND (new_var
, 0);
16535 gcc_assert (DECL_P (new_var
));
16536 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16537 gimple_seq_add_stmt (&new_body
,
16538 gimple_build_assign (new_var
, x
));
16542 tree type
= TREE_TYPE (var
);
16543 tree new_var
= lookup_decl (var
, ctx
);
16544 if (is_reference (var
))
16546 type
= TREE_TYPE (type
);
16547 if (TREE_CODE (type
) != ARRAY_TYPE
)
16549 tree v
= create_tmp_var_raw (type
, get_name (var
));
16550 gimple_add_tmp_var (v
);
16551 TREE_ADDRESSABLE (v
) = 1;
16552 x
= fold_convert (type
, x
);
16553 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16555 gimple_seq_add_stmt (&new_body
,
16556 gimple_build_assign (v
, x
));
16557 x
= build_fold_addr_expr (v
);
16560 new_var
= DECL_VALUE_EXPR (new_var
);
16561 x
= fold_convert (TREE_TYPE (new_var
), x
);
16562 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16563 gimple_seq_add_stmt (&new_body
,
16564 gimple_build_assign (new_var
, x
));
16568 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16569 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16570 are already handled. Similarly OMP_CLAUSE_PRIVATE for VLAs
16571 or references to VLAs. */
16572 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16573 switch (OMP_CLAUSE_CODE (c
))
16578 case OMP_CLAUSE_MAP
:
16579 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16580 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16582 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16583 HOST_WIDE_INT offset
= 0;
16585 var
= OMP_CLAUSE_DECL (c
);
16587 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
16588 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
16590 && varpool_node::get_create (var
)->offloadable
)
16592 if (TREE_CODE (var
) == INDIRECT_REF
16593 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
16594 var
= TREE_OPERAND (var
, 0);
16595 if (TREE_CODE (var
) == COMPONENT_REF
)
16597 var
= get_addr_base_and_unit_offset (var
, &offset
);
16598 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
16600 else if (DECL_SIZE (var
)
16601 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
16603 tree var2
= DECL_VALUE_EXPR (var
);
16604 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
16605 var2
= TREE_OPERAND (var2
, 0);
16606 gcc_assert (DECL_P (var2
));
16609 tree new_var
= lookup_decl (var
, ctx
), x
;
16610 tree type
= TREE_TYPE (new_var
);
16612 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
16613 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
16616 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
16618 new_var
= build2 (MEM_REF
, type
,
16619 build_fold_addr_expr (new_var
),
16620 build_int_cst (build_pointer_type (type
),
16623 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
16625 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
16626 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
16627 new_var
= build2 (MEM_REF
, type
,
16628 build_fold_addr_expr (new_var
),
16629 build_int_cst (build_pointer_type (type
),
16633 is_ref
= is_reference (var
);
16634 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16636 bool ref_to_array
= false;
16639 type
= TREE_TYPE (type
);
16640 if (TREE_CODE (type
) == ARRAY_TYPE
)
16642 type
= build_pointer_type (type
);
16643 ref_to_array
= true;
16646 else if (TREE_CODE (type
) == ARRAY_TYPE
)
16648 tree decl2
= DECL_VALUE_EXPR (new_var
);
16649 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
16650 decl2
= TREE_OPERAND (decl2
, 0);
16651 gcc_assert (DECL_P (decl2
));
16653 type
= TREE_TYPE (new_var
);
16655 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
16656 x
= fold_convert_loc (clause_loc
, type
, x
);
16657 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
16659 tree bias
= OMP_CLAUSE_SIZE (c
);
16661 bias
= lookup_decl (bias
, ctx
);
16662 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
16663 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
16665 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
16666 TREE_TYPE (x
), x
, bias
);
16669 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16670 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16671 if (is_ref
&& !ref_to_array
)
16673 tree t
= create_tmp_var_raw (type
, get_name (var
));
16674 gimple_add_tmp_var (t
);
16675 TREE_ADDRESSABLE (t
) = 1;
16676 gimple_seq_add_stmt (&new_body
,
16677 gimple_build_assign (t
, x
));
16678 x
= build_fold_addr_expr_loc (clause_loc
, t
);
16680 gimple_seq_add_stmt (&new_body
,
16681 gimple_build_assign (new_var
, x
));
16684 else if (OMP_CLAUSE_CHAIN (c
)
16685 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
16687 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16688 == GOMP_MAP_FIRSTPRIVATE_POINTER
16689 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16690 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16693 case OMP_CLAUSE_PRIVATE
:
16694 var
= OMP_CLAUSE_DECL (c
);
16695 if (is_variable_sized (var
))
16697 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16698 tree new_var
= lookup_decl (var
, ctx
);
16699 tree pvar
= DECL_VALUE_EXPR (var
);
16700 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16701 pvar
= TREE_OPERAND (pvar
, 0);
16702 gcc_assert (DECL_P (pvar
));
16703 tree new_pvar
= lookup_decl (pvar
, ctx
);
16704 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16705 tree al
= size_int (DECL_ALIGN (var
));
16706 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
16707 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16708 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
16709 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16710 gimple_seq_add_stmt (&new_body
,
16711 gimple_build_assign (new_pvar
, x
));
16713 else if (is_reference (var
) && !is_gimple_omp_oacc (ctx
->stmt
))
16715 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16716 tree new_var
= lookup_decl (var
, ctx
);
16717 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16718 if (TREE_CONSTANT (x
))
16723 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16724 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
16725 tree al
= size_int (TYPE_ALIGN (rtype
));
16726 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16729 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16730 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16731 gimple_seq_add_stmt (&new_body
,
16732 gimple_build_assign (new_var
, x
));
16737 gimple_seq fork_seq
= NULL
;
16738 gimple_seq join_seq
= NULL
;
16740 if (is_oacc_parallel (ctx
))
16742 /* If there are reductions on the offloaded region itself, treat
16743 them as a dummy GANG loop. */
16744 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
16746 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
16747 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
16750 gimple_seq_add_seq (&new_body
, fork_seq
);
16751 gimple_seq_add_seq (&new_body
, tgt_body
);
16752 gimple_seq_add_seq (&new_body
, join_seq
);
16755 new_body
= maybe_catch_exception (new_body
);
16757 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
16758 gimple_omp_set_body (stmt
, new_body
);
16761 bind
= gimple_build_bind (NULL
, NULL
,
16762 tgt_bind
? gimple_bind_block (tgt_bind
)
16764 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
16765 gimple_bind_add_seq (bind
, ilist
);
16766 gimple_bind_add_stmt (bind
, stmt
);
16767 gimple_bind_add_seq (bind
, olist
);
16769 pop_gimplify_context (NULL
);
16773 gimple_bind_add_seq (dep_bind
, dep_ilist
);
16774 gimple_bind_add_stmt (dep_bind
, bind
);
16775 gimple_bind_add_seq (dep_bind
, dep_olist
);
16776 pop_gimplify_context (dep_bind
);
16780 /* Expand code for an OpenMP teams directive. */
16783 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16785 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
16786 push_gimplify_context ();
16788 tree block
= make_node (BLOCK
);
16789 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
16790 gsi_replace (gsi_p
, bind
, true);
16791 gimple_seq bind_body
= NULL
;
16792 gimple_seq dlist
= NULL
;
16793 gimple_seq olist
= NULL
;
16795 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16796 OMP_CLAUSE_NUM_TEAMS
);
16797 if (num_teams
== NULL_TREE
)
16798 num_teams
= build_int_cst (unsigned_type_node
, 0);
16801 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16802 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16803 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16805 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16806 OMP_CLAUSE_THREAD_LIMIT
);
16807 if (thread_limit
== NULL_TREE
)
16808 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16811 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16812 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16813 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16817 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16818 &bind_body
, &dlist
, ctx
, NULL
);
16819 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16820 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16821 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16823 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16824 location_t loc
= gimple_location (teams_stmt
);
16825 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16826 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16827 gimple_set_location (call
, loc
);
16828 gimple_seq_add_stmt (&bind_body
, call
);
16831 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16832 gimple_omp_set_body (teams_stmt
, NULL
);
16833 gimple_seq_add_seq (&bind_body
, olist
);
16834 gimple_seq_add_seq (&bind_body
, dlist
);
16835 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16836 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16837 gimple_bind_set_body (bind
, bind_body
);
16839 pop_gimplify_context (bind
);
16841 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16842 BLOCK_VARS (block
) = ctx
->block_vars
;
16843 if (BLOCK_VARS (block
))
16844 TREE_USED (block
) = 1;
16847 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16850 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16852 gimple
*stmt
= gsi_stmt (*gsi_p
);
16853 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
16854 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
16855 gimple_build_omp_return (false));
16859 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16860 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16861 of OMP context, but with task_shared_vars set. */
16864 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16869 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16870 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16873 if (task_shared_vars
16875 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16878 /* If a global variable has been privatized, TREE_CONSTANT on
16879 ADDR_EXPR might be wrong. */
16880 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16881 recompute_tree_invariant_for_addr_expr (t
);
16883 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16887 /* Data to be communicated between lower_omp_regimplify_operands and
16888 lower_omp_regimplify_operands_p. */
16890 struct lower_omp_regimplify_operands_data
16896 /* Helper function for lower_omp_regimplify_operands. Find
16897 omp_member_access_dummy_var vars and adjust temporarily their
16898 DECL_VALUE_EXPRs if needed. */
16901 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16904 tree t
= omp_member_access_dummy_var (*tp
);
16907 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16908 lower_omp_regimplify_operands_data
*ldata
16909 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16910 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16913 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16914 ldata
->decls
->safe_push (*tp
);
16915 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16916 SET_DECL_VALUE_EXPR (*tp
, v
);
16919 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16923 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16924 of omp_member_access_dummy_var vars during regimplification. */
16927 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16928 gimple_stmt_iterator
*gsi_p
)
16930 auto_vec
<tree
, 10> decls
;
16933 struct walk_stmt_info wi
;
16934 memset (&wi
, '\0', sizeof (wi
));
16935 struct lower_omp_regimplify_operands_data data
;
16937 data
.decls
= &decls
;
16939 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16941 gimple_regimplify_operands (stmt
, gsi_p
);
16942 while (!decls
.is_empty ())
16944 tree t
= decls
.pop ();
16945 tree v
= decls
.pop ();
16946 SET_DECL_VALUE_EXPR (t
, v
);
16951 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16953 gimple
*stmt
= gsi_stmt (*gsi_p
);
16954 struct walk_stmt_info wi
;
16957 if (gimple_has_location (stmt
))
16958 input_location
= gimple_location (stmt
);
16960 if (task_shared_vars
)
16961 memset (&wi
, '\0', sizeof (wi
));
16963 /* If we have issued syntax errors, avoid doing any heavy lifting.
16964 Just replace the OMP directives with a NOP to avoid
16965 confusing RTL expansion. */
16966 if (seen_error () && is_gimple_omp (stmt
))
16968 gsi_replace (gsi_p
, gimple_build_nop (), true);
16972 switch (gimple_code (stmt
))
16976 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16977 if ((ctx
|| task_shared_vars
)
16978 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16979 lower_omp_regimplify_p
,
16980 ctx
? NULL
: &wi
, NULL
)
16981 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16982 lower_omp_regimplify_p
,
16983 ctx
? NULL
: &wi
, NULL
)))
16984 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16988 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16990 case GIMPLE_EH_FILTER
:
16991 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16994 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16995 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16997 case GIMPLE_TRANSACTION
:
16998 lower_omp (gimple_transaction_body_ptr (
16999 as_a
<gtransaction
*> (stmt
)),
17003 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
17005 case GIMPLE_OMP_PARALLEL
:
17006 case GIMPLE_OMP_TASK
:
17007 ctx
= maybe_lookup_ctx (stmt
);
17009 if (ctx
->cancellable
)
17010 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17011 lower_omp_taskreg (gsi_p
, ctx
);
17013 case GIMPLE_OMP_FOR
:
17014 ctx
= maybe_lookup_ctx (stmt
);
17016 if (ctx
->cancellable
)
17017 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17018 lower_omp_for (gsi_p
, ctx
);
17020 case GIMPLE_OMP_SECTIONS
:
17021 ctx
= maybe_lookup_ctx (stmt
);
17023 if (ctx
->cancellable
)
17024 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
17025 lower_omp_sections (gsi_p
, ctx
);
17027 case GIMPLE_OMP_SINGLE
:
17028 ctx
= maybe_lookup_ctx (stmt
);
17030 lower_omp_single (gsi_p
, ctx
);
17032 case GIMPLE_OMP_MASTER
:
17033 ctx
= maybe_lookup_ctx (stmt
);
17035 lower_omp_master (gsi_p
, ctx
);
17037 case GIMPLE_OMP_TASKGROUP
:
17038 ctx
= maybe_lookup_ctx (stmt
);
17040 lower_omp_taskgroup (gsi_p
, ctx
);
17042 case GIMPLE_OMP_ORDERED
:
17043 ctx
= maybe_lookup_ctx (stmt
);
17045 lower_omp_ordered (gsi_p
, ctx
);
17047 case GIMPLE_OMP_CRITICAL
:
17048 ctx
= maybe_lookup_ctx (stmt
);
17050 lower_omp_critical (gsi_p
, ctx
);
17052 case GIMPLE_OMP_ATOMIC_LOAD
:
17053 if ((ctx
|| task_shared_vars
)
17054 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
17055 as_a
<gomp_atomic_load
*> (stmt
)),
17056 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
17057 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17059 case GIMPLE_OMP_TARGET
:
17060 ctx
= maybe_lookup_ctx (stmt
);
17062 lower_omp_target (gsi_p
, ctx
);
17064 case GIMPLE_OMP_TEAMS
:
17065 ctx
= maybe_lookup_ctx (stmt
);
17067 lower_omp_teams (gsi_p
, ctx
);
17069 case GIMPLE_OMP_GRID_BODY
:
17070 ctx
= maybe_lookup_ctx (stmt
);
17072 lower_omp_grid_body (gsi_p
, ctx
);
17076 call_stmt
= as_a
<gcall
*> (stmt
);
17077 fndecl
= gimple_call_fndecl (call_stmt
);
17079 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
17080 switch (DECL_FUNCTION_CODE (fndecl
))
17082 case BUILT_IN_GOMP_BARRIER
:
17086 case BUILT_IN_GOMP_CANCEL
:
17087 case BUILT_IN_GOMP_CANCELLATION_POINT
:
17090 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
17091 cctx
= cctx
->outer
;
17092 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
17093 if (!cctx
->cancellable
)
17095 if (DECL_FUNCTION_CODE (fndecl
)
17096 == BUILT_IN_GOMP_CANCELLATION_POINT
)
17098 stmt
= gimple_build_nop ();
17099 gsi_replace (gsi_p
, stmt
, false);
17103 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
17105 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
17106 gimple_call_set_fndecl (call_stmt
, fndecl
);
17107 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
17110 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
17111 gimple_call_set_lhs (call_stmt
, lhs
);
17112 tree fallthru_label
;
17113 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
17115 g
= gimple_build_label (fallthru_label
);
17116 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17117 g
= gimple_build_cond (NE_EXPR
, lhs
,
17118 fold_convert (TREE_TYPE (lhs
),
17119 boolean_false_node
),
17120 cctx
->cancel_label
, fallthru_label
);
17121 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17128 if ((ctx
|| task_shared_vars
)
17129 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
17132 /* Just remove clobbers, this should happen only if we have
17133 "privatized" local addressable variables in SIMD regions,
17134 the clobber isn't needed in that case and gimplifying address
17135 of the ARRAY_REF into a pointer and creating MEM_REF based
17136 clobber would create worse code than we get with the clobber
17138 if (gimple_clobber_p (stmt
))
17140 gsi_replace (gsi_p
, gimple_build_nop (), true);
17143 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17150 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
17152 location_t saved_location
= input_location
;
17153 gimple_stmt_iterator gsi
;
17154 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17155 lower_omp_1 (&gsi
, ctx
);
17156 /* During gimplification, we haven't folded statments inside offloading
17157 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17158 if (target_nesting_level
|| taskreg_nesting_level
)
17159 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17161 input_location
= saved_location
;
17164 /* Returen true if STMT is an assignment of a register-type into a local
17168 grid_reg_assignment_to_local_var_p (gimple
*stmt
)
17170 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
17173 tree lhs
= gimple_assign_lhs (assign
);
17174 if (TREE_CODE (lhs
) != VAR_DECL
17175 || !is_gimple_reg_type (TREE_TYPE (lhs
))
17176 || is_global_var (lhs
))
17181 /* Return true if all statements in SEQ are assignments to local register-type
17185 grid_seq_only_contains_local_assignments (gimple_seq seq
)
17190 gimple_stmt_iterator gsi
;
17191 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17192 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi
)))
17197 /* Scan statements in SEQ and call itself recursively on any bind. If during
17198 whole search only assignments to register-type local variables and one
17199 single OMP statement is encountered, return true, otherwise return false.
17200 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17201 are used for dumping a note about a failure. */
17204 grid_find_single_omp_among_assignments_1 (gimple_seq seq
, location_t target_loc
,
17205 const char *name
, gimple
**ret
)
17207 gimple_stmt_iterator gsi
;
17208 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17210 gimple
*stmt
= gsi_stmt (gsi
);
17212 if (grid_reg_assignment_to_local_var_p (stmt
))
17214 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17216 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind
),
17217 target_loc
, name
, ret
))
17220 else if (is_gimple_omp (stmt
))
17224 if (dump_enabled_p ())
17225 dump_printf_loc (MSG_NOTE
, target_loc
,
17226 "Will not turn target construct into a simple "
17227 "GPGPU kernel because %s construct contains "
17228 "multiple OpenMP constructs\n", name
);
17235 if (dump_enabled_p ())
17236 dump_printf_loc (MSG_NOTE
, target_loc
,
17237 "Will not turn target construct into a simple "
17238 "GPGPU kernel because %s construct contains "
17239 "a complex statement\n", name
);
17246 /* Scan statements in SEQ and make sure that it and any binds in it contain
17247 only assignments to local register-type variables and one OMP construct. If
17248 so, return that construct, otherwise return NULL. If dumping is enabled and
17249 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17253 grid_find_single_omp_among_assignments (gimple_seq seq
, location_t target_loc
,
17258 if (dump_enabled_p ())
17259 dump_printf_loc (MSG_NOTE
, target_loc
,
17260 "Will not turn target construct into a simple "
17261 "GPGPU kernel because %s construct has empty "
17267 gimple
*ret
= NULL
;
17268 if (grid_find_single_omp_among_assignments_1 (seq
, target_loc
, name
, &ret
))
17270 if (!ret
&& dump_enabled_p ())
17271 dump_printf_loc (MSG_NOTE
, target_loc
,
17272 "Will not turn target construct into a simple "
17273 "GPGPU kernel because %s construct does not contain"
17274 "any other OpenMP construct\n", name
);
17281 /* Walker function looking for statements there is no point gridifying (and for
17282 noreturn function calls which we cannot do). Return non-NULL if such a
17283 function is found. */
17286 grid_find_ungridifiable_statement (gimple_stmt_iterator
*gsi
,
17287 bool *handled_ops_p
,
17288 struct walk_stmt_info
*wi
)
17290 *handled_ops_p
= false;
17291 gimple
*stmt
= gsi_stmt (*gsi
);
17292 switch (gimple_code (stmt
))
17295 if (gimple_call_noreturn_p (as_a
<gcall
*> (stmt
)))
17297 *handled_ops_p
= true;
17299 return error_mark_node
;
17303 /* We may reduce the following list if we find a way to implement the
17304 clauses, but now there is no point trying further. */
17305 case GIMPLE_OMP_CRITICAL
:
17306 case GIMPLE_OMP_TASKGROUP
:
17307 case GIMPLE_OMP_TASK
:
17308 case GIMPLE_OMP_SECTION
:
17309 case GIMPLE_OMP_SECTIONS
:
17310 case GIMPLE_OMP_SECTIONS_SWITCH
:
17311 case GIMPLE_OMP_TARGET
:
17312 case GIMPLE_OMP_ORDERED
:
17313 *handled_ops_p
= true;
17315 return error_mark_node
;
17317 case GIMPLE_OMP_FOR
:
17318 if ((gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
17319 && gimple_omp_for_combined_into_p (stmt
))
17321 *handled_ops_p
= true;
17323 return error_mark_node
;
17334 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17335 kernel, return true, otherwise return false. In the case of success, also
17336 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17340 grid_target_follows_gridifiable_pattern (gomp_target
*target
, tree
*group_size_p
)
17342 if (gimple_omp_target_kind (target
) != GF_OMP_TARGET_KIND_REGION
)
17345 location_t tloc
= gimple_location (target
);
17347 = grid_find_single_omp_among_assignments (gimple_omp_body (target
),
17351 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17352 tree group_size
= NULL
;
17355 dump_printf_loc (MSG_NOTE
, tloc
,
17356 "Will not turn target construct into a simple "
17357 "GPGPU kernel because it does not have a sole teams "
17358 "construct in it.\n");
17362 tree clauses
= gimple_omp_teams_clauses (teams
);
17365 switch (OMP_CLAUSE_CODE (clauses
))
17367 case OMP_CLAUSE_NUM_TEAMS
:
17368 if (dump_enabled_p ())
17369 dump_printf_loc (MSG_NOTE
, tloc
,
17370 "Will not turn target construct into a "
17371 "gridified GPGPU kernel because we cannot "
17372 "handle num_teams clause of teams "
17376 case OMP_CLAUSE_REDUCTION
:
17377 if (dump_enabled_p ())
17378 dump_printf_loc (MSG_NOTE
, tloc
,
17379 "Will not turn target construct into a "
17380 "gridified GPGPU kernel because a reduction "
17381 "clause is present\n ");
17384 case OMP_CLAUSE_LASTPRIVATE
:
17385 if (dump_enabled_p ())
17386 dump_printf_loc (MSG_NOTE
, tloc
,
17387 "Will not turn target construct into a "
17388 "gridified GPGPU kernel because a lastprivate "
17389 "clause is present\n ");
17392 case OMP_CLAUSE_THREAD_LIMIT
:
17393 group_size
= OMP_CLAUSE_OPERAND (clauses
, 0);
17399 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17402 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (teams
), tloc
,
17406 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17409 dump_printf_loc (MSG_NOTE
, tloc
,
17410 "Will not turn target construct into a simple "
17411 "GPGPU kernel because the teams construct does not have "
17412 "a sole distribute construct in it.\n");
17416 gcc_assert (gimple_omp_for_kind (dist
) == GF_OMP_FOR_KIND_DISTRIBUTE
);
17417 if (!gimple_omp_for_combined_p (dist
))
17419 if (dump_enabled_p ())
17420 dump_printf_loc (MSG_NOTE
, tloc
,
17421 "Will not turn target construct into a gridified GPGPU "
17422 "kernel because we cannot handle a standalone "
17423 "distribute construct\n ");
17426 if (dist
->collapse
> 1)
17428 if (dump_enabled_p ())
17429 dump_printf_loc (MSG_NOTE
, tloc
,
17430 "Will not turn target construct into a gridified GPGPU "
17431 "kernel because the distribute construct contains "
17432 "collapse clause\n");
17435 struct omp_for_data fd
;
17436 extract_omp_for_data (dist
, &fd
, NULL
);
17439 if (group_size
&& !operand_equal_p (group_size
, fd
.chunk_size
, 0))
17441 if (dump_enabled_p ())
17442 dump_printf_loc (MSG_NOTE
, tloc
,
17443 "Will not turn target construct into a "
17444 "gridified GPGPU kernel because the teams "
17445 "thread limit is different from distribute "
17446 "schedule chunk\n");
17449 group_size
= fd
.chunk_size
;
17451 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (dist
), tloc
,
17453 gomp_parallel
*par
;
17454 if (!stmt
|| !(par
= dyn_cast
<gomp_parallel
*> (stmt
)))
17457 clauses
= gimple_omp_parallel_clauses (par
);
17460 switch (OMP_CLAUSE_CODE (clauses
))
17462 case OMP_CLAUSE_NUM_THREADS
:
17463 if (dump_enabled_p ())
17464 dump_printf_loc (MSG_NOTE
, tloc
,
17465 "Will not turn target construct into a gridified"
17466 "GPGPU kernel because there is a num_threads "
17467 "clause of the parallel construct\n");
17470 case OMP_CLAUSE_REDUCTION
:
17471 if (dump_enabled_p ())
17472 dump_printf_loc (MSG_NOTE
, tloc
,
17473 "Will not turn target construct into a "
17474 "gridified GPGPU kernel because a reduction "
17475 "clause is present\n ");
17478 case OMP_CLAUSE_LASTPRIVATE
:
17479 if (dump_enabled_p ())
17480 dump_printf_loc (MSG_NOTE
, tloc
,
17481 "Will not turn target construct into a "
17482 "gridified GPGPU kernel because a lastprivate "
17483 "clause is present\n ");
17489 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17492 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (par
), tloc
,
17495 if (!stmt
|| !(gfor
= dyn_cast
<gomp_for
*> (stmt
)))
17498 if (gimple_omp_for_kind (gfor
) != GF_OMP_FOR_KIND_FOR
)
17500 if (dump_enabled_p ())
17501 dump_printf_loc (MSG_NOTE
, tloc
,
17502 "Will not turn target construct into a gridified GPGPU "
17503 "kernel because the inner loop is not a simple for "
17507 if (gfor
->collapse
> 1)
17509 if (dump_enabled_p ())
17510 dump_printf_loc (MSG_NOTE
, tloc
,
17511 "Will not turn target construct into a gridified GPGPU "
17512 "kernel because the inner loop contains collapse "
17517 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor
)))
17519 if (dump_enabled_p ())
17520 dump_printf_loc (MSG_NOTE
, tloc
,
17521 "Will not turn target construct into a gridified GPGPU "
17522 "kernel because the inner loop pre_body contains"
17523 "a complex instruction\n");
17527 clauses
= gimple_omp_for_clauses (gfor
);
17530 switch (OMP_CLAUSE_CODE (clauses
))
17532 case OMP_CLAUSE_SCHEDULE
:
17533 if (OMP_CLAUSE_SCHEDULE_KIND (clauses
) != OMP_CLAUSE_SCHEDULE_AUTO
)
17535 if (dump_enabled_p ())
17536 dump_printf_loc (MSG_NOTE
, tloc
,
17537 "Will not turn target construct into a "
17538 "gridified GPGPU kernel because the inner "
17539 "loop has a non-automatic scheduling clause\n");
17544 case OMP_CLAUSE_REDUCTION
:
17545 if (dump_enabled_p ())
17546 dump_printf_loc (MSG_NOTE
, tloc
,
17547 "Will not turn target construct into a "
17548 "gridified GPGPU kernel because a reduction "
17549 "clause is present\n ");
17552 case OMP_CLAUSE_LASTPRIVATE
:
17553 if (dump_enabled_p ())
17554 dump_printf_loc (MSG_NOTE
, tloc
,
17555 "Will not turn target construct into a "
17556 "gridified GPGPU kernel because a lastprivate "
17557 "clause is present\n ");
17563 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17566 struct walk_stmt_info wi
;
17567 memset (&wi
, 0, sizeof (wi
));
17568 if (walk_gimple_seq (gimple_omp_body (gfor
),
17569 grid_find_ungridifiable_statement
,
17572 gimple
*bad
= (gimple
*) wi
.info
;
17573 if (dump_enabled_p ())
17575 if (is_gimple_call (bad
))
17576 dump_printf_loc (MSG_NOTE
, tloc
,
17577 "Will not turn target construct into a gridified "
17578 " GPGPU kernel because the inner loop contains "
17579 "call to a noreturn function\n");
17580 if (gimple_code (bad
) == GIMPLE_OMP_FOR
)
17581 dump_printf_loc (MSG_NOTE
, tloc
,
17582 "Will not turn target construct into a gridified "
17583 " GPGPU kernel because the inner loop contains "
17584 "a simd construct\n");
17586 dump_printf_loc (MSG_NOTE
, tloc
,
17587 "Will not turn target construct into a gridified "
17588 "GPGPU kernel because the inner loop contains "
17589 "statement %s which cannot be transformed\n",
17590 gimple_code_name
[(int) gimple_code (bad
)]);
17595 *group_size_p
= group_size
;
17599 /* Operand walker, used to remap pre-body declarations according to a hash map
17600 provided in DATA. */
17603 grid_remap_prebody_decls (tree
*tp
, int *walk_subtrees
, void *data
)
17607 if (DECL_P (t
) || TYPE_P (t
))
17608 *walk_subtrees
= 0;
17610 *walk_subtrees
= 1;
17612 if (TREE_CODE (t
) == VAR_DECL
)
17614 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17615 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17616 tree
*repl
= declmap
->get (t
);
17623 /* Copy leading register-type assignments to local variables in SRC to just
17624 before DST, Creating temporaries, adjusting mapping of operands in WI and
17625 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17626 Return the first statement that does not conform to
17627 grid_reg_assignment_to_local_var_p or NULL. */
17630 grid_copy_leading_local_assignments (gimple_seq src
, gimple_stmt_iterator
*dst
,
17631 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17633 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17634 gimple_stmt_iterator gsi
;
17635 for (gsi
= gsi_start (src
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17637 gimple
*stmt
= gsi_stmt (gsi
);
17638 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17640 gimple
*r
= grid_copy_leading_local_assignments
17641 (gimple_bind_body (bind
), dst
, tgt_bind
, wi
);
17647 if (!grid_reg_assignment_to_local_var_p (stmt
))
17649 tree lhs
= gimple_assign_lhs (as_a
<gassign
*> (stmt
));
17650 tree repl
= copy_var_decl (lhs
, create_tmp_var_name (NULL
),
17652 DECL_CONTEXT (repl
) = current_function_decl
;
17653 gimple_bind_append_vars (tgt_bind
, repl
);
17655 declmap
->put (lhs
, repl
);
17656 gassign
*copy
= as_a
<gassign
*> (gimple_copy (stmt
));
17657 walk_gimple_op (copy
, grid_remap_prebody_decls
, wi
);
17658 gsi_insert_before (dst
, copy
, GSI_SAME_STMT
);
17663 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17664 components, mark them as part of kernel and return the inner loop, and copy
17665 assignment leading to them just before DST, remapping them using WI and
17666 adding new temporaries to TGT_BIND. */
17669 grid_process_kernel_body_copy (gimple_seq seq
, gimple_stmt_iterator
*dst
,
17670 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17672 gimple
*stmt
= grid_copy_leading_local_assignments (seq
, dst
, tgt_bind
, wi
);
17673 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17674 gcc_assert (teams
);
17675 gimple_omp_teams_set_grid_phony (teams
, true);
17676 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (teams
), dst
,
17678 gcc_checking_assert (stmt
);
17679 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17681 gimple_seq prebody
= gimple_omp_for_pre_body (dist
);
17683 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17684 gimple_omp_for_set_grid_phony (dist
, true);
17685 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (dist
), dst
,
17687 gcc_checking_assert (stmt
);
17689 gomp_parallel
*parallel
= as_a
<gomp_parallel
*> (stmt
);
17690 gimple_omp_parallel_set_grid_phony (parallel
, true);
17691 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (parallel
), dst
,
17693 gomp_for
*inner_loop
= as_a
<gomp_for
*> (stmt
);
17694 gimple_omp_for_set_kind (inner_loop
, GF_OMP_FOR_KIND_GRID_LOOP
);
17695 prebody
= gimple_omp_for_pre_body (inner_loop
);
17697 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17702 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17703 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17704 is the bind into which temporaries inserted before TARGET should be
17708 grid_attempt_target_gridification (gomp_target
*target
,
17709 gimple_stmt_iterator
*gsi
,
17713 if (!target
|| !grid_target_follows_gridifiable_pattern (target
, &group_size
))
17716 location_t loc
= gimple_location (target
);
17717 if (dump_enabled_p ())
17718 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
17719 "Target construct will be turned into a gridified GPGPU "
17722 /* Copy target body to a GPUKERNEL construct: */
17723 gimple_seq kernel_seq
= copy_gimple_seq_and_replace_locals
17724 (gimple_omp_body (target
));
17726 hash_map
<tree
, tree
> *declmap
= new hash_map
<tree
, tree
>;
17727 struct walk_stmt_info wi
;
17728 memset (&wi
, 0, sizeof (struct walk_stmt_info
));
17731 /* Copy assignments in between OMP statements before target, mark OMP
17732 statements within copy appropriatly. */
17733 gomp_for
*inner_loop
= grid_process_kernel_body_copy (kernel_seq
, gsi
,
17736 gbind
*old_bind
= as_a
<gbind
*> (gimple_seq_first (gimple_omp_body (target
)));
17737 gbind
*new_bind
= as_a
<gbind
*> (gimple_seq_first (kernel_seq
));
17738 tree new_block
= gimple_bind_block (new_bind
);
17739 tree enc_block
= BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind
));
17740 BLOCK_CHAIN (new_block
) = BLOCK_SUBBLOCKS (enc_block
);
17741 BLOCK_SUBBLOCKS (enc_block
) = new_block
;
17742 BLOCK_SUPERCONTEXT (new_block
) = enc_block
;
17743 gimple
*gpukernel
= gimple_build_omp_grid_body (kernel_seq
);
17744 gimple_seq_add_stmt
17745 (gimple_bind_body_ptr (as_a
<gbind
*> (gimple_omp_body (target
))),
17748 walk_tree (&group_size
, grid_remap_prebody_decls
, &wi
, NULL
);
17749 push_gimplify_context ();
17750 size_t collapse
= gimple_omp_for_collapse (inner_loop
);
17751 for (size_t i
= 0; i
< collapse
; i
++)
17753 tree itype
, type
= TREE_TYPE (gimple_omp_for_index (inner_loop
, i
));
17754 if (POINTER_TYPE_P (type
))
17755 itype
= signed_type_for (type
);
17759 enum tree_code cond_code
= gimple_omp_for_cond (inner_loop
, i
);
17760 tree n1
= unshare_expr (gimple_omp_for_initial (inner_loop
, i
));
17761 walk_tree (&n1
, grid_remap_prebody_decls
, &wi
, NULL
);
17762 tree n2
= unshare_expr (gimple_omp_for_final (inner_loop
, i
));
17763 walk_tree (&n2
, grid_remap_prebody_decls
, &wi
, NULL
);
17764 adjust_for_condition (loc
, &cond_code
, &n2
);
17766 step
= get_omp_for_step_from_incr (loc
,
17767 gimple_omp_for_incr (inner_loop
, i
));
17768 gimple_seq tmpseq
= NULL
;
17769 n1
= fold_convert (itype
, n1
);
17770 n2
= fold_convert (itype
, n2
);
17771 tree t
= build_int_cst (itype
, (cond_code
== LT_EXPR
? -1 : 1));
17772 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
17773 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
17774 t
= fold_build2 (MINUS_EXPR
, itype
, t
, n1
);
17775 if (TYPE_UNSIGNED (itype
) && cond_code
== GT_EXPR
)
17776 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
17777 fold_build1 (NEGATE_EXPR
, itype
, t
),
17778 fold_build1 (NEGATE_EXPR
, itype
, step
));
17780 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
17781 tree gs
= fold_convert (uint32_type_node
, t
);
17782 gimplify_expr (&gs
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17783 if (!gimple_seq_empty_p (tmpseq
))
17784 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17787 if (i
== 0 && group_size
)
17789 ws
= fold_convert (uint32_type_node
, group_size
);
17791 gimplify_expr (&ws
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17792 if (!gimple_seq_empty_p (tmpseq
))
17793 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17796 ws
= build_zero_cst (uint32_type_node
);
17798 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__GRIDDIM_
);
17799 OMP_CLAUSE__GRIDDIM__DIMENSION (c
) = i
;
17800 OMP_CLAUSE__GRIDDIM__SIZE (c
) = gs
;
17801 OMP_CLAUSE__GRIDDIM__GROUP (c
) = ws
;
17802 OMP_CLAUSE_CHAIN (c
) = gimple_omp_target_clauses (target
);
17803 gimple_omp_target_set_clauses (target
, c
);
17805 pop_gimplify_context (tgt_bind
);
17810 /* Walker function doing all the work for create_target_kernels. */
17813 grid_gridify_all_targets_stmt (gimple_stmt_iterator
*gsi
,
17814 bool *handled_ops_p
,
17815 struct walk_stmt_info
*incoming
)
17817 *handled_ops_p
= false;
17819 gimple
*stmt
= gsi_stmt (*gsi
);
17820 gomp_target
*target
= dyn_cast
<gomp_target
*> (stmt
);
17823 gbind
*tgt_bind
= (gbind
*) incoming
->info
;
17824 gcc_checking_assert (tgt_bind
);
17825 grid_attempt_target_gridification (target
, gsi
, tgt_bind
);
17828 gbind
*bind
= dyn_cast
<gbind
*> (stmt
);
17831 *handled_ops_p
= true;
17832 struct walk_stmt_info wi
;
17833 memset (&wi
, 0, sizeof (wi
));
17835 walk_gimple_seq_mod (gimple_bind_body_ptr (bind
),
17836 grid_gridify_all_targets_stmt
, NULL
, &wi
);
17841 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17842 have their bodies duplicated, with the new copy being put into a
17843 gimple_omp_grid_body statement. All kernel-related construct within the
17844 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17845 re-structuring is often needed, such as copying pre-bodies before the target
17846 construct so that kernel grid sizes can be computed. */
17849 grid_gridify_all_targets (gimple_seq
*body_p
)
17851 struct walk_stmt_info wi
;
17852 memset (&wi
, 0, sizeof (wi
));
17853 walk_gimple_seq_mod (body_p
, grid_gridify_all_targets_stmt
, NULL
, &wi
);
17857 /* Main entry point. */
17859 static unsigned int
17860 execute_lower_omp (void)
17866 /* This pass always runs, to provide PROP_gimple_lomp.
17867 But often, there is nothing to do. */
17868 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
17869 && flag_openmp_simd
== 0)
17872 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
17873 delete_omp_context
);
17875 body
= gimple_body (current_function_decl
);
17877 if (hsa_gen_requested_p ())
17878 grid_gridify_all_targets (&body
);
17880 scan_omp (&body
, NULL
);
17881 gcc_assert (taskreg_nesting_level
== 0);
17882 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
17883 finish_taskreg_scan (ctx
);
17884 taskreg_contexts
.release ();
17886 if (all_contexts
->root
)
17888 if (task_shared_vars
)
17889 push_gimplify_context ();
17890 lower_omp (&body
, NULL
);
17891 if (task_shared_vars
)
17892 pop_gimplify_context (NULL
);
17897 splay_tree_delete (all_contexts
);
17898 all_contexts
= NULL
;
17900 BITMAP_FREE (task_shared_vars
);
17906 const pass_data pass_data_lower_omp
=
17908 GIMPLE_PASS
, /* type */
17909 "omplower", /* name */
17910 OPTGROUP_NONE
, /* optinfo_flags */
17911 TV_NONE
, /* tv_id */
17912 PROP_gimple_any
, /* properties_required */
17913 PROP_gimple_lomp
, /* properties_provided */
17914 0, /* properties_destroyed */
17915 0, /* todo_flags_start */
17916 0, /* todo_flags_finish */
17919 class pass_lower_omp
: public gimple_opt_pass
17922 pass_lower_omp (gcc::context
*ctxt
)
17923 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
17926 /* opt_pass methods: */
17927 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
17929 }; // class pass_lower_omp
17931 } // anon namespace
17934 make_pass_lower_omp (gcc::context
*ctxt
)
17936 return new pass_lower_omp (ctxt
);
17939 /* The following is a utility to diagnose structured block violations.
17940 It is not part of the "omplower" pass, as that's invoked too late. It
17941 should be invoked by the respective front ends after gimplification. */
17943 static splay_tree all_labels
;
17945 /* Check for mismatched contexts and generate an error if needed. Return
17946 true if an error is detected. */
17949 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
17950 gimple
*branch_ctx
, gimple
*label_ctx
)
17952 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
17953 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
17955 if (label_ctx
== branch_ctx
)
17958 const char* kind
= NULL
;
17963 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
17964 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
17966 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
17967 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
17968 kind
= "Cilk Plus";
17972 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
17973 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
17975 gcc_checking_assert (kind
== NULL
);
17981 gcc_checking_assert (flag_openmp
);
17986 Previously we kept track of the label's entire context in diagnose_sb_[12]
17987 so we could traverse it and issue a correct "exit" or "enter" error
17988 message upon a structured block violation.
17990 We built the context by building a list with tree_cons'ing, but there is
17991 no easy counterpart in gimple tuples. It seems like far too much work
17992 for issuing exit/enter error messages. If someone really misses the
17993 distinct error message... patches welcome.
17997 /* Try to avoid confusing the user by producing and error message
17998 with correct "exit" or "enter" verbiage. We prefer "exit"
17999 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
18000 if (branch_ctx
== NULL
)
18006 if (TREE_VALUE (label_ctx
) == branch_ctx
)
18011 label_ctx
= TREE_CHAIN (label_ctx
);
18016 error ("invalid exit from %s structured block", kind
);
18018 error ("invalid entry to %s structured block", kind
);
18021 /* If it's obvious we have an invalid entry, be specific about the error. */
18022 if (branch_ctx
== NULL
)
18023 error ("invalid entry to %s structured block", kind
);
18026 /* Otherwise, be vague and lazy, but efficient. */
18027 error ("invalid branch to/from %s structured block", kind
);
18030 gsi_replace (gsi_p
, gimple_build_nop (), false);
18034 /* Pass 1: Create a minimal tree of structured blocks, and record
18035 where each label is found. */
18038 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18039 struct walk_stmt_info
*wi
)
18041 gimple
*context
= (gimple
*) wi
->info
;
18042 gimple
*inner_context
;
18043 gimple
*stmt
= gsi_stmt (*gsi_p
);
18045 *handled_ops_p
= true;
18047 switch (gimple_code (stmt
))
18051 case GIMPLE_OMP_PARALLEL
:
18052 case GIMPLE_OMP_TASK
:
18053 case GIMPLE_OMP_SECTIONS
:
18054 case GIMPLE_OMP_SINGLE
:
18055 case GIMPLE_OMP_SECTION
:
18056 case GIMPLE_OMP_MASTER
:
18057 case GIMPLE_OMP_ORDERED
:
18058 case GIMPLE_OMP_CRITICAL
:
18059 case GIMPLE_OMP_TARGET
:
18060 case GIMPLE_OMP_TEAMS
:
18061 case GIMPLE_OMP_TASKGROUP
:
18062 /* The minimal context here is just the current OMP construct. */
18063 inner_context
= stmt
;
18064 wi
->info
= inner_context
;
18065 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18066 wi
->info
= context
;
18069 case GIMPLE_OMP_FOR
:
18070 inner_context
= stmt
;
18071 wi
->info
= inner_context
;
18072 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18074 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
18075 diagnose_sb_1
, NULL
, wi
);
18076 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
18077 wi
->info
= context
;
18081 splay_tree_insert (all_labels
,
18082 (splay_tree_key
) gimple_label_label (
18083 as_a
<glabel
*> (stmt
)),
18084 (splay_tree_value
) context
);
18094 /* Pass 2: Check each branch and see if its context differs from that of
18095 the destination label's context. */
18098 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
18099 struct walk_stmt_info
*wi
)
18101 gimple
*context
= (gimple
*) wi
->info
;
18103 gimple
*stmt
= gsi_stmt (*gsi_p
);
18105 *handled_ops_p
= true;
18107 switch (gimple_code (stmt
))
18111 case GIMPLE_OMP_PARALLEL
:
18112 case GIMPLE_OMP_TASK
:
18113 case GIMPLE_OMP_SECTIONS
:
18114 case GIMPLE_OMP_SINGLE
:
18115 case GIMPLE_OMP_SECTION
:
18116 case GIMPLE_OMP_MASTER
:
18117 case GIMPLE_OMP_ORDERED
:
18118 case GIMPLE_OMP_CRITICAL
:
18119 case GIMPLE_OMP_TARGET
:
18120 case GIMPLE_OMP_TEAMS
:
18121 case GIMPLE_OMP_TASKGROUP
:
18123 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18124 wi
->info
= context
;
18127 case GIMPLE_OMP_FOR
:
18129 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18131 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
18132 diagnose_sb_2
, NULL
, wi
);
18133 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18134 wi
->info
= context
;
18139 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
18140 tree lab
= gimple_cond_true_label (cond_stmt
);
18143 n
= splay_tree_lookup (all_labels
,
18144 (splay_tree_key
) lab
);
18145 diagnose_sb_0 (gsi_p
, context
,
18146 n
? (gimple
*) n
->value
: NULL
);
18148 lab
= gimple_cond_false_label (cond_stmt
);
18151 n
= splay_tree_lookup (all_labels
,
18152 (splay_tree_key
) lab
);
18153 diagnose_sb_0 (gsi_p
, context
,
18154 n
? (gimple
*) n
->value
: NULL
);
18161 tree lab
= gimple_goto_dest (stmt
);
18162 if (TREE_CODE (lab
) != LABEL_DECL
)
18165 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18166 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
18170 case GIMPLE_SWITCH
:
18172 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
18174 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
18176 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
18177 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18178 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
18184 case GIMPLE_RETURN
:
18185 diagnose_sb_0 (gsi_p
, context
, NULL
);
18195 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18198 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
18201 gimple
*last
= last_stmt (bb
);
18202 enum gimple_code code
= gimple_code (last
);
18203 struct omp_region
*cur_region
= *region
;
18204 bool fallthru
= false;
18208 case GIMPLE_OMP_PARALLEL
:
18209 case GIMPLE_OMP_TASK
:
18210 case GIMPLE_OMP_FOR
:
18211 case GIMPLE_OMP_SINGLE
:
18212 case GIMPLE_OMP_TEAMS
:
18213 case GIMPLE_OMP_MASTER
:
18214 case GIMPLE_OMP_TASKGROUP
:
18215 case GIMPLE_OMP_CRITICAL
:
18216 case GIMPLE_OMP_SECTION
:
18217 case GIMPLE_OMP_GRID_BODY
:
18218 cur_region
= new_omp_region (bb
, code
, cur_region
);
18222 case GIMPLE_OMP_ORDERED
:
18223 cur_region
= new_omp_region (bb
, code
, cur_region
);
18225 if (find_omp_clause (gimple_omp_ordered_clauses
18226 (as_a
<gomp_ordered
*> (last
)),
18227 OMP_CLAUSE_DEPEND
))
18228 cur_region
= cur_region
->outer
;
18231 case GIMPLE_OMP_TARGET
:
18232 cur_region
= new_omp_region (bb
, code
, cur_region
);
18234 switch (gimple_omp_target_kind (last
))
18236 case GF_OMP_TARGET_KIND_REGION
:
18237 case GF_OMP_TARGET_KIND_DATA
:
18238 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
18239 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
18240 case GF_OMP_TARGET_KIND_OACC_DATA
:
18241 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
18243 case GF_OMP_TARGET_KIND_UPDATE
:
18244 case GF_OMP_TARGET_KIND_ENTER_DATA
:
18245 case GF_OMP_TARGET_KIND_EXIT_DATA
:
18246 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
18247 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
18248 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
18249 cur_region
= cur_region
->outer
;
18252 gcc_unreachable ();
18256 case GIMPLE_OMP_SECTIONS
:
18257 cur_region
= new_omp_region (bb
, code
, cur_region
);
18261 case GIMPLE_OMP_SECTIONS_SWITCH
:
18265 case GIMPLE_OMP_ATOMIC_LOAD
:
18266 case GIMPLE_OMP_ATOMIC_STORE
:
18270 case GIMPLE_OMP_RETURN
:
18271 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18272 somewhere other than the next block. This will be
18274 cur_region
->exit
= bb
;
18275 if (cur_region
->type
== GIMPLE_OMP_TASK
)
18276 /* Add an edge corresponding to not scheduling the task
18278 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
18279 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
18280 cur_region
= cur_region
->outer
;
18283 case GIMPLE_OMP_CONTINUE
:
18284 cur_region
->cont
= bb
;
18285 switch (cur_region
->type
)
18287 case GIMPLE_OMP_FOR
:
18288 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18289 succs edges as abnormal to prevent splitting
18291 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
18292 /* Make the loopback edge. */
18293 make_edge (bb
, single_succ (cur_region
->entry
),
18296 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18297 corresponds to the case that the body of the loop
18298 is not executed at all. */
18299 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
18300 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
18304 case GIMPLE_OMP_SECTIONS
:
18305 /* Wire up the edges into and out of the nested sections. */
18307 basic_block switch_bb
= single_succ (cur_region
->entry
);
18309 struct omp_region
*i
;
18310 for (i
= cur_region
->inner
; i
; i
= i
->next
)
18312 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
18313 make_edge (switch_bb
, i
->entry
, 0);
18314 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
18317 /* Make the loopback edge to the block with
18318 GIMPLE_OMP_SECTIONS_SWITCH. */
18319 make_edge (bb
, switch_bb
, 0);
18321 /* Make the edge from the switch to exit. */
18322 make_edge (switch_bb
, bb
->next_bb
, 0);
18327 case GIMPLE_OMP_TASK
:
18332 gcc_unreachable ();
18337 gcc_unreachable ();
18340 if (*region
!= cur_region
)
18342 *region
= cur_region
;
18344 *region_idx
= cur_region
->entry
->index
;
18352 static unsigned int
18353 diagnose_omp_structured_block_errors (void)
18355 struct walk_stmt_info wi
;
18356 gimple_seq body
= gimple_body (current_function_decl
);
18358 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
18360 memset (&wi
, 0, sizeof (wi
));
18361 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
18363 memset (&wi
, 0, sizeof (wi
));
18364 wi
.want_locations
= true;
18365 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
18367 gimple_set_body (current_function_decl
, body
);
18369 splay_tree_delete (all_labels
);
18377 const pass_data pass_data_diagnose_omp_blocks
=
18379 GIMPLE_PASS
, /* type */
18380 "*diagnose_omp_blocks", /* name */
18381 OPTGROUP_NONE
, /* optinfo_flags */
18382 TV_NONE
, /* tv_id */
18383 PROP_gimple_any
, /* properties_required */
18384 0, /* properties_provided */
18385 0, /* properties_destroyed */
18386 0, /* todo_flags_start */
18387 0, /* todo_flags_finish */
18390 class pass_diagnose_omp_blocks
: public gimple_opt_pass
18393 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18394 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
18397 /* opt_pass methods: */
18398 virtual bool gate (function
*)
18400 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
18402 virtual unsigned int execute (function
*)
18404 return diagnose_omp_structured_block_errors ();
18407 }; // class pass_diagnose_omp_blocks
18409 } // anon namespace
18412 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18414 return new pass_diagnose_omp_blocks (ctxt
);
18417 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18418 adds their addresses and sizes to constructor-vector V_CTOR. */
18420 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18421 vec
<constructor_elt
, va_gc
> *v_ctor
)
18423 unsigned len
= vec_safe_length (v_decls
);
18424 for (unsigned i
= 0; i
< len
; i
++)
18426 tree it
= (*v_decls
)[i
];
18427 bool is_var
= TREE_CODE (it
) == VAR_DECL
;
18430 #ifdef ACCEL_COMPILER
18431 && DECL_HAS_VALUE_EXPR_P (it
)
18433 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
18435 tree size
= NULL_TREE
;
18437 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
18441 addr
= build_fold_addr_expr (it
);
18444 #ifdef ACCEL_COMPILER
18445 /* For "omp declare target link" vars add address of the pointer to
18446 the target table, instead of address of the var. */
18447 tree value_expr
= DECL_VALUE_EXPR (it
);
18448 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
18449 varpool_node::finalize_decl (link_ptr_decl
);
18450 addr
= build_fold_addr_expr (link_ptr_decl
);
18452 addr
= build_fold_addr_expr (it
);
18455 /* Most significant bit of the size marks "omp declare target link"
18456 vars in host and target tables. */
18457 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
18458 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
18459 * BITS_PER_UNIT
- 1);
18460 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
18463 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
18465 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
18469 /* Create new symbols containing (address, size) pairs for global variables,
18470 marked with "omp declare target" attribute, as well as addresses for the
18471 functions, which are outlined offloading regions. */
18473 omp_finish_file (void)
18475 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18476 unsigned num_vars
= vec_safe_length (offload_vars
);
18478 if (num_funcs
== 0 && num_vars
== 0)
18481 if (targetm_common
.have_named_sections
)
18483 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18484 vec_alloc (v_f
, num_funcs
);
18485 vec_alloc (v_v
, num_vars
* 2);
18487 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18488 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18490 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18492 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18494 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18495 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18496 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18497 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18498 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18499 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18500 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18501 get_identifier (".offload_func_table"),
18503 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18504 get_identifier (".offload_var_table"),
18506 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18507 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18508 otherwise a joint table in a binary will contain padding between
18509 tables from multiple object files. */
18510 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18511 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
18512 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
18513 DECL_INITIAL (funcs_decl
) = ctor_f
;
18514 DECL_INITIAL (vars_decl
) = ctor_v
;
18515 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18516 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18518 varpool_node::finalize_decl (vars_decl
);
18519 varpool_node::finalize_decl (funcs_decl
);
18523 for (unsigned i
= 0; i
< num_funcs
; i
++)
18525 tree it
= (*offload_funcs
)[i
];
18526 targetm
.record_offload_symbol (it
);
18528 for (unsigned i
= 0; i
< num_vars
; i
++)
18530 tree it
= (*offload_vars
)[i
];
18531 targetm
.record_offload_symbol (it
);
18536 /* Find the number of threads (POS = false), or thread number (POS =
18537 true) for an OpenACC region partitioned as MASK. Setup code
18538 required for the calculation is added to SEQ. */
18541 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18543 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18546 /* Start at gang level, and examine relevant dimension indices. */
18547 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18548 if (GOMP_DIM_MASK (ix
) & mask
)
18550 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18554 /* We had an outer index, so scale that by the size of
18556 tree n
= create_tmp_var (integer_type_node
);
18558 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18560 gimple_call_set_lhs (call
, n
);
18561 gimple_seq_add_stmt (seq
, call
);
18562 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18566 /* Determine index in this dimension. */
18567 tree id
= create_tmp_var (integer_type_node
);
18568 gimple
*call
= gimple_build_call_internal
18569 (IFN_GOACC_DIM_POS
, 1, arg
);
18571 gimple_call_set_lhs (call
, id
);
18572 gimple_seq_add_stmt (seq
, call
);
18574 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18580 if (res
== NULL_TREE
)
18581 res
= integer_zero_node
;
18586 /* Transform IFN_GOACC_LOOP calls to actual code. See
18587 expand_oacc_for for where these are generated. At the vector
18588 level, we stride loops, such that each member of a warp will
18589 operate on adjacent iterations. At the worker and gang level,
18590 each gang/warp executes a set of contiguous iterations. Chunking
18591 can override this such that each iteration engine executes a
18592 contiguous chunk, and then moves on to stride to the next chunk. */
18595 oacc_xform_loop (gcall
*call
)
18597 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18598 enum ifn_goacc_loop_kind code
18599 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18600 tree dir
= gimple_call_arg (call
, 1);
18601 tree range
= gimple_call_arg (call
, 2);
18602 tree step
= gimple_call_arg (call
, 3);
18603 tree chunk_size
= NULL_TREE
;
18604 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18605 tree lhs
= gimple_call_lhs (call
);
18606 tree type
= TREE_TYPE (lhs
);
18607 tree diff_type
= TREE_TYPE (range
);
18608 tree r
= NULL_TREE
;
18609 gimple_seq seq
= NULL
;
18610 bool chunking
= false, striding
= true;
18611 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18612 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18614 #ifdef ACCEL_COMPILER
18615 chunk_size
= gimple_call_arg (call
, 4);
18616 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18617 || integer_zerop (chunk_size
)) /* Default (also static). */
18619 /* If we're at the gang level, we want each to execute a
18620 contiguous run of iterations. Otherwise we want each element
18622 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18627 /* Chunk of size 1 is striding. */
18628 striding
= integer_onep (chunk_size
);
18629 chunking
= !striding
;
18633 /* striding=true, chunking=true
18635 striding=true, chunking=false
18637 striding=false,chunking=true
18638 -> chunks=ceil (range/(chunksize*threads*step))
18639 striding=false,chunking=false
18640 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18641 push_gimplify_context (true);
18645 default: gcc_unreachable ();
18647 case IFN_GOACC_LOOP_CHUNKS
:
18649 r
= build_int_cst (type
, 1);
18653 = (range - dir) / (chunks * step * num_threads) + dir */
18654 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18655 per
= fold_convert (type
, per
);
18656 chunk_size
= fold_convert (type
, chunk_size
);
18657 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18658 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18659 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18660 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18661 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18665 case IFN_GOACC_LOOP_STEP
:
18667 /* If striding, step by the entire compute volume, otherwise
18668 step by the inner volume. */
18669 unsigned volume
= striding
? mask
: inner_mask
;
18671 r
= oacc_thread_numbers (false, volume
, &seq
);
18672 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18676 case IFN_GOACC_LOOP_OFFSET
:
18679 r
= oacc_thread_numbers (true, mask
, &seq
);
18680 r
= fold_convert (diff_type
, r
);
18684 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18685 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18686 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18687 inner_size
, outer_size
);
18689 volume
= fold_convert (diff_type
, volume
);
18691 chunk_size
= fold_convert (diff_type
, chunk_size
);
18694 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18696 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18697 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18698 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18701 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18702 fold_convert (diff_type
, inner_size
));
18703 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18704 r
= fold_convert (diff_type
, r
);
18705 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18707 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18708 inner
= fold_convert (diff_type
, inner
);
18709 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18713 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18715 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18716 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18718 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18721 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18722 if (type
!= diff_type
)
18723 r
= fold_convert (type
, r
);
18726 case IFN_GOACC_LOOP_BOUND
:
18731 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18732 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18733 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18734 inner_size
, outer_size
);
18736 volume
= fold_convert (diff_type
, volume
);
18738 chunk_size
= fold_convert (diff_type
, chunk_size
);
18741 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18743 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18744 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18745 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18748 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18749 fold_convert (diff_type
, inner_size
));
18751 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18753 tree offset
= gimple_call_arg (call
, 6);
18754 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18755 fold_convert (diff_type
, offset
));
18756 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18757 diff_type
, r
, range
);
18759 if (diff_type
!= type
)
18760 r
= fold_convert (type
, r
);
18764 gimplify_assign (lhs
, r
, &seq
);
18766 pop_gimplify_context (NULL
);
18768 gsi_replace_with_seq (&gsi
, seq
, true);
18771 /* Default partitioned and minimum partitioned dimensions. */
18773 static int oacc_default_dims
[GOMP_DIM_MAX
];
18774 static int oacc_min_dims
[GOMP_DIM_MAX
];
18776 /* Parse the default dimension parameter. This is a set of
18777 :-separated optional compute dimensions. Each specified dimension
18778 is a positive integer. When device type support is added, it is
18779 planned to be a comma separated list of such compute dimensions,
18780 with all but the first prefixed by the colon-terminated device
18784 oacc_parse_default_dims (const char *dims
)
18788 for (ix
= GOMP_DIM_MAX
; ix
--;)
18790 oacc_default_dims
[ix
] = -1;
18791 oacc_min_dims
[ix
] = 1;
18794 #ifndef ACCEL_COMPILER
18795 /* Cannot be overridden on the host. */
18800 const char *pos
= dims
;
18802 for (ix
= 0; *pos
&& ix
!= GOMP_DIM_MAX
; ix
++)
18817 val
= strtol (pos
, CONST_CAST (char **, &eptr
), 10);
18818 if (errno
|| val
<= 0 || (int) val
!= val
)
18821 oacc_default_dims
[ix
] = (int) val
;
18827 error_at (UNKNOWN_LOCATION
,
18828 "-fopenacc-dim operand is malformed at '%s'", pos
);
18832 /* Allow the backend to validate the dimensions. */
18833 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_default_dims
, -1);
18834 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_min_dims
, -2);
18837 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18838 raw attribute. DIMS is an array of dimensions, which is filled in.
18839 LEVEL is the partitioning level of a routine, or -1 for an offload
18840 region itself. USED is the mask of partitioned execution in the
18844 oacc_validate_dims (tree fn
, tree attrs
, int *dims
, int level
, unsigned used
)
18846 tree purpose
[GOMP_DIM_MAX
];
18848 tree pos
= TREE_VALUE (attrs
);
18849 bool is_kernel
= oacc_fn_attrib_kernels_p (attrs
);
18851 /* Make sure the attribute creator attached the dimension
18855 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18857 purpose
[ix
] = TREE_PURPOSE (pos
);
18858 tree val
= TREE_VALUE (pos
);
18859 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18860 pos
= TREE_CHAIN (pos
);
18863 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, level
);
18865 /* Default anything left to 1 or a partitioned default. */
18866 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18869 /* The OpenACC spec says 'If the [num_gangs] clause is not
18870 specified, an implementation-defined default will be used;
18871 the default may depend on the code within the construct.'
18872 (2.5.6). Thus an implementation is free to choose
18873 non-unity default for a parallel region that doesn't have
18874 any gang-partitioned loops. However, it appears that there
18875 is a sufficient body of user code that expects non-gang
18876 partitioned regions to not execute in gang-redundant mode.
18877 So we (a) don't warn about the non-portability and (b) pick
18878 the minimum permissible dimension size when there is no
18879 partitioned execution. Otherwise we pick the global
18880 default for the dimension, which the user can control. The
18881 same wording and logic applies to num_workers and
18882 vector_length, however the worker- or vector- single
18883 execution doesn't have the same impact as gang-redundant
18884 execution. (If the minimum gang-level partioning is not 1,
18885 the target is probably too confusing.) */
18886 dims
[ix
] = (used
& GOMP_DIM_MASK (ix
)
18887 ? oacc_default_dims
[ix
] : oacc_min_dims
[ix
]);
18893 /* Replace the attribute with new values. */
18895 for (ix
= GOMP_DIM_MAX
; ix
--;)
18897 pos
= tree_cons (purpose
[ix
],
18898 build_int_cst (integer_type_node
, dims
[ix
]),
18901 TREE_PUBLIC (pos
) = 1;
18903 replace_oacc_fn_attrib (fn
, pos
);
18907 /* Create an empty OpenACC loop structure at LOC. */
18910 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18912 oacc_loop
*loop
= XCNEW (oacc_loop
);
18914 loop
->parent
= parent
;
18915 loop
->child
= loop
->sibling
= NULL
;
18919 loop
->sibling
= parent
->child
;
18920 parent
->child
= loop
;
18924 loop
->marker
= NULL
;
18925 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18926 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18927 loop
->routine
= NULL_TREE
;
18929 loop
->mask
= loop
->flags
= 0;
18931 loop
->chunk_size
= 0;
18932 loop
->head_end
= NULL
;
18937 /* Create an outermost, dummy OpenACC loop for offloaded function
18941 new_oacc_loop_outer (tree decl
)
18943 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18946 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18947 Link into PARENT loop. Return the new loop. */
18950 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18952 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18954 loop
->marker
= marker
;
18956 /* TODO: This is where device_type flattening would occur for the loop
18959 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18961 tree chunk_size
= integer_zero_node
;
18962 if (loop
->flags
& OLF_GANG_STATIC
)
18963 chunk_size
= gimple_call_arg (marker
, 4);
18964 loop
->chunk_size
= chunk_size
;
18969 /* Create a dummy loop encompassing a call to a openACC routine.
18970 Extract the routine's partitioning requirements. */
18973 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18975 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18976 int level
= oacc_fn_attrib_level (attrs
);
18978 gcc_assert (level
>= 0);
18980 loop
->marker
= call
;
18981 loop
->routine
= decl
;
18982 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18983 ^ (GOMP_DIM_MASK (level
) - 1));
18986 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18987 Return the parent loop. */
18990 finish_oacc_loop (oacc_loop
*loop
)
18992 /* If the loop has been collapsed, don't partition it. */
18994 loop
->mask
= loop
->flags
= 0;
18995 return loop
->parent
;
18998 /* Free all OpenACC loop structures within LOOP (inclusive). */
19001 free_oacc_loop (oacc_loop
*loop
)
19004 free_oacc_loop (loop
->sibling
);
19006 free_oacc_loop (loop
->child
);
19011 /* Dump out the OpenACC loop head or tail beginning at FROM. */
19014 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
19015 const char *title
, int level
)
19017 enum ifn_unique_kind kind
19018 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19020 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
19021 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19023 gimple
*stmt
= gsi_stmt (gsi
);
19025 if (is_gimple_call (stmt
)
19026 && gimple_call_internal_p (stmt
)
19027 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19029 enum ifn_unique_kind k
19030 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
19031 (gimple_call_arg (stmt
, 0)));
19033 if (k
== kind
&& stmt
!= from
)
19036 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
19039 while (gsi_end_p (gsi
))
19040 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19044 /* Dump OpenACC loops LOOP, its siblings and its children. */
19047 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
19051 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
19052 loop
->flags
, loop
->mask
,
19053 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
19056 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
19059 fprintf (file
, "%*sRoutine %s:%u:%s\n",
19060 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
19061 DECL_SOURCE_LINE (loop
->routine
),
19062 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
19064 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
19065 if (loop
->heads
[ix
])
19066 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
19067 for (ix
= GOMP_DIM_MAX
; ix
--;)
19068 if (loop
->tails
[ix
])
19069 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
19072 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
19074 dump_oacc_loop (file
, loop
->sibling
, depth
);
19077 void debug_oacc_loop (oacc_loop
*);
19079 /* Dump loops to stderr. */
19081 DEBUG_FUNCTION
void
19082 debug_oacc_loop (oacc_loop
*loop
)
19084 dump_oacc_loop (stderr
, loop
, 0);
19087 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
19088 structures as we go. By construction these loops are properly
19092 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
19097 if (bb
->flags
& BB_VISITED
)
19101 bb
->flags
|= BB_VISITED
;
19103 /* Scan for loop markers. */
19104 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
19107 gimple
*stmt
= gsi_stmt (gsi
);
19109 if (!is_gimple_call (stmt
))
19112 gcall
*call
= as_a
<gcall
*> (stmt
);
19114 /* If this is a routine, make a dummy loop for it. */
19115 if (tree decl
= gimple_call_fndecl (call
))
19116 if (tree attrs
= get_oacc_fn_attrib (decl
))
19118 gcc_assert (!marker
);
19119 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
19122 if (!gimple_call_internal_p (call
))
19125 switch (gimple_call_internal_fn (call
))
19130 case IFN_GOACC_LOOP
:
19131 /* Count the goacc loop abstraction fns, to determine if the
19132 loop was collapsed already. */
19137 enum ifn_unique_kind kind
19138 = (enum ifn_unique_kind
) (TREE_INT_CST_LOW
19139 (gimple_call_arg (call
, 0)));
19140 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
19141 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19143 if (gimple_call_num_args (call
) == 2)
19145 gcc_assert (marker
&& !remaining
);
19147 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19148 loop
= finish_oacc_loop (loop
);
19150 loop
->head_end
= call
;
19154 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
19158 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19159 loop
= new_oacc_loop (loop
, call
);
19162 gcc_assert (count
== remaining
);
19166 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19167 loop
->heads
[marker
] = call
;
19169 loop
->tails
[remaining
] = call
;
19176 if (remaining
|| marker
)
19178 bb
= single_succ (bb
);
19179 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
19183 /* Walk successor blocks. */
19187 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
19188 oacc_loop_discover_walk (loop
, e
->dest
);
19191 /* LOOP is the first sibling. Reverse the order in place and return
19192 the new first sibling. Recurse to child loops. */
19195 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
19197 oacc_loop
*last
= NULL
;
19201 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
19203 oacc_loop
*next
= loop
->sibling
;
19204 loop
->sibling
= last
;
19213 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19214 the current function. */
19217 oacc_loop_discovery ()
19221 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19222 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19224 /* The siblings were constructed in reverse order, reverse them so
19225 that diagnostics come out in an unsurprising order. */
19226 top
= oacc_loop_sibling_nreverse (top
);
19228 /* Reset the visited flags. */
19229 FOR_ALL_BB_FN (bb
, cfun
)
19230 bb
->flags
&= ~BB_VISITED
;
19235 /* Transform the abstract internal function markers starting at FROM
19236 to be for partitioning level LEVEL. Stop when we meet another HEAD
19240 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19242 enum ifn_unique_kind kind
19243 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19244 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19246 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19248 gimple
*stmt
= gsi_stmt (gsi
);
19250 if (is_gimple_call (stmt
)
19251 && gimple_call_internal_p (stmt
)
19252 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19254 enum ifn_unique_kind k
19255 = ((enum ifn_unique_kind
)
19256 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19258 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19259 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19260 else if (k
== kind
&& stmt
!= from
)
19263 else if (is_gimple_call (stmt
)
19264 && gimple_call_internal_p (stmt
)
19265 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
19266 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
19269 while (gsi_end_p (gsi
))
19270 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19274 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19275 determined partitioning mask and chunking argument. END_MARKER
19276 points at the end IFN_HEAD_TAIL call intgroducing the loop. IFNS
19277 is the number of IFN_GOACC_LOOP calls for the loop. MASK_ARG is
19278 the replacement partitioning mask and CHUNK_ARG is the replacement
19282 oacc_loop_xform_loop (gcall
*end_marker
, unsigned ifns
,
19283 tree mask_arg
, tree chunk_arg
)
19285 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19287 gcc_checking_assert (ifns
);
19290 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19292 gimple
*stmt
= gsi_stmt (gsi
);
19294 if (!is_gimple_call (stmt
))
19297 gcall
*call
= as_a
<gcall
*> (stmt
);
19299 if (!gimple_call_internal_p (call
))
19302 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19305 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19306 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19312 /* The LOOP_BOUND ifn could be in the single successor
19314 basic_block bb
= single_succ (gsi_bb (gsi
));
19315 gsi
= gsi_start_bb (bb
);
19319 /* Process the discovered OpenACC loops, setting the correct
19320 partitioning level etc. */
19323 oacc_loop_process (oacc_loop
*loop
)
19326 oacc_loop_process (loop
->child
);
19328 if (loop
->mask
&& !loop
->routine
)
19331 unsigned mask
= loop
->mask
;
19332 unsigned dim
= GOMP_DIM_GANG
;
19333 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19334 tree chunk_arg
= loop
->chunk_size
;
19336 oacc_loop_xform_loop (loop
->head_end
, loop
->ifns
, mask_arg
, chunk_arg
);
19338 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
19342 while (!(GOMP_DIM_MASK (dim
) & mask
))
19345 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19346 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19348 mask
^= GOMP_DIM_MASK (dim
);
19353 oacc_loop_process (loop
->sibling
);
19356 /* Walk the OpenACC loop heirarchy checking and assigning the
19357 programmer-specified partitionings. OUTER_MASK is the partitioning
19358 this loop is contained within. Return mask of partitioning
19359 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
19363 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19365 unsigned this_mask
= loop
->mask
;
19366 unsigned mask_all
= 0;
19369 #ifdef ACCEL_COMPILER
19370 /* When device_type is supported, we want the device compiler to be
19371 noisy, if the loop parameters are device_type-specific. */
19375 if (!loop
->routine
)
19377 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19378 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19380 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19381 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19383 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19386 error_at (loop
->loc
,
19388 ? "%<seq%> overrides other OpenACC loop specifiers"
19389 : "%<auto%> conflicts with other OpenACC loop specifiers");
19391 loop
->flags
&= ~OLF_AUTO
;
19395 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19399 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19400 mask_all
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19403 if (this_mask
& outer_mask
)
19405 const oacc_loop
*outer
;
19406 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19407 if (outer
->mask
& this_mask
)
19414 error_at (loop
->loc
,
19415 "%s uses same OpenACC parallelism as containing loop",
19416 loop
->routine
? "routine call" : "inner loop");
19417 inform (outer
->loc
, "containing loop here");
19420 error_at (loop
->loc
,
19421 "%s uses OpenACC parallelism disallowed by containing routine",
19422 loop
->routine
? "routine call" : "loop");
19425 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19426 "routine %qD declared here", loop
->routine
);
19428 this_mask
&= ~outer_mask
;
19432 unsigned outermost
= this_mask
& -this_mask
;
19434 if (outermost
&& outermost
<= outer_mask
)
19438 error_at (loop
->loc
,
19439 "incorrectly nested OpenACC loop parallelism");
19441 const oacc_loop
*outer
;
19442 for (outer
= loop
->parent
;
19443 outer
->flags
&& outer
->flags
< outermost
;
19444 outer
= outer
->parent
)
19446 inform (outer
->loc
, "containing loop here");
19449 this_mask
&= ~outermost
;
19453 loop
->mask
= this_mask
;
19454 mask_all
|= this_mask
;
19457 mask_all
|= oacc_loop_fixed_partitions (loop
->child
,
19458 outer_mask
| this_mask
);
19461 mask_all
|= oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
);
19466 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
19467 OUTER_MASK is the partitioning this loop is contained within.
19468 Return the cumulative partitioning used by this loop, siblings and
19472 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19474 unsigned inner_mask
= 0;
19477 #ifdef ACCEL_COMPILER
19478 /* When device_type is supported, we want the device compiler to be
19479 noisy, if the loop parameters are device_type-specific. */
19484 inner_mask
|= oacc_loop_auto_partitions (loop
->child
,
19485 outer_mask
| loop
->mask
);
19487 if ((loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
))
19489 unsigned this_mask
= 0;
19491 /* Determine the outermost partitioning used within this loop. */
19492 this_mask
= inner_mask
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
19493 this_mask
= (this_mask
& -this_mask
);
19495 /* Pick the partitioning just inside that one. */
19498 /* And avoid picking one use by an outer loop. */
19499 this_mask
&= ~outer_mask
;
19501 if (!this_mask
&& noisy
)
19502 warning_at (loop
->loc
, 0,
19503 "insufficient partitioning available to parallelize loop");
19506 fprintf (dump_file
, "Auto loop %s:%d assigned %d\n",
19507 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
19510 loop
->mask
= this_mask
;
19512 inner_mask
|= loop
->mask
;
19515 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
19520 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19521 axes. Return mask of partitioning. */
19524 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
19526 unsigned mask_all
= oacc_loop_fixed_partitions (loop
, outer_mask
);
19528 if (mask_all
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
19530 mask_all
^= GOMP_DIM_MASK (GOMP_DIM_MAX
);
19531 mask_all
|= oacc_loop_auto_partitions (loop
, outer_mask
);
19536 /* Default fork/join early expander. Delete the function calls if
19537 there is no RTL expander. */
19540 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19541 const int *ARG_UNUSED (dims
), bool is_fork
)
19544 return targetm
.have_oacc_fork ();
19546 return targetm
.have_oacc_join ();
19549 /* Default goacc.reduction early expander.
19551 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19552 If RES_PTR is not integer-zerop:
19553 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19554 TEARDOWN - emit '*RES_PTR = VAR'
19556 emit 'LHS = VAR' */
19559 default_goacc_reduction (gcall
*call
)
19561 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19562 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19563 tree lhs
= gimple_call_lhs (call
);
19564 tree var
= gimple_call_arg (call
, 2);
19565 gimple_seq seq
= NULL
;
19567 if (code
== IFN_GOACC_REDUCTION_SETUP
19568 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19570 /* Setup and Teardown need to copy from/to the receiver object,
19571 if there is one. */
19572 tree ref_to_res
= gimple_call_arg (call
, 1);
19574 if (!integer_zerop (ref_to_res
))
19576 tree dst
= build_simple_mem_ref (ref_to_res
);
19579 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19585 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19589 /* Copy VAR to LHS, if there is an LHS. */
19591 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19593 gsi_replace_with_seq (&gsi
, seq
, true);
19596 /* Main entry point for oacc transformations which run on the device
19597 compiler after LTO, so we know what the target device is at this
19598 point (including the host fallback). */
19600 static unsigned int
19601 execute_oacc_device_lower ()
19603 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19606 /* Not an offloaded function. */
19609 /* Parse the default dim argument exactly once. */
19610 if ((const void *)flag_openacc_dims
!= &flag_openacc_dims
)
19612 oacc_parse_default_dims (flag_openacc_dims
);
19613 flag_openacc_dims
= (char *)&flag_openacc_dims
;
19616 /* Discover, partition and process the loops. */
19617 oacc_loop
*loops
= oacc_loop_discovery ();
19618 int fn_level
= oacc_fn_attrib_level (attrs
);
19621 fprintf (dump_file
, oacc_fn_attrib_kernels_p (attrs
)
19622 ? "Function is kernels offload\n"
19623 : fn_level
< 0 ? "Function is parallel offload\n"
19624 : "Function is routine level %d\n", fn_level
);
19626 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
19627 unsigned used_mask
= oacc_loop_partition (loops
, outer_mask
);
19628 int dims
[GOMP_DIM_MAX
];
19630 oacc_validate_dims (current_function_decl
, attrs
, dims
, fn_level
, used_mask
);
19634 const char *comma
= "Compute dimensions [";
19635 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, comma
= ", ")
19636 fprintf (dump_file
, "%s%d", comma
, dims
[ix
]);
19637 fprintf (dump_file
, "]\n");
19640 oacc_loop_process (loops
);
19643 fprintf (dump_file
, "OpenACC loops\n");
19644 dump_oacc_loop (dump_file
, loops
, 0);
19645 fprintf (dump_file
, "\n");
19648 /* Offloaded targets may introduce new basic blocks, which require
19649 dominance information to update SSA. */
19650 calculate_dominance_info (CDI_DOMINATORS
);
19652 /* Now lower internal loop functions to target-specific code
19655 FOR_ALL_BB_FN (bb
, cfun
)
19656 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19658 gimple
*stmt
= gsi_stmt (gsi
);
19659 if (!is_gimple_call (stmt
))
19665 gcall
*call
= as_a
<gcall
*> (stmt
);
19666 if (!gimple_call_internal_p (call
))
19672 /* Rewind to allow rescan. */
19674 bool rescan
= false, remove
= false;
19675 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19681 case IFN_GOACC_LOOP
:
19682 oacc_xform_loop (call
);
19686 case IFN_GOACC_REDUCTION
:
19687 /* Mark the function for SSA renaming. */
19688 mark_virtual_operands_for_renaming (cfun
);
19690 /* If the level is -1, this ended up being an unused
19691 axis. Handle as a default. */
19692 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19693 default_goacc_reduction (call
);
19695 targetm
.goacc
.reduction (call
);
19701 enum ifn_unique_kind kind
19702 = ((enum ifn_unique_kind
)
19703 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19708 gcc_unreachable ();
19710 case IFN_UNIQUE_OACC_FORK
:
19711 case IFN_UNIQUE_OACC_JOIN
:
19712 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19714 else if (!targetm
.goacc
.fork_join
19715 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19719 case IFN_UNIQUE_OACC_HEAD_MARK
:
19720 case IFN_UNIQUE_OACC_TAIL_MARK
:
19728 if (gsi_end_p (gsi
))
19729 /* We rewound past the beginning of the BB. */
19730 gsi
= gsi_start_bb (bb
);
19732 /* Undo the rewind. */
19737 if (gimple_vdef (call
))
19738 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19739 if (gimple_call_lhs (call
))
19741 /* Propagate the data dependency var. */
19742 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19743 gimple_call_arg (call
, 1));
19744 gsi_replace (&gsi
, ass
, false);
19747 gsi_remove (&gsi
, true);
19750 /* If not rescanning, advance over the call. */
19754 free_oacc_loop (loops
);
19759 /* Default launch dimension validator. Force everything to 1. A
19760 backend that wants to provide larger dimensions must override this
19764 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19765 int ARG_UNUSED (fn_level
))
19767 bool changed
= false;
19769 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19781 /* Default dimension bound is unknown on accelerator and 1 on host. */
19784 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19786 #ifdef ACCEL_COMPILER
19795 const pass_data pass_data_oacc_device_lower
=
19797 GIMPLE_PASS
, /* type */
19798 "oaccdevlow", /* name */
19799 OPTGROUP_NONE
, /* optinfo_flags */
19800 TV_NONE
, /* tv_id */
19801 PROP_cfg
, /* properties_required */
19802 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19803 0, /* properties_destroyed */
19804 0, /* todo_flags_start */
19805 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19808 class pass_oacc_device_lower
: public gimple_opt_pass
19811 pass_oacc_device_lower (gcc::context
*ctxt
)
19812 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19815 /* opt_pass methods: */
19816 virtual unsigned int execute (function
*)
19818 bool gate
= flag_openacc
!= 0;
19823 return execute_oacc_device_lower ();
19826 }; // class pass_oacc_device_lower
19828 } // anon namespace
19831 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19833 return new pass_oacc_device_lower (ctxt
);
19836 /* "omp declare target link" handling pass. */
19840 const pass_data pass_data_omp_target_link
=
19842 GIMPLE_PASS
, /* type */
19843 "omptargetlink", /* name */
19844 OPTGROUP_NONE
, /* optinfo_flags */
19845 TV_NONE
, /* tv_id */
19846 PROP_ssa
, /* properties_required */
19847 0, /* properties_provided */
19848 0, /* properties_destroyed */
19849 0, /* todo_flags_start */
19850 TODO_update_ssa
, /* todo_flags_finish */
19853 class pass_omp_target_link
: public gimple_opt_pass
19856 pass_omp_target_link (gcc::context
*ctxt
)
19857 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
19860 /* opt_pass methods: */
19861 virtual bool gate (function
*fun
)
19863 #ifdef ACCEL_COMPILER
19864 tree attrs
= DECL_ATTRIBUTES (fun
->decl
);
19865 return lookup_attribute ("omp declare target", attrs
)
19866 || lookup_attribute ("omp target entrypoint", attrs
);
19873 virtual unsigned execute (function
*);
19876 /* Callback for walk_gimple_stmt used to scan for link var operands. */
19879 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
19883 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
)
19884 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
19886 *walk_subtrees
= 0;
19894 pass_omp_target_link::execute (function
*fun
)
19897 FOR_EACH_BB_FN (bb
, fun
)
19899 gimple_stmt_iterator gsi
;
19900 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
19901 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
19902 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
19908 } // anon namespace
19911 make_pass_omp_target_link (gcc::context
*ctxt
)
19913 return new pass_omp_target_link (ctxt
);
19916 #include "gt-omp-low.h"