1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
38 #include "internal-fn.h"
39 #include "gimple-fold.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
50 #include "tree-into-ssa.h"
52 #include "insn-config.h"
63 #include "tree-pass.h"
65 #include "splay-tree.h"
66 #include "insn-codes.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "pretty-print.h"
75 #include "alloc-pool.h"
76 #include "symbol-summary.h"
78 #include "tree-nested.h"
82 #include "lto-section-names.h"
83 #include "gomp-constants.h"
84 #include "gimple-pretty-print.h"
86 /* Lowering of OMP parallel and workshare constructs proceeds in two
87 phases. The first phase scans the function looking for OMP statements
88 and then for variables that must be replaced to satisfy data sharing
89 clauses. The second phase expands code for the constructs, as well as
90 re-gimplifying things when variables have been replaced with complex
93 Final code generation is done by pass_expand_omp. The flowgraph is
94 scanned for regions which are then moved to a new
95 function, to be invoked by the thread library, or offloaded. */
97 /* OMP region information. Every parallel and workshare
98 directive is enclosed between two markers, the OMP_* directive
99 and a corresponding GIMPLE_OMP_RETURN statement. */
103 /* The enclosing region. */
104 struct omp_region
*outer
;
106 /* First child region. */
107 struct omp_region
*inner
;
109 /* Next peer region. */
110 struct omp_region
*next
;
112 /* Block containing the omp directive as its last stmt. */
115 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
118 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
121 /* If this is a combined parallel+workshare region, this is a list
122 of additional arguments needed by the combined parallel+workshare
124 vec
<tree
, va_gc
> *ws_args
;
126 /* The code for the omp directive of this region. */
127 enum gimple_code type
;
129 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
130 enum omp_clause_schedule_kind sched_kind
;
132 /* True if this is a combined parallel+workshare region. */
133 bool is_combined_parallel
;
135 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
137 gomp_ordered
*ord_stmt
;
140 /* Levels of parallelism as defined by OpenACC. Increasing numbers
141 correspond to deeper loop nesting levels. */
143 #define MASK_WORKER 2
144 #define MASK_VECTOR 4
146 /* Context structure. Used to store information about each parallel
147 directive in the code. */
151 /* This field must be at the beginning, as we do "inheritance": Some
152 callback functions for tree-inline.c (e.g., omp_copy_decl)
153 receive a copy_body_data pointer that is up-casted to an
154 omp_context pointer. */
157 /* The tree of contexts corresponding to the encountered constructs. */
158 struct omp_context
*outer
;
161 /* Map variables to fields in a structure that allows communication
162 between sending and receiving threads. */
163 splay_tree field_map
;
168 /* These are used just by task contexts, if task firstprivate fn is
169 needed. srecord_type is used to communicate from the thread
170 that encountered the task construct to task firstprivate fn,
171 record_type is allocated by GOMP_task, initialized by task firstprivate
172 fn and passed to the task body fn. */
173 splay_tree sfield_map
;
176 /* A chain of variables to add to the top-level block surrounding the
177 construct. In the case of a parallel, this is in the child function. */
180 /* A map of reduction pointer variables. For accelerators, each
181 reduction variable is replaced with an array. Each thread, in turn,
182 is assigned to a slot on that array. */
183 splay_tree reduction_map
;
185 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
186 barriers should jump to during omplower pass. */
189 /* What to do with variables with implicitly determined sharing
191 enum omp_clause_default_kind default_kind
;
193 /* Nesting depth of this context. Used to beautify error messages re
194 invalid gotos. The outermost ctx is depth 1, with depth 0 being
195 reserved for the main body of the function. */
198 /* True if this parallel directive is nested within another. */
201 /* True if this construct can be cancelled. */
205 /* A structure holding the elements of:
206 for (V = N1; V cond N2; V += STEP) [...] */
208 struct omp_for_data_loop
210 tree v
, n1
, n2
, step
;
211 enum tree_code cond_code
;
214 /* A structure describing the main elements of a parallel loop. */
218 struct omp_for_data_loop loop
;
224 bool have_nowait
, have_ordered
, simd_schedule
;
225 enum omp_clause_schedule_kind sched_kind
;
226 struct omp_for_data_loop
*loops
;
229 /* Describe the OpenACC looping structure of a function. The entire
230 function is held in a 'NULL' loop. */
234 oacc_loop
*parent
; /* Containing loop. */
236 oacc_loop
*child
; /* First inner loop. */
238 oacc_loop
*sibling
; /* Next loop within same parent. */
240 location_t loc
; /* Location of the loop start. */
242 gcall
*marker
; /* Initial head marker. */
244 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
245 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
247 tree routine
; /* Pseudo-loop enclosing a routine. */
249 unsigned mask
; /* Partitioning mask. */
250 unsigned flags
; /* Partitioning flags. */
251 tree chunk_size
; /* Chunk size. */
252 gcall
*head_end
; /* Final marker of head sequence. */
255 /* Flags for an OpenACC loop. */
257 enum oacc_loop_flags
{
258 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
259 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
260 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
261 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
263 /* Explicitly specified loop axes. */
265 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
266 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
267 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
269 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
273 static splay_tree all_contexts
;
274 static int taskreg_nesting_level
;
275 static int target_nesting_level
;
276 static struct omp_region
*root_omp_region
;
277 static bitmap task_shared_vars
;
278 static vec
<omp_context
*> taskreg_contexts
;
280 static void scan_omp (gimple_seq
*, omp_context
*);
281 static tree
scan_omp_1_op (tree
*, int *, void *);
282 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
284 #define WALK_SUBSTMTS \
288 case GIMPLE_EH_FILTER: \
289 case GIMPLE_TRANSACTION: \
290 /* The sub-statements for these should be walked. */ \
291 *handled_ops_p = false; \
294 /* Return true if CTX corresponds to an oacc parallel region. */
297 is_oacc_parallel (omp_context
*ctx
)
299 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
300 return ((outer_type
== GIMPLE_OMP_TARGET
)
301 && (gimple_omp_target_kind (ctx
->stmt
)
302 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
305 /* Return true if CTX corresponds to an oacc kernels region. */
308 is_oacc_kernels (omp_context
*ctx
)
310 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
311 return ((outer_type
== GIMPLE_OMP_TARGET
)
312 && (gimple_omp_target_kind (ctx
->stmt
)
313 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
316 /* Helper function to get the name of the array containing the partial
317 reductions for OpenACC reductions. */
319 oacc_get_reduction_array_id (tree node
)
321 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
322 int len
= strlen ("OACC") + strlen (id
);
323 char *temp_name
= XALLOCAVEC (char, len
+ 1);
324 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
325 return IDENTIFIER_POINTER (get_identifier (temp_name
));
328 /* Determine the number of threads OpenACC threads used to determine the
329 size of the array of partial reductions. Currently, this is num_gangs
330 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
331 because it is independed of the device used. */
334 oacc_max_threads (omp_context
*ctx
)
336 tree nthreads
, vector_length
, gangs
, clauses
;
338 gangs
= fold_convert (sizetype
, integer_one_node
);
339 vector_length
= gangs
;
341 /* The reduction clause may be nested inside a loop directive.
342 Scan for the innermost vector_length clause. */
343 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
345 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
346 || (gimple_omp_target_kind (oc
->stmt
)
347 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
350 clauses
= gimple_omp_target_clauses (oc
->stmt
);
352 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
354 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
356 OMP_CLAUSE_VECTOR_LENGTH_EXPR
359 vector_length
= fold_convert (sizetype
, integer_one_node
);
361 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
363 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
364 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
366 gangs
= fold_convert (sizetype
, integer_one_node
);
371 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
376 /* If DECL is the artificial dummy VAR_DECL created for non-static
377 data member privatization, return the underlying "this" parameter,
378 otherwise return NULL. */
381 omp_member_access_dummy_var (tree decl
)
384 || !DECL_ARTIFICIAL (decl
)
385 || !DECL_IGNORED_P (decl
)
386 || !DECL_HAS_VALUE_EXPR_P (decl
)
387 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
390 tree v
= DECL_VALUE_EXPR (decl
);
391 if (TREE_CODE (v
) != COMPONENT_REF
)
395 switch (TREE_CODE (v
))
401 case POINTER_PLUS_EXPR
:
402 v
= TREE_OPERAND (v
, 0);
405 if (DECL_CONTEXT (v
) == current_function_decl
406 && DECL_ARTIFICIAL (v
)
407 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
415 /* Helper for unshare_and_remap, called through walk_tree. */
418 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
420 tree
*pair
= (tree
*) data
;
423 *tp
= unshare_expr (pair
[1]);
426 else if (IS_TYPE_OR_DECL_P (*tp
))
431 /* Return unshare_expr (X) with all occurrences of FROM
435 unshare_and_remap (tree x
, tree from
, tree to
)
437 tree pair
[2] = { from
, to
};
438 x
= unshare_expr (x
);
439 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
443 /* Holds offload tables with decls. */
444 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
446 /* Convenience function for calling scan_omp_1_op on tree operands. */
449 scan_omp_op (tree
*tp
, omp_context
*ctx
)
451 struct walk_stmt_info wi
;
453 memset (&wi
, 0, sizeof (wi
));
455 wi
.want_locations
= true;
457 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
460 static void lower_omp (gimple_seq
*, omp_context
*);
461 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
462 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
464 /* Find an OMP clause of type KIND within CLAUSES. */
467 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
469 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
470 if (OMP_CLAUSE_CODE (clauses
) == kind
)
476 /* Return true if CTX is for an omp parallel. */
479 is_parallel_ctx (omp_context
*ctx
)
481 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
485 /* Return true if CTX is for an omp task. */
488 is_task_ctx (omp_context
*ctx
)
490 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
494 /* Return true if CTX is for an omp taskloop. */
497 is_taskloop_ctx (omp_context
*ctx
)
499 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
500 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
504 /* Return true if CTX is for an omp parallel or omp task. */
507 is_taskreg_ctx (omp_context
*ctx
)
509 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
513 /* Return true if REGION is a combined parallel+workshare region. */
516 is_combined_parallel (struct omp_region
*region
)
518 return region
->is_combined_parallel
;
522 /* Extract the header elements of parallel loop FOR_STMT and store
526 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
527 struct omp_for_data_loop
*loops
)
529 tree t
, var
, *collapse_iter
, *collapse_count
;
530 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
531 struct omp_for_data_loop
*loop
;
533 struct omp_for_data_loop dummy_loop
;
534 location_t loc
= gimple_location (for_stmt
);
535 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
536 bool distribute
= gimple_omp_for_kind (for_stmt
)
537 == GF_OMP_FOR_KIND_DISTRIBUTE
;
538 bool taskloop
= gimple_omp_for_kind (for_stmt
)
539 == GF_OMP_FOR_KIND_TASKLOOP
;
542 fd
->for_stmt
= for_stmt
;
544 if (gimple_omp_for_collapse (for_stmt
) > 1)
547 fd
->loops
= &fd
->loop
;
549 fd
->have_nowait
= distribute
|| simd
;
550 fd
->have_ordered
= false;
553 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
554 fd
->chunk_size
= NULL_TREE
;
555 fd
->simd_schedule
= false;
556 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
557 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
558 collapse_iter
= NULL
;
559 collapse_count
= NULL
;
561 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
562 switch (OMP_CLAUSE_CODE (t
))
564 case OMP_CLAUSE_NOWAIT
:
565 fd
->have_nowait
= true;
567 case OMP_CLAUSE_ORDERED
:
568 fd
->have_ordered
= true;
569 if (OMP_CLAUSE_ORDERED_EXPR (t
))
570 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
572 case OMP_CLAUSE_SCHEDULE
:
573 gcc_assert (!distribute
&& !taskloop
);
574 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
575 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
576 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
578 case OMP_CLAUSE_DIST_SCHEDULE
:
579 gcc_assert (distribute
);
580 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
582 case OMP_CLAUSE_COLLAPSE
:
583 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
584 if (fd
->collapse
> 1)
586 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
587 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
593 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
598 collapse_iter
= &iterv
;
599 collapse_count
= &countv
;
602 /* FIXME: for now map schedule(auto) to schedule(static).
603 There should be analysis to determine whether all iterations
604 are approximately the same amount of work (then schedule(static)
605 is best) or if it varies (then schedule(dynamic,N) is better). */
606 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
608 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
609 gcc_assert (fd
->chunk_size
== NULL
);
611 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
613 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
614 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
615 gcc_assert (fd
->chunk_size
== NULL
);
616 else if (fd
->chunk_size
== NULL
)
618 /* We only need to compute a default chunk size for ordered
619 static loops and dynamic loops. */
620 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
622 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
623 ? integer_zero_node
: integer_one_node
;
626 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
627 for (i
= 0; i
< cnt
; i
++)
629 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
631 else if (loops
!= NULL
)
636 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
637 gcc_assert (SSA_VAR_P (loop
->v
));
638 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
639 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
640 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
641 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
643 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
644 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
645 switch (loop
->cond_code
)
651 gcc_assert (gimple_omp_for_kind (for_stmt
)
652 == GF_OMP_FOR_KIND_CILKSIMD
653 || (gimple_omp_for_kind (for_stmt
)
654 == GF_OMP_FOR_KIND_CILKFOR
));
657 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
658 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
660 loop
->n2
= fold_build2_loc (loc
,
661 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
662 build_int_cst (TREE_TYPE (loop
->n2
), 1));
663 loop
->cond_code
= LT_EXPR
;
666 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
667 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
669 loop
->n2
= fold_build2_loc (loc
,
670 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
671 build_int_cst (TREE_TYPE (loop
->n2
), 1));
672 loop
->cond_code
= GT_EXPR
;
678 t
= gimple_omp_for_incr (for_stmt
, i
);
679 gcc_assert (TREE_OPERAND (t
, 0) == var
);
680 switch (TREE_CODE (t
))
683 loop
->step
= TREE_OPERAND (t
, 1);
685 case POINTER_PLUS_EXPR
:
686 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
689 loop
->step
= TREE_OPERAND (t
, 1);
690 loop
->step
= fold_build1_loc (loc
,
691 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
699 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
700 && !fd
->have_ordered
))
702 if (fd
->collapse
== 1)
703 iter_type
= TREE_TYPE (loop
->v
);
705 || TYPE_PRECISION (iter_type
)
706 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
708 = build_nonstandard_integer_type
709 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
711 else if (iter_type
!= long_long_unsigned_type_node
)
713 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
714 iter_type
= long_long_unsigned_type_node
;
715 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
716 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
717 >= TYPE_PRECISION (iter_type
))
721 if (loop
->cond_code
== LT_EXPR
)
722 n
= fold_build2_loc (loc
,
723 PLUS_EXPR
, TREE_TYPE (loop
->v
),
724 loop
->n2
, loop
->step
);
727 if (TREE_CODE (n
) != INTEGER_CST
728 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
729 iter_type
= long_long_unsigned_type_node
;
731 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
732 > TYPE_PRECISION (iter_type
))
736 if (loop
->cond_code
== LT_EXPR
)
739 n2
= fold_build2_loc (loc
,
740 PLUS_EXPR
, TREE_TYPE (loop
->v
),
741 loop
->n2
, loop
->step
);
745 n1
= fold_build2_loc (loc
,
746 MINUS_EXPR
, TREE_TYPE (loop
->v
),
747 loop
->n2
, loop
->step
);
750 if (TREE_CODE (n1
) != INTEGER_CST
751 || TREE_CODE (n2
) != INTEGER_CST
752 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
753 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
754 iter_type
= long_long_unsigned_type_node
;
758 if (i
>= fd
->collapse
)
761 if (collapse_count
&& *collapse_count
== NULL
)
763 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
764 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
765 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
766 if (t
&& integer_zerop (t
))
767 count
= build_zero_cst (long_long_unsigned_type_node
);
768 else if ((i
== 0 || count
!= NULL_TREE
)
769 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
770 && TREE_CONSTANT (loop
->n1
)
771 && TREE_CONSTANT (loop
->n2
)
772 && TREE_CODE (loop
->step
) == INTEGER_CST
)
774 tree itype
= TREE_TYPE (loop
->v
);
776 if (POINTER_TYPE_P (itype
))
777 itype
= signed_type_for (itype
);
778 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
779 t
= fold_build2_loc (loc
,
781 fold_convert_loc (loc
, itype
, loop
->step
), t
);
782 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
783 fold_convert_loc (loc
, itype
, loop
->n2
));
784 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
785 fold_convert_loc (loc
, itype
, loop
->n1
));
786 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
787 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
788 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
789 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
790 fold_convert_loc (loc
, itype
,
793 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
794 fold_convert_loc (loc
, itype
, loop
->step
));
795 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
796 if (count
!= NULL_TREE
)
797 count
= fold_build2_loc (loc
,
798 MULT_EXPR
, long_long_unsigned_type_node
,
802 if (TREE_CODE (count
) != INTEGER_CST
)
805 else if (count
&& !integer_zerop (count
))
812 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
813 || fd
->have_ordered
))
815 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
816 iter_type
= long_long_unsigned_type_node
;
818 iter_type
= long_integer_type_node
;
820 else if (collapse_iter
&& *collapse_iter
!= NULL
)
821 iter_type
= TREE_TYPE (*collapse_iter
);
822 fd
->iter_type
= iter_type
;
823 if (collapse_iter
&& *collapse_iter
== NULL
)
824 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
825 if (collapse_count
&& *collapse_count
== NULL
)
828 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
830 *collapse_count
= create_tmp_var (iter_type
, ".count");
833 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
835 fd
->loop
.v
= *collapse_iter
;
836 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
837 fd
->loop
.n2
= *collapse_count
;
838 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
839 fd
->loop
.cond_code
= LT_EXPR
;
844 /* For OpenACC loops, force a chunk size of one, as this avoids the default
845 scheduling where several subsequent iterations are being executed by the
847 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
849 gcc_assert (fd
->chunk_size
== NULL_TREE
);
850 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
855 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
856 is the immediate dominator of PAR_ENTRY_BB, return true if there
857 are no data dependencies that would prevent expanding the parallel
858 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
860 When expanding a combined parallel+workshare region, the call to
861 the child function may need additional arguments in the case of
862 GIMPLE_OMP_FOR regions. In some cases, these arguments are
863 computed out of variables passed in from the parent to the child
864 via 'struct .omp_data_s'. For instance:
866 #pragma omp parallel for schedule (guided, i * 4)
871 # BLOCK 2 (PAR_ENTRY_BB)
873 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
875 # BLOCK 3 (WS_ENTRY_BB)
876 .omp_data_i = &.omp_data_o;
877 D.1667 = .omp_data_i->i;
879 #pragma omp for schedule (guided, D.1598)
881 When we outline the parallel region, the call to the child function
882 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
883 that value is computed *after* the call site. So, in principle we
884 cannot do the transformation.
886 To see whether the code in WS_ENTRY_BB blocks the combined
887 parallel+workshare call, we collect all the variables used in the
888 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
889 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
892 FIXME. If we had the SSA form built at this point, we could merely
893 hoist the code in block 3 into block 2 and be done with it. But at
894 this point we don't have dataflow information and though we could
895 hack something up here, it is really not worth the aggravation. */
898 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
900 struct omp_for_data fd
;
901 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
903 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
906 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
908 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
910 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
912 if (fd
.iter_type
!= long_integer_type_node
)
915 /* FIXME. We give up too easily here. If any of these arguments
916 are not constants, they will likely involve variables that have
917 been mapped into fields of .omp_data_s for sharing with the child
918 function. With appropriate data flow, it would be possible to
920 if (!is_gimple_min_invariant (fd
.loop
.n1
)
921 || !is_gimple_min_invariant (fd
.loop
.n2
)
922 || !is_gimple_min_invariant (fd
.loop
.step
)
923 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
930 static int omp_max_vf (void);
932 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
933 presence (SIMD_SCHEDULE). */
936 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
941 int vf
= omp_max_vf ();
945 tree type
= TREE_TYPE (chunk_size
);
946 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
947 build_int_cst (type
, vf
- 1));
948 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
949 build_int_cst (type
, -vf
));
953 /* Collect additional arguments needed to emit a combined
954 parallel+workshare call. WS_STMT is the workshare directive being
957 static vec
<tree
, va_gc
> *
958 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
961 location_t loc
= gimple_location (ws_stmt
);
962 vec
<tree
, va_gc
> *ws_args
;
964 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
966 struct omp_for_data fd
;
969 extract_omp_for_data (for_stmt
, &fd
, NULL
);
973 if (gimple_omp_for_combined_into_p (for_stmt
))
976 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
977 OMP_CLAUSE__LOOPTEMP_
);
979 n1
= OMP_CLAUSE_DECL (innerc
);
980 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
981 OMP_CLAUSE__LOOPTEMP_
);
983 n2
= OMP_CLAUSE_DECL (innerc
);
986 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
988 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
989 ws_args
->quick_push (t
);
991 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
992 ws_args
->quick_push (t
);
994 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
995 ws_args
->quick_push (t
);
999 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
1000 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
1001 ws_args
->quick_push (t
);
1006 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
1008 /* Number of sections is equal to the number of edges from the
1009 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
1010 the exit of the sections region. */
1011 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
1012 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
1013 vec_alloc (ws_args
, 1);
1014 ws_args
->quick_push (t
);
1022 /* Discover whether REGION is a combined parallel+workshare region. */
1025 determine_parallel_type (struct omp_region
*region
)
1027 basic_block par_entry_bb
, par_exit_bb
;
1028 basic_block ws_entry_bb
, ws_exit_bb
;
1030 if (region
== NULL
|| region
->inner
== NULL
1031 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
1032 || region
->inner
->cont
== NULL
)
1035 /* We only support parallel+for and parallel+sections. */
1036 if (region
->type
!= GIMPLE_OMP_PARALLEL
1037 || (region
->inner
->type
!= GIMPLE_OMP_FOR
1038 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
1041 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
1042 WS_EXIT_BB -> PAR_EXIT_BB. */
1043 par_entry_bb
= region
->entry
;
1044 par_exit_bb
= region
->exit
;
1045 ws_entry_bb
= region
->inner
->entry
;
1046 ws_exit_bb
= region
->inner
->exit
;
1048 if (single_succ (par_entry_bb
) == ws_entry_bb
1049 && single_succ (ws_exit_bb
) == par_exit_bb
1050 && workshare_safe_to_combine_p (ws_entry_bb
)
1051 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
1052 || (last_and_only_stmt (ws_entry_bb
)
1053 && last_and_only_stmt (par_exit_bb
))))
1055 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1056 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1058 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1060 /* If this is a combined parallel loop, we need to determine
1061 whether or not to use the combined library calls. There
1062 are two cases where we do not apply the transformation:
1063 static loops and any kind of ordered loop. In the first
1064 case, we already open code the loop so there is no need
1065 to do anything else. In the latter case, the combined
1066 parallel loop call would still need extra synchronization
1067 to implement ordered semantics, so there would not be any
1068 gain in using the combined call. */
1069 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1070 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1072 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
1073 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1075 region
->is_combined_parallel
= false;
1076 region
->inner
->is_combined_parallel
= false;
1081 region
->is_combined_parallel
= true;
1082 region
->inner
->is_combined_parallel
= true;
1083 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1088 /* Return true if EXPR is variable sized. */
1091 is_variable_sized (const_tree expr
)
1093 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1096 /* Return true if DECL is a reference type. */
1099 is_reference (tree decl
)
1101 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1104 /* Return the type of a decl. If the decl is reference type,
1105 return its base type. */
1107 get_base_type (tree decl
)
1109 tree type
= TREE_TYPE (decl
);
1110 if (is_reference (decl
))
1111 type
= TREE_TYPE (type
);
1115 /* Lookup variables. The "maybe" form
1116 allows for the variable form to not have been entered, otherwise we
1117 assert that the variable must have been entered. */
1120 lookup_decl (tree var
, omp_context
*ctx
)
1122 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1127 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1129 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1130 return n
? *n
: NULL_TREE
;
1134 lookup_field (tree var
, omp_context
*ctx
)
1137 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1138 return (tree
) n
->value
;
1142 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1145 n
= splay_tree_lookup (ctx
->sfield_map
1146 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1147 return (tree
) n
->value
;
1151 lookup_sfield (tree var
, omp_context
*ctx
)
1153 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1157 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1160 n
= splay_tree_lookup (ctx
->field_map
, key
);
1161 return n
? (tree
) n
->value
: NULL_TREE
;
1165 maybe_lookup_field (tree var
, omp_context
*ctx
)
1167 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1171 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
1174 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
1175 return (tree
) n
->value
;
1179 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1181 splay_tree_node n
= NULL
;
1182 if (ctx
->reduction_map
)
1183 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1184 return n
? (tree
) n
->value
: NULL_TREE
;
1187 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1188 the parallel context if DECL is to be shared. */
1191 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1193 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1196 /* We can only use copy-in/copy-out semantics for shared variables
1197 when we know the value is not accessible from an outer scope. */
1200 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1202 /* ??? Trivially accessible from anywhere. But why would we even
1203 be passing an address in this case? Should we simply assert
1204 this to be false, or should we have a cleanup pass that removes
1205 these from the list of mappings? */
1206 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1209 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1210 without analyzing the expression whether or not its location
1211 is accessible to anyone else. In the case of nested parallel
1212 regions it certainly may be. */
1213 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1216 /* Do not use copy-in/copy-out for variables that have their
1218 if (TREE_ADDRESSABLE (decl
))
1221 /* lower_send_shared_vars only uses copy-in, but not copy-out
1223 if (TREE_READONLY (decl
)
1224 || ((TREE_CODE (decl
) == RESULT_DECL
1225 || TREE_CODE (decl
) == PARM_DECL
)
1226 && DECL_BY_REFERENCE (decl
)))
1229 /* Disallow copy-in/out in nested parallel if
1230 decl is shared in outer parallel, otherwise
1231 each thread could store the shared variable
1232 in its own copy-in location, making the
1233 variable no longer really shared. */
1234 if (shared_ctx
->is_nested
)
1238 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1239 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1246 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1247 c
; c
= OMP_CLAUSE_CHAIN (c
))
1248 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1249 && OMP_CLAUSE_DECL (c
) == decl
)
1253 goto maybe_mark_addressable_and_ret
;
1257 /* For tasks avoid using copy-in/out. As tasks can be
1258 deferred or executed in different thread, when GOMP_task
1259 returns, the task hasn't necessarily terminated. */
1260 if (is_task_ctx (shared_ctx
))
1263 maybe_mark_addressable_and_ret
:
1264 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1265 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1267 /* Taking address of OUTER in lower_send_shared_vars
1268 might need regimplification of everything that uses the
1270 if (!task_shared_vars
)
1271 task_shared_vars
= BITMAP_ALLOC (NULL
);
1272 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1273 TREE_ADDRESSABLE (outer
) = 1;
1282 /* Construct a new automatic decl similar to VAR. */
1285 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1287 tree copy
= copy_var_decl (var
, name
, type
);
1289 DECL_CONTEXT (copy
) = current_function_decl
;
1290 DECL_CHAIN (copy
) = ctx
->block_vars
;
1291 /* If VAR is listed in task_shared_vars, it means it wasn't
1292 originally addressable and is just because task needs to take
1293 it's address. But we don't need to take address of privatizations
1295 if (TREE_ADDRESSABLE (var
)
1297 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1298 TREE_ADDRESSABLE (copy
) = 0;
1299 ctx
->block_vars
= copy
;
1305 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1307 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1310 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1313 omp_build_component_ref (tree obj
, tree field
)
1315 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1316 if (TREE_THIS_VOLATILE (field
))
1317 TREE_THIS_VOLATILE (ret
) |= 1;
1318 if (TREE_READONLY (field
))
1319 TREE_READONLY (ret
) |= 1;
1323 /* Build tree nodes to access the field for VAR on the receiver side. */
1326 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1328 tree x
, field
= lookup_field (var
, ctx
);
1330 /* If the receiver record type was remapped in the child function,
1331 remap the field into the new record type. */
1332 x
= maybe_lookup_field (field
, ctx
);
1336 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1337 TREE_THIS_NOTRAP (x
) = 1;
1338 x
= omp_build_component_ref (x
, field
);
1340 x
= build_simple_mem_ref (x
);
1345 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1346 of a parallel, this is a component reference; for workshare constructs
1347 this is some variable. */
1350 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1354 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1356 else if (is_variable_sized (var
))
1358 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1359 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1360 x
= build_simple_mem_ref (x
);
1362 else if (is_taskreg_ctx (ctx
))
1364 bool by_ref
= use_pointer_for_field (var
, NULL
);
1365 x
= build_receiver_ref (var
, by_ref
, ctx
);
1367 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1368 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1370 /* #pragma omp simd isn't a worksharing construct, and can reference even
1371 private vars in its linear etc. clauses. */
1373 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1374 x
= lookup_decl (var
, ctx
->outer
);
1375 else if (ctx
->outer
)
1376 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1380 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1382 gcc_assert (ctx
->outer
);
1384 = splay_tree_lookup (ctx
->outer
->field_map
,
1385 (splay_tree_key
) &DECL_UID (var
));
1388 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1391 x
= lookup_decl (var
, ctx
->outer
);
1395 tree field
= (tree
) n
->value
;
1396 /* If the receiver record type was remapped in the child function,
1397 remap the field into the new record type. */
1398 x
= maybe_lookup_field (field
, ctx
->outer
);
1402 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1403 x
= omp_build_component_ref (x
, field
);
1404 if (use_pointer_for_field (var
, ctx
->outer
))
1405 x
= build_simple_mem_ref (x
);
1408 else if (ctx
->outer
)
1409 x
= lookup_decl (var
, ctx
->outer
);
1410 else if (is_reference (var
))
1411 /* This can happen with orphaned constructs. If var is reference, it is
1412 possible it is shared and as such valid. */
1414 else if (omp_member_access_dummy_var (var
))
1421 tree t
= omp_member_access_dummy_var (var
);
1424 x
= DECL_VALUE_EXPR (var
);
1425 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1427 x
= unshare_and_remap (x
, t
, o
);
1429 x
= unshare_expr (x
);
1433 if (is_reference (var
))
1434 x
= build_simple_mem_ref (x
);
1439 /* Build tree nodes to access the field for VAR on the sender side. */
1442 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1444 tree field
= lookup_sfield (key
, ctx
);
1445 return omp_build_component_ref (ctx
->sender_decl
, field
);
1449 build_sender_ref (tree var
, omp_context
*ctx
)
1451 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1454 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1457 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1459 tree field
, type
, sfield
= NULL_TREE
;
1460 splay_tree_key key
= (splay_tree_key
) var
;
1462 if ((mask
& 8) != 0)
1464 key
= (splay_tree_key
) &DECL_UID (var
);
1465 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1467 gcc_assert ((mask
& 1) == 0
1468 || !splay_tree_lookup (ctx
->field_map
, key
));
1469 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1470 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1471 gcc_assert ((mask
& 3) == 3
1472 || !is_gimple_omp_oacc (ctx
->stmt
));
1474 type
= TREE_TYPE (var
);
1477 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1478 type
= build_pointer_type (build_pointer_type (type
));
1481 type
= build_pointer_type (type
);
1482 else if ((mask
& 3) == 1 && is_reference (var
))
1483 type
= TREE_TYPE (type
);
1485 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1486 FIELD_DECL
, DECL_NAME (var
), type
);
1488 /* Remember what variable this field was created for. This does have a
1489 side effect of making dwarf2out ignore this member, so for helpful
1490 debugging we clear it later in delete_omp_context. */
1491 DECL_ABSTRACT_ORIGIN (field
) = var
;
1492 if (type
== TREE_TYPE (var
))
1494 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1495 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1496 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1499 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1501 if ((mask
& 3) == 3)
1503 insert_field_into_struct (ctx
->record_type
, field
);
1504 if (ctx
->srecord_type
)
1506 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1507 FIELD_DECL
, DECL_NAME (var
), type
);
1508 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1509 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1510 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1511 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1512 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1517 if (ctx
->srecord_type
== NULL_TREE
)
1521 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1522 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1523 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1525 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1526 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1527 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1528 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1529 splay_tree_insert (ctx
->sfield_map
,
1530 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1531 (splay_tree_value
) sfield
);
1535 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1536 : ctx
->srecord_type
, field
);
1540 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1541 if ((mask
& 2) && ctx
->sfield_map
)
1542 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1546 install_var_local (tree var
, omp_context
*ctx
)
1548 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1549 insert_decl_map (&ctx
->cb
, var
, new_var
);
1553 /* Adjust the replacement for DECL in CTX for the new context. This means
1554 copying the DECL_VALUE_EXPR, and fixing up the type. */
1557 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1559 tree new_decl
, size
;
1561 new_decl
= lookup_decl (decl
, ctx
);
1563 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1565 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1566 && DECL_HAS_VALUE_EXPR_P (decl
))
1568 tree ve
= DECL_VALUE_EXPR (decl
);
1569 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1570 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1571 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1574 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1576 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1577 if (size
== error_mark_node
)
1578 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1579 DECL_SIZE (new_decl
) = size
;
1581 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1582 if (size
== error_mark_node
)
1583 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1584 DECL_SIZE_UNIT (new_decl
) = size
;
1588 /* The callback for remap_decl. Search all containing contexts for a
1589 mapping of the variable; this avoids having to duplicate the splay
1590 tree ahead of time. We know a mapping doesn't already exist in the
1591 given context. Create new mappings to implement default semantics. */
1594 omp_copy_decl (tree var
, copy_body_data
*cb
)
1596 omp_context
*ctx
= (omp_context
*) cb
;
1599 if (TREE_CODE (var
) == LABEL_DECL
)
1601 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1602 DECL_CONTEXT (new_var
) = current_function_decl
;
1603 insert_decl_map (&ctx
->cb
, var
, new_var
);
1607 while (!is_taskreg_ctx (ctx
))
1612 new_var
= maybe_lookup_decl (var
, ctx
);
1617 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1620 return error_mark_node
;
1624 /* Debugging dumps for parallel regions. */
1625 void dump_omp_region (FILE *, struct omp_region
*, int);
1626 void debug_omp_region (struct omp_region
*);
1627 void debug_all_omp_regions (void);
1629 /* Dump the parallel region tree rooted at REGION. */
1632 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1634 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1635 gimple_code_name
[region
->type
]);
1638 dump_omp_region (file
, region
->inner
, indent
+ 4);
1642 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1643 region
->cont
->index
);
1647 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1648 region
->exit
->index
);
1650 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1653 dump_omp_region (file
, region
->next
, indent
);
1657 debug_omp_region (struct omp_region
*region
)
1659 dump_omp_region (stderr
, region
, 0);
1663 debug_all_omp_regions (void)
1665 dump_omp_region (stderr
, root_omp_region
, 0);
1669 /* Create a new parallel region starting at STMT inside region PARENT. */
1671 static struct omp_region
*
1672 new_omp_region (basic_block bb
, enum gimple_code type
,
1673 struct omp_region
*parent
)
1675 struct omp_region
*region
= XCNEW (struct omp_region
);
1677 region
->outer
= parent
;
1679 region
->type
= type
;
1683 /* This is a nested region. Add it to the list of inner
1684 regions in PARENT. */
1685 region
->next
= parent
->inner
;
1686 parent
->inner
= region
;
1690 /* This is a toplevel region. Add it to the list of toplevel
1691 regions in ROOT_OMP_REGION. */
1692 region
->next
= root_omp_region
;
1693 root_omp_region
= region
;
1699 /* Release the memory associated with the region tree rooted at REGION. */
1702 free_omp_region_1 (struct omp_region
*region
)
1704 struct omp_region
*i
, *n
;
1706 for (i
= region
->inner
; i
; i
= n
)
1709 free_omp_region_1 (i
);
1715 /* Release the memory for the entire omp region tree. */
1718 free_omp_regions (void)
1720 struct omp_region
*r
, *n
;
1721 for (r
= root_omp_region
; r
; r
= n
)
1724 free_omp_region_1 (r
);
1726 root_omp_region
= NULL
;
1730 /* Create a new context, with OUTER_CTX being the surrounding context. */
1732 static omp_context
*
1733 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1735 omp_context
*ctx
= XCNEW (omp_context
);
1737 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1738 (splay_tree_value
) ctx
);
1743 ctx
->outer
= outer_ctx
;
1744 ctx
->cb
= outer_ctx
->cb
;
1745 ctx
->cb
.block
= NULL
;
1746 ctx
->depth
= outer_ctx
->depth
+ 1;
1747 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1751 ctx
->cb
.src_fn
= current_function_decl
;
1752 ctx
->cb
.dst_fn
= current_function_decl
;
1753 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1754 gcc_checking_assert (ctx
->cb
.src_node
);
1755 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1756 ctx
->cb
.src_cfun
= cfun
;
1757 ctx
->cb
.copy_decl
= omp_copy_decl
;
1758 ctx
->cb
.eh_lp_nr
= 0;
1759 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1763 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1768 static gimple_seq
maybe_catch_exception (gimple_seq
);
1770 /* Finalize task copyfn. */
1773 finalize_task_copyfn (gomp_task
*task_stmt
)
1775 struct function
*child_cfun
;
1777 gimple_seq seq
= NULL
, new_seq
;
1780 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1781 if (child_fn
== NULL_TREE
)
1784 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1785 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1787 push_cfun (child_cfun
);
1788 bind
= gimplify_body (child_fn
, false);
1789 gimple_seq_add_stmt (&seq
, bind
);
1790 new_seq
= maybe_catch_exception (seq
);
1793 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1795 gimple_seq_add_stmt (&seq
, bind
);
1797 gimple_set_body (child_fn
, seq
);
1800 /* Inform the callgraph about the new function. */
1801 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1802 node
->parallelized_function
= 1;
1803 cgraph_node::add_new_function (child_fn
, false);
1806 /* Destroy a omp_context data structures. Called through the splay tree
1807 value delete callback. */
1810 delete_omp_context (splay_tree_value value
)
1812 omp_context
*ctx
= (omp_context
*) value
;
1814 delete ctx
->cb
.decl_map
;
1817 splay_tree_delete (ctx
->field_map
);
1818 if (ctx
->sfield_map
)
1819 splay_tree_delete (ctx
->sfield_map
);
1820 /* Reduction map is copied to nested contexts, so only delete it in the
1822 if (ctx
->reduction_map
1823 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1824 && is_gimple_omp_offloaded (ctx
->stmt
)
1825 && is_gimple_omp_oacc (ctx
->stmt
))
1826 splay_tree_delete (ctx
->reduction_map
);
1828 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1829 it produces corrupt debug information. */
1830 if (ctx
->record_type
)
1833 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1834 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1836 if (ctx
->srecord_type
)
1839 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1840 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1843 if (is_task_ctx (ctx
))
1844 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1849 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1853 fixup_child_record_type (omp_context
*ctx
)
1855 tree f
, type
= ctx
->record_type
;
1857 /* ??? It isn't sufficient to just call remap_type here, because
1858 variably_modified_type_p doesn't work the way we expect for
1859 record types. Testing each field for whether it needs remapping
1860 and creating a new record by hand works, however. */
1861 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1862 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1866 tree name
, new_fields
= NULL
;
1868 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1869 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1870 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1871 TYPE_DECL
, name
, type
);
1872 TYPE_NAME (type
) = name
;
1874 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1876 tree new_f
= copy_node (f
);
1877 DECL_CONTEXT (new_f
) = type
;
1878 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1879 DECL_CHAIN (new_f
) = new_fields
;
1880 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1881 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1883 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1887 /* Arrange to be able to look up the receiver field
1888 given the sender field. */
1889 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1890 (splay_tree_value
) new_f
);
1892 TYPE_FIELDS (type
) = nreverse (new_fields
);
1896 /* In a target region we never modify any of the pointers in *.omp_data_i,
1897 so attempt to help the optimizers. */
1898 if (is_gimple_omp_offloaded (ctx
->stmt
))
1899 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1901 TREE_TYPE (ctx
->receiver_decl
)
1902 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1905 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1906 specified by CLAUSES. */
1909 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1912 bool scan_array_reductions
= false;
1914 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1918 switch (OMP_CLAUSE_CODE (c
))
1920 case OMP_CLAUSE_PRIVATE
:
1921 decl
= OMP_CLAUSE_DECL (c
);
1922 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1924 else if (!is_variable_sized (decl
))
1925 install_var_local (decl
, ctx
);
1928 case OMP_CLAUSE_SHARED
:
1929 decl
= OMP_CLAUSE_DECL (c
);
1930 /* Ignore shared directives in teams construct. */
1931 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1933 /* Global variables don't need to be copied,
1934 the receiver side will use them directly. */
1935 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1936 if (is_global_var (odecl
))
1938 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1941 gcc_assert (is_taskreg_ctx (ctx
));
1942 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1943 || !is_variable_sized (decl
));
1944 /* Global variables don't need to be copied,
1945 the receiver side will use them directly. */
1946 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1948 by_ref
= use_pointer_for_field (decl
, ctx
);
1949 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1951 if (! TREE_READONLY (decl
)
1952 || TREE_ADDRESSABLE (decl
)
1954 || is_reference (decl
))
1956 install_var_field (decl
, by_ref
, 3, ctx
);
1957 install_var_local (decl
, ctx
);
1960 /* We don't need to copy const scalar vars back. */
1961 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1964 case OMP_CLAUSE_REDUCTION
:
1965 decl
= OMP_CLAUSE_DECL (c
);
1966 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1967 && TREE_CODE (decl
) == MEM_REF
)
1969 tree t
= TREE_OPERAND (decl
, 0);
1970 if (TREE_CODE (t
) == INDIRECT_REF
1971 || TREE_CODE (t
) == ADDR_EXPR
)
1972 t
= TREE_OPERAND (t
, 0);
1973 install_var_local (t
, ctx
);
1974 if (is_taskreg_ctx (ctx
)
1975 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1976 && !is_variable_sized (t
))
1978 by_ref
= use_pointer_for_field (t
, ctx
);
1979 install_var_field (t
, by_ref
, 3, ctx
);
1985 case OMP_CLAUSE_LASTPRIVATE
:
1986 /* Let the corresponding firstprivate clause create
1988 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1992 case OMP_CLAUSE_FIRSTPRIVATE
:
1993 if (is_gimple_omp_oacc (ctx
->stmt
))
1995 sorry ("clause not supported yet");
1999 case OMP_CLAUSE_LINEAR
:
2000 decl
= OMP_CLAUSE_DECL (c
);
2002 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2003 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2004 && is_gimple_omp_offloaded (ctx
->stmt
))
2006 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
2007 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
2008 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2009 install_var_field (decl
, true, 3, ctx
);
2011 install_var_field (decl
, false, 3, ctx
);
2013 if (is_variable_sized (decl
))
2015 if (is_task_ctx (ctx
))
2016 install_var_field (decl
, false, 1, ctx
);
2019 else if (is_taskreg_ctx (ctx
))
2022 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
2023 by_ref
= use_pointer_for_field (decl
, NULL
);
2025 if (is_task_ctx (ctx
)
2026 && (global
|| by_ref
|| is_reference (decl
)))
2028 install_var_field (decl
, false, 1, ctx
);
2030 install_var_field (decl
, by_ref
, 2, ctx
);
2033 install_var_field (decl
, by_ref
, 3, ctx
);
2035 install_var_local (decl
, ctx
);
2036 if (is_gimple_omp_oacc (ctx
->stmt
)
2037 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
2039 /* Create a decl for the reduction array. */
2040 tree var
= OMP_CLAUSE_DECL (c
);
2041 tree type
= get_base_type (var
);
2042 tree ptype
= build_pointer_type (type
);
2043 tree array
= create_tmp_var (ptype
,
2044 oacc_get_reduction_array_id (var
));
2045 omp_context
*octx
= (ctx
->field_map
? ctx
: ctx
->outer
);
2046 install_var_field (array
, true, 3, octx
);
2047 install_var_local (array
, octx
);
2049 /* Insert it into the current context. */
2050 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
2051 oacc_get_reduction_array_id (var
),
2052 (splay_tree_value
) array
);
2053 splay_tree_insert (ctx
->reduction_map
,
2054 (splay_tree_key
) array
,
2055 (splay_tree_value
) array
);
2059 case OMP_CLAUSE_USE_DEVICE_PTR
:
2060 decl
= OMP_CLAUSE_DECL (c
);
2061 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2062 install_var_field (decl
, true, 3, ctx
);
2064 install_var_field (decl
, false, 3, ctx
);
2065 if (DECL_SIZE (decl
)
2066 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2068 tree decl2
= DECL_VALUE_EXPR (decl
);
2069 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2070 decl2
= TREE_OPERAND (decl2
, 0);
2071 gcc_assert (DECL_P (decl2
));
2072 install_var_local (decl2
, ctx
);
2074 install_var_local (decl
, ctx
);
2077 case OMP_CLAUSE_IS_DEVICE_PTR
:
2078 decl
= OMP_CLAUSE_DECL (c
);
2081 case OMP_CLAUSE__LOOPTEMP_
:
2082 gcc_assert (is_taskreg_ctx (ctx
));
2083 decl
= OMP_CLAUSE_DECL (c
);
2084 install_var_field (decl
, false, 3, ctx
);
2085 install_var_local (decl
, ctx
);
2088 case OMP_CLAUSE_COPYPRIVATE
:
2089 case OMP_CLAUSE_COPYIN
:
2090 decl
= OMP_CLAUSE_DECL (c
);
2091 by_ref
= use_pointer_for_field (decl
, NULL
);
2092 install_var_field (decl
, by_ref
, 3, ctx
);
2095 case OMP_CLAUSE_DEFAULT
:
2096 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2099 case OMP_CLAUSE_FINAL
:
2101 case OMP_CLAUSE_NUM_THREADS
:
2102 case OMP_CLAUSE_NUM_TEAMS
:
2103 case OMP_CLAUSE_THREAD_LIMIT
:
2104 case OMP_CLAUSE_DEVICE
:
2105 case OMP_CLAUSE_SCHEDULE
:
2106 case OMP_CLAUSE_DIST_SCHEDULE
:
2107 case OMP_CLAUSE_DEPEND
:
2108 case OMP_CLAUSE_PRIORITY
:
2109 case OMP_CLAUSE_GRAINSIZE
:
2110 case OMP_CLAUSE_NUM_TASKS
:
2111 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2112 case OMP_CLAUSE_NUM_GANGS
:
2113 case OMP_CLAUSE_NUM_WORKERS
:
2114 case OMP_CLAUSE_VECTOR_LENGTH
:
2116 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2120 case OMP_CLAUSE_FROM
:
2121 case OMP_CLAUSE_MAP
:
2123 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2124 decl
= OMP_CLAUSE_DECL (c
);
2125 /* Global variables with "omp declare target" attribute
2126 don't need to be copied, the receiver side will use them
2128 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2130 && (OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2131 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2132 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2133 && varpool_node::get_create (decl
)->offloadable
)
2135 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2136 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2138 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2139 not offloaded; there is nothing to map for those. */
2140 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2141 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2142 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2145 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2146 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2148 if (TREE_CODE (decl
) == COMPONENT_REF
2149 || (TREE_CODE (decl
) == INDIRECT_REF
2150 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2151 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2152 == REFERENCE_TYPE
)))
2154 if (DECL_SIZE (decl
)
2155 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2157 tree decl2
= DECL_VALUE_EXPR (decl
);
2158 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2159 decl2
= TREE_OPERAND (decl2
, 0);
2160 gcc_assert (DECL_P (decl2
));
2161 install_var_local (decl2
, ctx
);
2163 install_var_local (decl
, ctx
);
2168 if (DECL_SIZE (decl
)
2169 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2171 tree decl2
= DECL_VALUE_EXPR (decl
);
2172 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2173 decl2
= TREE_OPERAND (decl2
, 0);
2174 gcc_assert (DECL_P (decl2
));
2175 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2176 && OMP_CLAUSE_MAP_PRIVATE (c
))
2177 install_var_field (decl2
, true, 11, ctx
);
2179 install_var_field (decl2
, true, 3, ctx
);
2180 install_var_local (decl2
, ctx
);
2181 install_var_local (decl
, ctx
);
2185 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2186 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2187 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2188 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2189 install_var_field (decl
, true, 7, ctx
);
2190 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2191 && OMP_CLAUSE_MAP_PRIVATE (c
))
2192 install_var_field (decl
, true, 11, ctx
);
2194 install_var_field (decl
, true, 3, ctx
);
2195 if (is_gimple_omp_offloaded (ctx
->stmt
))
2196 install_var_local (decl
, ctx
);
2201 tree base
= get_base_address (decl
);
2202 tree nc
= OMP_CLAUSE_CHAIN (c
);
2205 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2206 && OMP_CLAUSE_DECL (nc
) == base
2207 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2208 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2210 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2211 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2217 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2218 decl
= OMP_CLAUSE_DECL (c
);
2220 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2221 (splay_tree_key
) decl
));
2223 = build_decl (OMP_CLAUSE_LOCATION (c
),
2224 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2225 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2226 insert_field_into_struct (ctx
->record_type
, field
);
2227 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2228 (splay_tree_value
) field
);
2233 case OMP_CLAUSE_NOWAIT
:
2234 case OMP_CLAUSE_ORDERED
:
2235 case OMP_CLAUSE_COLLAPSE
:
2236 case OMP_CLAUSE_UNTIED
:
2237 case OMP_CLAUSE_MERGEABLE
:
2238 case OMP_CLAUSE_PROC_BIND
:
2239 case OMP_CLAUSE_SAFELEN
:
2240 case OMP_CLAUSE_SIMDLEN
:
2241 case OMP_CLAUSE_THREADS
:
2242 case OMP_CLAUSE_SIMD
:
2243 case OMP_CLAUSE_NOGROUP
:
2244 case OMP_CLAUSE_DEFAULTMAP
:
2245 case OMP_CLAUSE_ASYNC
:
2246 case OMP_CLAUSE_WAIT
:
2247 case OMP_CLAUSE_GANG
:
2248 case OMP_CLAUSE_WORKER
:
2249 case OMP_CLAUSE_VECTOR
:
2252 case OMP_CLAUSE_ALIGNED
:
2253 decl
= OMP_CLAUSE_DECL (c
);
2254 if (is_global_var (decl
)
2255 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2256 install_var_local (decl
, ctx
);
2259 case OMP_CLAUSE_DEVICE_RESIDENT
:
2260 case OMP_CLAUSE_USE_DEVICE
:
2261 case OMP_CLAUSE__CACHE_
:
2262 case OMP_CLAUSE_INDEPENDENT
:
2263 case OMP_CLAUSE_AUTO
:
2264 case OMP_CLAUSE_SEQ
:
2265 sorry ("Clause not supported yet");
2273 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2275 switch (OMP_CLAUSE_CODE (c
))
2277 case OMP_CLAUSE_LASTPRIVATE
:
2278 /* Let the corresponding firstprivate clause create
2280 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2281 scan_array_reductions
= true;
2282 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2286 case OMP_CLAUSE_FIRSTPRIVATE
:
2287 if (is_gimple_omp_oacc (ctx
->stmt
))
2289 sorry ("clause not supported yet");
2293 case OMP_CLAUSE_PRIVATE
:
2294 case OMP_CLAUSE_LINEAR
:
2295 case OMP_CLAUSE_IS_DEVICE_PTR
:
2296 decl
= OMP_CLAUSE_DECL (c
);
2297 if (is_variable_sized (decl
))
2299 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2300 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2301 && is_gimple_omp_offloaded (ctx
->stmt
))
2303 tree decl2
= DECL_VALUE_EXPR (decl
);
2304 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2305 decl2
= TREE_OPERAND (decl2
, 0);
2306 gcc_assert (DECL_P (decl2
));
2307 install_var_local (decl2
, ctx
);
2308 fixup_remapped_decl (decl2
, ctx
, false);
2310 install_var_local (decl
, ctx
);
2312 fixup_remapped_decl (decl
, ctx
,
2313 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2314 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2315 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2316 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2317 scan_array_reductions
= true;
2320 case OMP_CLAUSE_REDUCTION
:
2321 decl
= OMP_CLAUSE_DECL (c
);
2322 if (TREE_CODE (decl
) != MEM_REF
)
2324 if (is_variable_sized (decl
))
2325 install_var_local (decl
, ctx
);
2326 fixup_remapped_decl (decl
, ctx
, false);
2328 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2329 scan_array_reductions
= true;
2332 case OMP_CLAUSE_SHARED
:
2333 /* Ignore shared directives in teams construct. */
2334 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2336 decl
= OMP_CLAUSE_DECL (c
);
2337 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2339 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2341 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2344 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2345 install_var_field (decl
, by_ref
, 11, ctx
);
2348 fixup_remapped_decl (decl
, ctx
, false);
2351 case OMP_CLAUSE_MAP
:
2352 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2354 decl
= OMP_CLAUSE_DECL (c
);
2356 && (OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2357 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2358 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2359 && varpool_node::get_create (decl
)->offloadable
)
2363 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2364 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2365 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2366 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2368 tree new_decl
= lookup_decl (decl
, ctx
);
2369 TREE_TYPE (new_decl
)
2370 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2372 else if (DECL_SIZE (decl
)
2373 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2375 tree decl2
= DECL_VALUE_EXPR (decl
);
2376 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2377 decl2
= TREE_OPERAND (decl2
, 0);
2378 gcc_assert (DECL_P (decl2
));
2379 fixup_remapped_decl (decl2
, ctx
, false);
2380 fixup_remapped_decl (decl
, ctx
, true);
2383 fixup_remapped_decl (decl
, ctx
, false);
2387 case OMP_CLAUSE_COPYPRIVATE
:
2388 case OMP_CLAUSE_COPYIN
:
2389 case OMP_CLAUSE_DEFAULT
:
2391 case OMP_CLAUSE_NUM_THREADS
:
2392 case OMP_CLAUSE_NUM_TEAMS
:
2393 case OMP_CLAUSE_THREAD_LIMIT
:
2394 case OMP_CLAUSE_DEVICE
:
2395 case OMP_CLAUSE_SCHEDULE
:
2396 case OMP_CLAUSE_DIST_SCHEDULE
:
2397 case OMP_CLAUSE_NOWAIT
:
2398 case OMP_CLAUSE_ORDERED
:
2399 case OMP_CLAUSE_COLLAPSE
:
2400 case OMP_CLAUSE_UNTIED
:
2401 case OMP_CLAUSE_FINAL
:
2402 case OMP_CLAUSE_MERGEABLE
:
2403 case OMP_CLAUSE_PROC_BIND
:
2404 case OMP_CLAUSE_SAFELEN
:
2405 case OMP_CLAUSE_SIMDLEN
:
2406 case OMP_CLAUSE_ALIGNED
:
2407 case OMP_CLAUSE_DEPEND
:
2408 case OMP_CLAUSE__LOOPTEMP_
:
2410 case OMP_CLAUSE_FROM
:
2411 case OMP_CLAUSE_PRIORITY
:
2412 case OMP_CLAUSE_GRAINSIZE
:
2413 case OMP_CLAUSE_NUM_TASKS
:
2414 case OMP_CLAUSE_THREADS
:
2415 case OMP_CLAUSE_SIMD
:
2416 case OMP_CLAUSE_NOGROUP
:
2417 case OMP_CLAUSE_DEFAULTMAP
:
2418 case OMP_CLAUSE_USE_DEVICE_PTR
:
2419 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2420 case OMP_CLAUSE_ASYNC
:
2421 case OMP_CLAUSE_WAIT
:
2422 case OMP_CLAUSE_NUM_GANGS
:
2423 case OMP_CLAUSE_NUM_WORKERS
:
2424 case OMP_CLAUSE_VECTOR_LENGTH
:
2425 case OMP_CLAUSE_GANG
:
2426 case OMP_CLAUSE_WORKER
:
2427 case OMP_CLAUSE_VECTOR
:
2430 case OMP_CLAUSE_DEVICE_RESIDENT
:
2431 case OMP_CLAUSE_USE_DEVICE
:
2432 case OMP_CLAUSE__CACHE_
:
2433 case OMP_CLAUSE_INDEPENDENT
:
2434 case OMP_CLAUSE_AUTO
:
2435 case OMP_CLAUSE_SEQ
:
2436 sorry ("Clause not supported yet");
2444 gcc_checking_assert (!scan_array_reductions
2445 || !is_gimple_omp_oacc (ctx
->stmt
));
2446 if (scan_array_reductions
)
2447 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2448 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2449 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2451 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2452 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2454 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2455 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2456 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2457 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2458 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2459 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2462 /* Create a new name for omp child function. Returns an identifier. If
2463 IS_CILK_FOR is true then the suffix for the child function is
2467 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2470 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2471 return clone_function_name (current_function_decl
,
2472 task_copy
? "_omp_cpyfn" : "_omp_fn");
2475 /* Returns the type of the induction variable for the child function for
2476 _Cilk_for and the types for _high and _low variables based on TYPE. */
2479 cilk_for_check_loop_diff_type (tree type
)
2481 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2483 if (TYPE_UNSIGNED (type
))
2484 return uint32_type_node
;
2486 return integer_type_node
;
2490 if (TYPE_UNSIGNED (type
))
2491 return uint64_type_node
;
2493 return long_long_integer_type_node
;
2497 /* Build a decl for the omp child function. It'll not contain a body
2498 yet, just the bare decl. */
2501 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2503 tree decl
, type
, name
, t
;
2506 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2507 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2508 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2509 tree cilk_var_type
= NULL_TREE
;
2511 name
= create_omp_child_function_name (task_copy
,
2512 cilk_for_count
!= NULL_TREE
);
2514 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2515 ptr_type_node
, NULL_TREE
);
2516 else if (cilk_for_count
)
2518 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2519 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2520 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2521 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2524 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2526 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2528 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2531 ctx
->cb
.dst_fn
= decl
;
2533 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2535 TREE_STATIC (decl
) = 1;
2536 TREE_USED (decl
) = 1;
2537 DECL_ARTIFICIAL (decl
) = 1;
2538 DECL_IGNORED_P (decl
) = 0;
2539 TREE_PUBLIC (decl
) = 0;
2540 DECL_UNINLINABLE (decl
) = 1;
2541 DECL_EXTERNAL (decl
) = 0;
2542 DECL_CONTEXT (decl
) = NULL_TREE
;
2543 DECL_INITIAL (decl
) = make_node (BLOCK
);
2544 if (cgraph_node::get (current_function_decl
)->offloadable
)
2545 cgraph_node::get_create (decl
)->offloadable
= 1;
2549 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2550 if (is_gimple_omp_offloaded (octx
->stmt
))
2552 cgraph_node::get_create (decl
)->offloadable
= 1;
2553 #ifdef ENABLE_OFFLOADING
2554 g
->have_offload
= true;
2560 if (cgraph_node::get_create (decl
)->offloadable
2561 && !lookup_attribute ("omp declare target",
2562 DECL_ATTRIBUTES (current_function_decl
)))
2563 DECL_ATTRIBUTES (decl
)
2564 = tree_cons (get_identifier ("omp target entrypoint"),
2565 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2567 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2568 RESULT_DECL
, NULL_TREE
, void_type_node
);
2569 DECL_ARTIFICIAL (t
) = 1;
2570 DECL_IGNORED_P (t
) = 1;
2571 DECL_CONTEXT (t
) = decl
;
2572 DECL_RESULT (decl
) = t
;
2574 /* _Cilk_for's child function requires two extra parameters called
2575 __low and __high that are set the by Cilk runtime when it calls this
2579 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2580 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2581 DECL_ARTIFICIAL (t
) = 1;
2582 DECL_NAMELESS (t
) = 1;
2583 DECL_ARG_TYPE (t
) = ptr_type_node
;
2584 DECL_CONTEXT (t
) = current_function_decl
;
2586 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2587 DECL_ARGUMENTS (decl
) = t
;
2589 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2590 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2591 DECL_ARTIFICIAL (t
) = 1;
2592 DECL_NAMELESS (t
) = 1;
2593 DECL_ARG_TYPE (t
) = ptr_type_node
;
2594 DECL_CONTEXT (t
) = current_function_decl
;
2596 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2597 DECL_ARGUMENTS (decl
) = t
;
2600 tree data_name
= get_identifier (".omp_data_i");
2601 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2603 DECL_ARTIFICIAL (t
) = 1;
2604 DECL_NAMELESS (t
) = 1;
2605 DECL_ARG_TYPE (t
) = ptr_type_node
;
2606 DECL_CONTEXT (t
) = current_function_decl
;
2608 TREE_READONLY (t
) = 1;
2610 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2611 DECL_ARGUMENTS (decl
) = t
;
2613 ctx
->receiver_decl
= t
;
2616 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2617 PARM_DECL
, get_identifier (".omp_data_o"),
2619 DECL_ARTIFICIAL (t
) = 1;
2620 DECL_NAMELESS (t
) = 1;
2621 DECL_ARG_TYPE (t
) = ptr_type_node
;
2622 DECL_CONTEXT (t
) = current_function_decl
;
2624 TREE_ADDRESSABLE (t
) = 1;
2625 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2626 DECL_ARGUMENTS (decl
) = t
;
2629 /* Allocate memory for the function structure. The call to
2630 allocate_struct_function clobbers CFUN, so we need to restore
2632 push_struct_function (decl
);
2633 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2637 /* Callback for walk_gimple_seq. Check if combined parallel
2638 contains gimple_omp_for_combined_into_p OMP_FOR. */
2641 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2642 bool *handled_ops_p
,
2643 struct walk_stmt_info
*wi
)
2645 gimple
*stmt
= gsi_stmt (*gsi_p
);
2647 *handled_ops_p
= true;
2648 switch (gimple_code (stmt
))
2652 case GIMPLE_OMP_FOR
:
2653 if (gimple_omp_for_combined_into_p (stmt
)
2654 && gimple_omp_for_kind (stmt
)
2655 == *(const enum gf_mask
*) (wi
->info
))
2658 return integer_zero_node
;
2667 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2670 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2671 omp_context
*outer_ctx
)
2673 struct walk_stmt_info wi
;
2675 memset (&wi
, 0, sizeof (wi
));
2677 wi
.info
= (void *) &msk
;
2678 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2679 if (wi
.info
!= (void *) &msk
)
2681 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2682 struct omp_for_data fd
;
2683 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2684 /* We need two temporaries with fd.loop.v type (istart/iend)
2685 and then (fd.collapse - 1) temporaries with the same
2686 type for count2 ... countN-1 vars if not constant. */
2687 size_t count
= 2, i
;
2688 tree type
= fd
.iter_type
;
2690 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2692 count
+= fd
.collapse
- 1;
2693 /* For taskloop, if there are lastprivate clauses on the inner
2694 GIMPLE_OMP_FOR, add one more temporaries for the total number
2695 of iterations (product of count1 ... countN-1). */
2696 if (msk
== GF_OMP_FOR_KIND_TASKLOOP
2697 && find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2698 OMP_CLAUSE_LASTPRIVATE
))
2701 for (i
= 0; i
< count
; i
++)
2703 tree temp
= create_tmp_var (type
);
2704 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2705 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2706 OMP_CLAUSE_DECL (c
) = temp
;
2707 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2708 gimple_omp_taskreg_set_clauses (stmt
, c
);
2713 /* Scan an OpenMP parallel directive. */
2716 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2720 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2722 /* Ignore parallel directives with empty bodies, unless there
2723 are copyin clauses. */
2725 && empty_body_p (gimple_omp_body (stmt
))
2726 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2727 OMP_CLAUSE_COPYIN
) == NULL
)
2729 gsi_replace (gsi
, gimple_build_nop (), false);
2733 if (gimple_omp_parallel_combined_p (stmt
))
2734 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2736 ctx
= new_omp_context (stmt
, outer_ctx
);
2737 taskreg_contexts
.safe_push (ctx
);
2738 if (taskreg_nesting_level
> 1)
2739 ctx
->is_nested
= true;
2740 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2741 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2742 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2743 name
= create_tmp_var_name (".omp_data_s");
2744 name
= build_decl (gimple_location (stmt
),
2745 TYPE_DECL
, name
, ctx
->record_type
);
2746 DECL_ARTIFICIAL (name
) = 1;
2747 DECL_NAMELESS (name
) = 1;
2748 TYPE_NAME (ctx
->record_type
) = name
;
2749 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2750 create_omp_child_function (ctx
, false);
2751 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2753 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2754 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2756 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2757 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2760 /* Scan an OpenMP task directive. */
2763 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2767 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2769 /* Ignore task directives with empty bodies. */
2771 && empty_body_p (gimple_omp_body (stmt
)))
2773 gsi_replace (gsi
, gimple_build_nop (), false);
2777 if (gimple_omp_task_taskloop_p (stmt
))
2778 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2780 ctx
= new_omp_context (stmt
, outer_ctx
);
2781 taskreg_contexts
.safe_push (ctx
);
2782 if (taskreg_nesting_level
> 1)
2783 ctx
->is_nested
= true;
2784 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2785 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2786 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2787 name
= create_tmp_var_name (".omp_data_s");
2788 name
= build_decl (gimple_location (stmt
),
2789 TYPE_DECL
, name
, ctx
->record_type
);
2790 DECL_ARTIFICIAL (name
) = 1;
2791 DECL_NAMELESS (name
) = 1;
2792 TYPE_NAME (ctx
->record_type
) = name
;
2793 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2794 create_omp_child_function (ctx
, false);
2795 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2797 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2799 if (ctx
->srecord_type
)
2801 name
= create_tmp_var_name (".omp_data_a");
2802 name
= build_decl (gimple_location (stmt
),
2803 TYPE_DECL
, name
, ctx
->srecord_type
);
2804 DECL_ARTIFICIAL (name
) = 1;
2805 DECL_NAMELESS (name
) = 1;
2806 TYPE_NAME (ctx
->srecord_type
) = name
;
2807 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2808 create_omp_child_function (ctx
, true);
2811 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2813 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2815 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2816 t
= build_int_cst (long_integer_type_node
, 0);
2817 gimple_omp_task_set_arg_size (stmt
, t
);
2818 t
= build_int_cst (long_integer_type_node
, 1);
2819 gimple_omp_task_set_arg_align (stmt
, t
);
2824 /* If any decls have been made addressable during scan_omp,
2825 adjust their fields if needed, and layout record types
2826 of parallel/task constructs. */
2829 finish_taskreg_scan (omp_context
*ctx
)
2831 if (ctx
->record_type
== NULL_TREE
)
2834 /* If any task_shared_vars were needed, verify all
2835 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2836 statements if use_pointer_for_field hasn't changed
2837 because of that. If it did, update field types now. */
2838 if (task_shared_vars
)
2842 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2843 c
; c
= OMP_CLAUSE_CHAIN (c
))
2844 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2845 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2847 tree decl
= OMP_CLAUSE_DECL (c
);
2849 /* Global variables don't need to be copied,
2850 the receiver side will use them directly. */
2851 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2853 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2854 || !use_pointer_for_field (decl
, ctx
))
2856 tree field
= lookup_field (decl
, ctx
);
2857 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2858 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2860 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2861 TREE_THIS_VOLATILE (field
) = 0;
2862 DECL_USER_ALIGN (field
) = 0;
2863 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2864 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2865 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2866 if (ctx
->srecord_type
)
2868 tree sfield
= lookup_sfield (decl
, ctx
);
2869 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2870 TREE_THIS_VOLATILE (sfield
) = 0;
2871 DECL_USER_ALIGN (sfield
) = 0;
2872 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2873 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2874 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2879 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2881 layout_type (ctx
->record_type
);
2882 fixup_child_record_type (ctx
);
2886 location_t loc
= gimple_location (ctx
->stmt
);
2887 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2888 /* Move VLA fields to the end. */
2889 p
= &TYPE_FIELDS (ctx
->record_type
);
2891 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2892 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2895 *p
= TREE_CHAIN (*p
);
2896 TREE_CHAIN (*q
) = NULL_TREE
;
2897 q
= &TREE_CHAIN (*q
);
2900 p
= &DECL_CHAIN (*p
);
2902 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2904 /* Move fields corresponding to first and second _looptemp_
2905 clause first. There are filled by GOMP_taskloop
2906 and thus need to be in specific positions. */
2907 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2908 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2909 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2910 OMP_CLAUSE__LOOPTEMP_
);
2911 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2912 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2913 p
= &TYPE_FIELDS (ctx
->record_type
);
2915 if (*p
== f1
|| *p
== f2
)
2916 *p
= DECL_CHAIN (*p
);
2918 p
= &DECL_CHAIN (*p
);
2919 DECL_CHAIN (f1
) = f2
;
2920 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2921 TYPE_FIELDS (ctx
->record_type
) = f1
;
2922 if (ctx
->srecord_type
)
2924 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2925 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2926 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2928 if (*p
== f1
|| *p
== f2
)
2929 *p
= DECL_CHAIN (*p
);
2931 p
= &DECL_CHAIN (*p
);
2932 DECL_CHAIN (f1
) = f2
;
2933 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2934 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2937 layout_type (ctx
->record_type
);
2938 fixup_child_record_type (ctx
);
2939 if (ctx
->srecord_type
)
2940 layout_type (ctx
->srecord_type
);
2941 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2942 TYPE_SIZE_UNIT (ctx
->record_type
));
2943 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2944 t
= build_int_cst (long_integer_type_node
,
2945 TYPE_ALIGN_UNIT (ctx
->record_type
));
2946 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2950 /* Find the enclosing offload context. */
2952 static omp_context
*
2953 enclosing_target_ctx (omp_context
*ctx
)
2955 for (; ctx
; ctx
= ctx
->outer
)
2956 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2962 /* Return true if ctx is part of an oacc kernels region. */
2965 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2967 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2969 gimple
*stmt
= ctx
->stmt
;
2970 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2971 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2978 /* Check the parallelism clauses inside a kernels regions.
2979 Until kernels handling moves to use the same loop indirection
2980 scheme as parallel, we need to do this checking early. */
2983 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2985 bool checking
= true;
2986 unsigned outer_mask
= 0;
2987 unsigned this_mask
= 0;
2988 bool has_seq
= false, has_auto
= false;
2991 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2995 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2997 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
3000 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3002 switch (OMP_CLAUSE_CODE (c
))
3004 case OMP_CLAUSE_GANG
:
3005 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
3007 case OMP_CLAUSE_WORKER
:
3008 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
3010 case OMP_CLAUSE_VECTOR
:
3011 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
3013 case OMP_CLAUSE_SEQ
:
3016 case OMP_CLAUSE_AUTO
:
3026 if (has_seq
&& (this_mask
|| has_auto
))
3027 error_at (gimple_location (stmt
), "%<seq%> overrides other"
3028 " OpenACC loop specifiers");
3029 else if (has_auto
&& this_mask
)
3030 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
3031 " OpenACC loop specifiers");
3033 if (this_mask
& outer_mask
)
3034 error_at (gimple_location (stmt
), "inner loop uses same"
3035 " OpenACC parallelism as containing loop");
3038 return outer_mask
| this_mask
;
3041 /* Scan a GIMPLE_OMP_FOR. */
3044 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
3048 tree clauses
= gimple_omp_for_clauses (stmt
);
3050 ctx
= new_omp_context (stmt
, outer_ctx
);
3052 if (is_gimple_omp_oacc (stmt
))
3054 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
3056 if (!tgt
|| is_oacc_parallel (tgt
))
3057 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3059 char const *check
= NULL
;
3061 switch (OMP_CLAUSE_CODE (c
))
3063 case OMP_CLAUSE_GANG
:
3067 case OMP_CLAUSE_WORKER
:
3071 case OMP_CLAUSE_VECTOR
:
3079 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3080 error_at (gimple_location (stmt
),
3081 "argument not permitted on %qs clause in"
3082 " OpenACC %<parallel%>", check
);
3085 if (tgt
&& is_oacc_kernels (tgt
))
3087 /* Strip out reductions, as they are not handled yet. */
3088 tree
*prev_ptr
= &clauses
;
3090 while (tree probe
= *prev_ptr
)
3092 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3094 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3095 *prev_ptr
= *next_ptr
;
3097 prev_ptr
= next_ptr
;
3100 gimple_omp_for_set_clauses (stmt
, clauses
);
3101 check_oacc_kernel_gwv (stmt
, ctx
);
3105 scan_sharing_clauses (clauses
, ctx
);
3107 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3108 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3110 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3111 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3112 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3113 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3115 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3118 /* Scan an OpenMP sections directive. */
3121 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3125 ctx
= new_omp_context (stmt
, outer_ctx
);
3126 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3127 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3130 /* Scan an OpenMP single directive. */
3133 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3138 ctx
= new_omp_context (stmt
, outer_ctx
);
3139 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3140 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3141 name
= create_tmp_var_name (".omp_copy_s");
3142 name
= build_decl (gimple_location (stmt
),
3143 TYPE_DECL
, name
, ctx
->record_type
);
3144 TYPE_NAME (ctx
->record_type
) = name
;
3146 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3147 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3149 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3150 ctx
->record_type
= NULL
;
3152 layout_type (ctx
->record_type
);
3155 /* Scan a GIMPLE_OMP_TARGET. */
3158 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3162 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3163 tree clauses
= gimple_omp_target_clauses (stmt
);
3165 ctx
= new_omp_context (stmt
, outer_ctx
);
3166 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3167 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3168 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3169 name
= create_tmp_var_name (".omp_data_t");
3170 name
= build_decl (gimple_location (stmt
),
3171 TYPE_DECL
, name
, ctx
->record_type
);
3172 DECL_ARTIFICIAL (name
) = 1;
3173 DECL_NAMELESS (name
) = 1;
3174 TYPE_NAME (ctx
->record_type
) = name
;
3175 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3178 if (is_gimple_omp_oacc (stmt
))
3179 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
3182 create_omp_child_function (ctx
, false);
3183 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3186 scan_sharing_clauses (clauses
, ctx
);
3187 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3189 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3190 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3193 TYPE_FIELDS (ctx
->record_type
)
3194 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3197 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3198 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3200 field
= DECL_CHAIN (field
))
3201 gcc_assert (DECL_ALIGN (field
) == align
);
3203 layout_type (ctx
->record_type
);
3205 fixup_child_record_type (ctx
);
3209 /* Scan an OpenMP teams directive. */
3212 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3214 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3215 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3216 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3219 /* Check nesting restrictions. */
3221 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3225 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3226 inside an OpenACC CTX. */
3227 if (!(is_gimple_omp (stmt
)
3228 && is_gimple_omp_oacc (stmt
)))
3230 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
3231 if (is_gimple_omp (ctx_
->stmt
)
3232 && is_gimple_omp_oacc (ctx_
->stmt
))
3234 error_at (gimple_location (stmt
),
3235 "non-OpenACC construct inside of OpenACC region");
3242 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3243 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3246 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3248 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3249 if (c
&& OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
)
3252 error_at (gimple_location (stmt
),
3253 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3254 " may not be nested inside simd region");
3257 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3259 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3260 || (gimple_omp_for_kind (stmt
)
3261 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3262 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3264 error_at (gimple_location (stmt
),
3265 "only distribute or parallel constructs are allowed to "
3266 "be closely nested inside teams construct");
3271 switch (gimple_code (stmt
))
3273 case GIMPLE_OMP_FOR
:
3274 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3276 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3278 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3280 error_at (gimple_location (stmt
),
3281 "distribute construct must be closely nested inside "
3287 /* We split taskloop into task and nested taskloop in it. */
3288 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3290 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3295 switch (gimple_code (ctx
->stmt
))
3297 case GIMPLE_OMP_FOR
:
3298 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3299 == GF_OMP_FOR_KIND_OACC_LOOP
);
3302 case GIMPLE_OMP_TARGET
:
3303 switch (gimple_omp_target_kind (ctx
->stmt
))
3305 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3306 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3317 else if (get_oacc_fn_attrib (current_function_decl
))
3321 error_at (gimple_location (stmt
),
3322 "OpenACC loop directive must be associated with"
3323 " an OpenACC compute region");
3329 if (is_gimple_call (stmt
)
3330 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3331 == BUILT_IN_GOMP_CANCEL
3332 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3333 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3335 const char *bad
= NULL
;
3336 const char *kind
= NULL
;
3339 error_at (gimple_location (stmt
), "orphaned %qs construct",
3340 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3341 == BUILT_IN_GOMP_CANCEL
3342 ? "#pragma omp cancel"
3343 : "#pragma omp cancellation point");
3346 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3347 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3351 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3352 bad
= "#pragma omp parallel";
3353 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3354 == BUILT_IN_GOMP_CANCEL
3355 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3356 ctx
->cancellable
= true;
3360 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3361 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3362 bad
= "#pragma omp for";
3363 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3364 == BUILT_IN_GOMP_CANCEL
3365 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3367 ctx
->cancellable
= true;
3368 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3370 warning_at (gimple_location (stmt
), 0,
3371 "%<#pragma omp cancel for%> inside "
3372 "%<nowait%> for construct");
3373 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3374 OMP_CLAUSE_ORDERED
))
3375 warning_at (gimple_location (stmt
), 0,
3376 "%<#pragma omp cancel for%> inside "
3377 "%<ordered%> for construct");
3382 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3383 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3384 bad
= "#pragma omp sections";
3385 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3386 == BUILT_IN_GOMP_CANCEL
3387 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3389 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3391 ctx
->cancellable
= true;
3392 if (find_omp_clause (gimple_omp_sections_clauses
3395 warning_at (gimple_location (stmt
), 0,
3396 "%<#pragma omp cancel sections%> inside "
3397 "%<nowait%> sections construct");
3401 gcc_assert (ctx
->outer
3402 && gimple_code (ctx
->outer
->stmt
)
3403 == GIMPLE_OMP_SECTIONS
);
3404 ctx
->outer
->cancellable
= true;
3405 if (find_omp_clause (gimple_omp_sections_clauses
3408 warning_at (gimple_location (stmt
), 0,
3409 "%<#pragma omp cancel sections%> inside "
3410 "%<nowait%> sections construct");
3416 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3417 bad
= "#pragma omp task";
3419 ctx
->cancellable
= true;
3423 error_at (gimple_location (stmt
), "invalid arguments");
3428 error_at (gimple_location (stmt
),
3429 "%<%s %s%> construct not closely nested inside of %qs",
3430 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3431 == BUILT_IN_GOMP_CANCEL
3432 ? "#pragma omp cancel"
3433 : "#pragma omp cancellation point", kind
, bad
);
3438 case GIMPLE_OMP_SECTIONS
:
3439 case GIMPLE_OMP_SINGLE
:
3440 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3441 switch (gimple_code (ctx
->stmt
))
3443 case GIMPLE_OMP_FOR
:
3444 case GIMPLE_OMP_SECTIONS
:
3445 case GIMPLE_OMP_SINGLE
:
3446 case GIMPLE_OMP_ORDERED
:
3447 case GIMPLE_OMP_MASTER
:
3448 case GIMPLE_OMP_TASK
:
3449 case GIMPLE_OMP_CRITICAL
:
3450 if (is_gimple_call (stmt
))
3452 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3453 != BUILT_IN_GOMP_BARRIER
)
3455 error_at (gimple_location (stmt
),
3456 "barrier region may not be closely nested inside "
3457 "of work-sharing, critical, ordered, master or "
3458 "explicit task region");
3461 error_at (gimple_location (stmt
),
3462 "work-sharing region may not be closely nested inside "
3463 "of work-sharing, critical, ordered, master or explicit "
3466 case GIMPLE_OMP_PARALLEL
:
3472 case GIMPLE_OMP_MASTER
:
3473 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3474 switch (gimple_code (ctx
->stmt
))
3476 case GIMPLE_OMP_FOR
:
3477 case GIMPLE_OMP_SECTIONS
:
3478 case GIMPLE_OMP_SINGLE
:
3479 case GIMPLE_OMP_TASK
:
3480 error_at (gimple_location (stmt
),
3481 "master region may not be closely nested inside "
3482 "of work-sharing or explicit task region");
3484 case GIMPLE_OMP_PARALLEL
:
3490 case GIMPLE_OMP_TASK
:
3491 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3492 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3493 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3494 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3496 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3497 error_at (OMP_CLAUSE_LOCATION (c
),
3498 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3499 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3503 case GIMPLE_OMP_ORDERED
:
3504 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3505 c
; c
= OMP_CLAUSE_CHAIN (c
))
3507 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3509 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3511 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
));
3514 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3515 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3516 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3519 /* Look for containing ordered(N) loop. */
3521 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3523 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3524 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3526 error_at (OMP_CLAUSE_LOCATION (c
),
3527 "%<depend%> clause must be closely nested "
3528 "inside an ordered loop");
3531 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3533 error_at (OMP_CLAUSE_LOCATION (c
),
3534 "%<depend%> clause must be closely nested "
3535 "inside a loop with %<ordered%> clause with "
3542 error_at (OMP_CLAUSE_LOCATION (c
),
3543 "invalid depend kind in omp ordered depend");
3547 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3548 switch (gimple_code (ctx
->stmt
))
3550 case GIMPLE_OMP_CRITICAL
:
3551 case GIMPLE_OMP_TASK
:
3552 error_at (gimple_location (stmt
),
3553 "ordered region may not be closely nested inside "
3554 "of critical or explicit task region");
3556 case GIMPLE_OMP_FOR
:
3557 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3558 OMP_CLAUSE_ORDERED
) == NULL
)
3560 error_at (gimple_location (stmt
),
3561 "ordered region must be closely nested inside "
3562 "a loop region with an ordered clause");
3566 case GIMPLE_OMP_PARALLEL
:
3567 error_at (gimple_location (stmt
),
3568 "ordered region must be closely nested inside "
3569 "a loop region with an ordered clause");
3575 case GIMPLE_OMP_CRITICAL
:
3578 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3579 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3580 if (gomp_critical
*other_crit
3581 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3582 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3584 error_at (gimple_location (stmt
),
3585 "critical region may not be nested inside a critical "
3586 "region with the same name");
3591 case GIMPLE_OMP_TEAMS
:
3593 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3594 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3596 error_at (gimple_location (stmt
),
3597 "teams construct not closely nested inside of target "
3602 case GIMPLE_OMP_TARGET
:
3603 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3604 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3605 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3606 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3608 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3609 error_at (OMP_CLAUSE_LOCATION (c
),
3610 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3611 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3614 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3616 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3618 if (is_gimple_omp (stmt
)
3619 && is_gimple_omp_oacc (stmt
)
3620 && is_gimple_omp (ctx
->stmt
))
3622 error_at (gimple_location (stmt
),
3623 "OpenACC construct inside of non-OpenACC region");
3629 const char *stmt_name
, *ctx_stmt_name
;
3630 switch (gimple_omp_target_kind (stmt
))
3632 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3633 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3634 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3635 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3636 stmt_name
= "target enter data"; break;
3637 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3638 stmt_name
= "target exit data"; break;
3639 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3640 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3641 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3642 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3643 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3644 stmt_name
= "enter/exit data"; break;
3645 default: gcc_unreachable ();
3647 switch (gimple_omp_target_kind (ctx
->stmt
))
3649 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3650 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3651 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3652 ctx_stmt_name
= "parallel"; break;
3653 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3654 ctx_stmt_name
= "kernels"; break;
3655 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3656 default: gcc_unreachable ();
3659 /* OpenACC/OpenMP mismatch? */
3660 if (is_gimple_omp_oacc (stmt
)
3661 != is_gimple_omp_oacc (ctx
->stmt
))
3663 error_at (gimple_location (stmt
),
3664 "%s %s construct inside of %s %s region",
3665 (is_gimple_omp_oacc (stmt
)
3666 ? "OpenACC" : "OpenMP"), stmt_name
,
3667 (is_gimple_omp_oacc (ctx
->stmt
)
3668 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3671 if (is_gimple_omp_offloaded (ctx
->stmt
))
3673 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3674 if (is_gimple_omp_oacc (ctx
->stmt
))
3676 error_at (gimple_location (stmt
),
3677 "%s construct inside of %s region",
3678 stmt_name
, ctx_stmt_name
);
3683 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3684 warning_at (gimple_location (stmt
), 0,
3685 "%s construct inside of %s region",
3686 stmt_name
, ctx_stmt_name
);
3698 /* Helper function scan_omp.
3700 Callback for walk_tree or operators in walk_gimple_stmt used to
3701 scan for OMP directives in TP. */
3704 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3706 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3707 omp_context
*ctx
= (omp_context
*) wi
->info
;
3710 switch (TREE_CODE (t
))
3717 *tp
= remap_decl (t
, &ctx
->cb
);
3721 if (ctx
&& TYPE_P (t
))
3722 *tp
= remap_type (t
, &ctx
->cb
);
3723 else if (!DECL_P (t
))
3728 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3729 if (tem
!= TREE_TYPE (t
))
3731 if (TREE_CODE (t
) == INTEGER_CST
)
3732 *tp
= wide_int_to_tree (tem
, t
);
3734 TREE_TYPE (t
) = tem
;
3744 /* Return true if FNDECL is a setjmp or a longjmp. */
3747 setjmp_or_longjmp_p (const_tree fndecl
)
3749 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3750 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3751 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3754 tree declname
= DECL_NAME (fndecl
);
3757 const char *name
= IDENTIFIER_POINTER (declname
);
3758 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3762 /* Helper function for scan_omp.
3764 Callback for walk_gimple_stmt used to scan for OMP directives in
3765 the current statement in GSI. */
3768 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3769 struct walk_stmt_info
*wi
)
3771 gimple
*stmt
= gsi_stmt (*gsi
);
3772 omp_context
*ctx
= (omp_context
*) wi
->info
;
3774 if (gimple_has_location (stmt
))
3775 input_location
= gimple_location (stmt
);
3777 /* Check the nesting restrictions. */
3778 bool remove
= false;
3779 if (is_gimple_omp (stmt
))
3780 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3781 else if (is_gimple_call (stmt
))
3783 tree fndecl
= gimple_call_fndecl (stmt
);
3786 if (setjmp_or_longjmp_p (fndecl
)
3788 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3789 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3792 error_at (gimple_location (stmt
),
3793 "setjmp/longjmp inside simd construct");
3795 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3796 switch (DECL_FUNCTION_CODE (fndecl
))
3798 case BUILT_IN_GOMP_BARRIER
:
3799 case BUILT_IN_GOMP_CANCEL
:
3800 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3801 case BUILT_IN_GOMP_TASKYIELD
:
3802 case BUILT_IN_GOMP_TASKWAIT
:
3803 case BUILT_IN_GOMP_TASKGROUP_START
:
3804 case BUILT_IN_GOMP_TASKGROUP_END
:
3805 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3814 stmt
= gimple_build_nop ();
3815 gsi_replace (gsi
, stmt
, false);
3818 *handled_ops_p
= true;
3820 switch (gimple_code (stmt
))
3822 case GIMPLE_OMP_PARALLEL
:
3823 taskreg_nesting_level
++;
3824 scan_omp_parallel (gsi
, ctx
);
3825 taskreg_nesting_level
--;
3828 case GIMPLE_OMP_TASK
:
3829 taskreg_nesting_level
++;
3830 scan_omp_task (gsi
, ctx
);
3831 taskreg_nesting_level
--;
3834 case GIMPLE_OMP_FOR
:
3835 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3838 case GIMPLE_OMP_SECTIONS
:
3839 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3842 case GIMPLE_OMP_SINGLE
:
3843 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3846 case GIMPLE_OMP_SECTION
:
3847 case GIMPLE_OMP_MASTER
:
3848 case GIMPLE_OMP_TASKGROUP
:
3849 case GIMPLE_OMP_ORDERED
:
3850 case GIMPLE_OMP_CRITICAL
:
3851 ctx
= new_omp_context (stmt
, ctx
);
3852 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3855 case GIMPLE_OMP_TARGET
:
3856 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3859 case GIMPLE_OMP_TEAMS
:
3860 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3867 *handled_ops_p
= false;
3869 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3871 var
= DECL_CHAIN (var
))
3872 insert_decl_map (&ctx
->cb
, var
, var
);
3876 *handled_ops_p
= false;
3884 /* Scan all the statements starting at the current statement. CTX
3885 contains context information about the OMP directives and
3886 clauses found during the scan. */
3889 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3891 location_t saved_location
;
3892 struct walk_stmt_info wi
;
3894 memset (&wi
, 0, sizeof (wi
));
3896 wi
.want_locations
= true;
3898 saved_location
= input_location
;
3899 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3900 input_location
= saved_location
;
3903 /* Re-gimplification and code generation routines. */
3905 /* Build a call to GOMP_barrier. */
3908 build_omp_barrier (tree lhs
)
3910 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3911 : BUILT_IN_GOMP_BARRIER
);
3912 gcall
*g
= gimple_build_call (fndecl
, 0);
3914 gimple_call_set_lhs (g
, lhs
);
3918 /* If a context was created for STMT when it was scanned, return it. */
3920 static omp_context
*
3921 maybe_lookup_ctx (gimple
*stmt
)
3924 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3925 return n
? (omp_context
*) n
->value
: NULL
;
3929 /* Find the mapping for DECL in CTX or the immediately enclosing
3930 context that has a mapping for DECL.
3932 If CTX is a nested parallel directive, we may have to use the decl
3933 mappings created in CTX's parent context. Suppose that we have the
3934 following parallel nesting (variable UIDs showed for clarity):
3937 #omp parallel shared(iD.1562) -> outer parallel
3938 iD.1562 = iD.1562 + 1;
3940 #omp parallel shared (iD.1562) -> inner parallel
3941 iD.1562 = iD.1562 - 1;
3943 Each parallel structure will create a distinct .omp_data_s structure
3944 for copying iD.1562 in/out of the directive:
3946 outer parallel .omp_data_s.1.i -> iD.1562
3947 inner parallel .omp_data_s.2.i -> iD.1562
3949 A shared variable mapping will produce a copy-out operation before
3950 the parallel directive and a copy-in operation after it. So, in
3951 this case we would have:
3954 .omp_data_o.1.i = iD.1562;
3955 #omp parallel shared(iD.1562) -> outer parallel
3956 .omp_data_i.1 = &.omp_data_o.1
3957 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3959 .omp_data_o.2.i = iD.1562; -> **
3960 #omp parallel shared(iD.1562) -> inner parallel
3961 .omp_data_i.2 = &.omp_data_o.2
3962 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3965 ** This is a problem. The symbol iD.1562 cannot be referenced
3966 inside the body of the outer parallel region. But since we are
3967 emitting this copy operation while expanding the inner parallel
3968 directive, we need to access the CTX structure of the outer
3969 parallel directive to get the correct mapping:
3971 .omp_data_o.2.i = .omp_data_i.1->i
3973 Since there may be other workshare or parallel directives enclosing
3974 the parallel directive, it may be necessary to walk up the context
3975 parent chain. This is not a problem in general because nested
3976 parallelism happens only rarely. */
3979 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3984 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3985 t
= maybe_lookup_decl (decl
, up
);
3987 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3989 return t
? t
: decl
;
3993 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3994 in outer contexts. */
3997 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4002 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4003 t
= maybe_lookup_decl (decl
, up
);
4005 return t
? t
: decl
;
4009 /* Construct the initialization value for reduction operation OP. */
4012 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4021 case TRUTH_ORIF_EXPR
:
4022 case TRUTH_XOR_EXPR
:
4024 return build_zero_cst (type
);
4027 case TRUTH_AND_EXPR
:
4028 case TRUTH_ANDIF_EXPR
:
4030 return fold_convert_loc (loc
, type
, integer_one_node
);
4033 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4036 if (SCALAR_FLOAT_TYPE_P (type
))
4038 REAL_VALUE_TYPE max
, min
;
4039 if (HONOR_INFINITIES (type
))
4042 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4045 real_maxval (&min
, 1, TYPE_MODE (type
));
4046 return build_real (type
, min
);
4048 else if (POINTER_TYPE_P (type
))
4051 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4052 return wide_int_to_tree (type
, min
);
4056 gcc_assert (INTEGRAL_TYPE_P (type
));
4057 return TYPE_MIN_VALUE (type
);
4061 if (SCALAR_FLOAT_TYPE_P (type
))
4063 REAL_VALUE_TYPE max
;
4064 if (HONOR_INFINITIES (type
))
4067 real_maxval (&max
, 0, TYPE_MODE (type
));
4068 return build_real (type
, max
);
4070 else if (POINTER_TYPE_P (type
))
4073 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4074 return wide_int_to_tree (type
, max
);
4078 gcc_assert (INTEGRAL_TYPE_P (type
));
4079 return TYPE_MAX_VALUE (type
);
4087 /* Construct the initialization value for reduction CLAUSE. */
4090 omp_reduction_init (tree clause
, tree type
)
4092 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4093 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4096 /* Return alignment to be assumed for var in CLAUSE, which should be
4097 OMP_CLAUSE_ALIGNED. */
4100 omp_clause_aligned_alignment (tree clause
)
4102 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4103 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4105 /* Otherwise return implementation defined alignment. */
4106 unsigned int al
= 1;
4107 machine_mode mode
, vmode
;
4108 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4110 vs
= 1 << floor_log2 (vs
);
4111 static enum mode_class classes
[]
4112 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4113 for (int i
= 0; i
< 4; i
+= 2)
4114 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4116 mode
= GET_MODE_WIDER_MODE (mode
))
4118 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4119 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4122 && GET_MODE_SIZE (vmode
) < vs
4123 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4124 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4126 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4127 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4129 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4130 / GET_MODE_SIZE (mode
));
4131 if (TYPE_MODE (type
) != vmode
)
4133 if (TYPE_ALIGN_UNIT (type
) > al
)
4134 al
= TYPE_ALIGN_UNIT (type
);
4136 return build_int_cst (integer_type_node
, al
);
4139 /* Return maximum possible vectorization factor for the target. */
4146 || !flag_tree_loop_optimize
4147 || (!flag_tree_loop_vectorize
4148 && (global_options_set
.x_flag_tree_loop_vectorize
4149 || global_options_set
.x_flag_tree_vectorize
)))
4152 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4155 vs
= 1 << floor_log2 (vs
);
4158 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4159 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4160 return GET_MODE_NUNITS (vqimode
);
4164 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4168 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4169 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4173 max_vf
= omp_max_vf ();
4176 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4177 OMP_CLAUSE_SAFELEN
);
4178 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4180 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4182 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4186 idx
= create_tmp_var (unsigned_type_node
);
4187 lane
= create_tmp_var (unsigned_type_node
);
4193 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4194 tree avar
= create_tmp_var_raw (atype
);
4195 if (TREE_ADDRESSABLE (new_var
))
4196 TREE_ADDRESSABLE (avar
) = 1;
4197 DECL_ATTRIBUTES (avar
)
4198 = tree_cons (get_identifier ("omp simd array"), NULL
,
4199 DECL_ATTRIBUTES (avar
));
4200 gimple_add_tmp_var (avar
);
4201 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4202 NULL_TREE
, NULL_TREE
);
4203 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4204 NULL_TREE
, NULL_TREE
);
4205 if (DECL_P (new_var
))
4207 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4208 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4213 /* Helper function of lower_rec_input_clauses. For a reference
4214 in simd reduction, add an underlying variable it will reference. */
4217 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4219 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4220 if (TREE_CONSTANT (z
))
4222 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4223 get_name (new_vard
));
4224 gimple_add_tmp_var (z
);
4225 TREE_ADDRESSABLE (z
) = 1;
4226 z
= build_fold_addr_expr_loc (loc
, z
);
4227 gimplify_assign (new_vard
, z
, ilist
);
4231 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4232 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4233 private variables. Initialization statements go in ILIST, while calls
4234 to destructors go in DLIST. */
4237 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4238 omp_context
*ctx
, struct omp_for_data
*fd
)
4240 tree c
, dtor
, copyin_seq
, x
, ptr
;
4241 bool copyin_by_ref
= false;
4242 bool lastprivate_firstprivate
= false;
4243 bool reduction_omp_orig_ref
= false;
4245 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4246 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4248 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4249 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4250 gimple_seq llist
[2] = { NULL
, NULL
};
4254 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4255 with data sharing clauses referencing variable sized vars. That
4256 is unnecessarily hard to support and very unlikely to result in
4257 vectorized code anyway. */
4259 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4260 switch (OMP_CLAUSE_CODE (c
))
4262 case OMP_CLAUSE_LINEAR
:
4263 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4266 case OMP_CLAUSE_PRIVATE
:
4267 case OMP_CLAUSE_FIRSTPRIVATE
:
4268 case OMP_CLAUSE_LASTPRIVATE
:
4269 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4272 case OMP_CLAUSE_REDUCTION
:
4273 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4274 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4281 /* Do all the fixed sized types in the first pass, and the variable sized
4282 types in the second pass. This makes sure that the scalar arguments to
4283 the variable sized types are processed before we use them in the
4284 variable sized operations. */
4285 for (pass
= 0; pass
< 2; ++pass
)
4287 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4289 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4292 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4296 case OMP_CLAUSE_PRIVATE
:
4297 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4300 case OMP_CLAUSE_SHARED
:
4301 /* Ignore shared directives in teams construct. */
4302 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4304 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4306 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4307 || is_global_var (OMP_CLAUSE_DECL (c
)));
4310 case OMP_CLAUSE_FIRSTPRIVATE
:
4311 case OMP_CLAUSE_COPYIN
:
4313 case OMP_CLAUSE_LINEAR
:
4314 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4315 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4316 lastprivate_firstprivate
= true;
4318 case OMP_CLAUSE_REDUCTION
:
4319 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4320 reduction_omp_orig_ref
= true;
4322 case OMP_CLAUSE__LOOPTEMP_
:
4323 /* Handle _looptemp_ clauses only on parallel/task. */
4327 case OMP_CLAUSE_LASTPRIVATE
:
4328 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4330 lastprivate_firstprivate
= true;
4331 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4334 /* Even without corresponding firstprivate, if
4335 decl is Fortran allocatable, it needs outer var
4338 && lang_hooks
.decls
.omp_private_outer_ref
4339 (OMP_CLAUSE_DECL (c
)))
4340 lastprivate_firstprivate
= true;
4342 case OMP_CLAUSE_ALIGNED
:
4345 var
= OMP_CLAUSE_DECL (c
);
4346 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4347 && !is_global_var (var
))
4349 new_var
= maybe_lookup_decl (var
, ctx
);
4350 if (new_var
== NULL_TREE
)
4351 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4352 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4353 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4354 omp_clause_aligned_alignment (c
));
4355 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4356 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4357 gimplify_and_add (x
, ilist
);
4359 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4360 && is_global_var (var
))
4362 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4363 new_var
= lookup_decl (var
, ctx
);
4364 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4365 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4366 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4367 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4368 omp_clause_aligned_alignment (c
));
4369 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4370 x
= create_tmp_var (ptype
);
4371 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4372 gimplify_and_add (t
, ilist
);
4373 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4374 SET_DECL_VALUE_EXPR (new_var
, t
);
4375 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4382 new_var
= var
= OMP_CLAUSE_DECL (c
);
4383 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4385 var
= TREE_OPERAND (var
, 0);
4386 if (TREE_CODE (var
) == INDIRECT_REF
4387 || TREE_CODE (var
) == ADDR_EXPR
)
4388 var
= TREE_OPERAND (var
, 0);
4389 if (is_variable_sized (var
))
4391 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4392 var
= DECL_VALUE_EXPR (var
);
4393 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4394 var
= TREE_OPERAND (var
, 0);
4395 gcc_assert (DECL_P (var
));
4399 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4400 new_var
= lookup_decl (var
, ctx
);
4402 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4407 /* C/C++ array section reductions. */
4408 else if (c_kind
== OMP_CLAUSE_REDUCTION
4409 && var
!= OMP_CLAUSE_DECL (c
))
4414 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4415 if (TREE_CODE (orig_var
) == INDIRECT_REF
4416 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4417 orig_var
= TREE_OPERAND (orig_var
, 0);
4418 tree d
= OMP_CLAUSE_DECL (c
);
4419 tree type
= TREE_TYPE (d
);
4420 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4421 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4422 const char *name
= get_name (orig_var
);
4423 if (TREE_CONSTANT (v
))
4425 x
= create_tmp_var_raw (type
, name
);
4426 gimple_add_tmp_var (x
);
4427 TREE_ADDRESSABLE (x
) = 1;
4428 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4433 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4434 tree t
= maybe_lookup_decl (v
, ctx
);
4438 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4439 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4440 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4442 build_int_cst (TREE_TYPE (v
), 1));
4443 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4445 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4446 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4447 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4450 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4451 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4452 tree y
= create_tmp_var (ptype
, name
);
4453 gimplify_assign (y
, x
, ilist
);
4455 if (TREE_CODE (TREE_OPERAND (d
, 0)) == ADDR_EXPR
)
4457 if (orig_var
!= var
)
4459 gcc_assert (is_variable_sized (orig_var
));
4460 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4462 gimplify_assign (new_var
, x
, ilist
);
4463 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4464 tree t
= build_fold_indirect_ref (new_var
);
4465 DECL_IGNORED_P (new_var
) = 0;
4466 TREE_THIS_NOTRAP (t
);
4467 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4468 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4472 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4473 build_int_cst (ptype
, 0));
4474 SET_DECL_VALUE_EXPR (new_var
, x
);
4475 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4480 gcc_assert (orig_var
== var
);
4481 if (TREE_CODE (TREE_OPERAND (d
, 0)) == INDIRECT_REF
)
4483 x
= create_tmp_var (ptype
, name
);
4484 TREE_ADDRESSABLE (x
) = 1;
4485 gimplify_assign (x
, y
, ilist
);
4486 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4488 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4489 gimplify_assign (new_var
, x
, ilist
);
4491 tree y1
= create_tmp_var (ptype
, NULL
);
4492 gimplify_assign (y1
, y
, ilist
);
4493 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4494 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4495 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4496 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4498 y2
= create_tmp_var (ptype
, NULL
);
4499 gimplify_assign (y2
, y
, ilist
);
4500 tree ref
= build_outer_var_ref (var
, ctx
);
4501 /* For ref build_outer_var_ref already performs this. */
4502 if (TREE_CODE (TREE_OPERAND (d
, 0)) == INDIRECT_REF
)
4503 gcc_assert (is_reference (var
));
4504 else if (TREE_CODE (TREE_OPERAND (d
, 0)) == ADDR_EXPR
)
4505 ref
= build_fold_addr_expr (ref
);
4506 else if (is_reference (var
))
4507 ref
= build_fold_addr_expr (ref
);
4508 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4509 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4510 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4512 y3
= create_tmp_var (ptype
, NULL
);
4513 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4517 y4
= create_tmp_var (ptype
, NULL
);
4518 gimplify_assign (y4
, ref
, dlist
);
4521 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4522 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4523 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4524 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4525 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4528 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4529 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4530 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4531 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4532 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4534 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4536 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4537 tree decl_placeholder
4538 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4539 SET_DECL_VALUE_EXPR (decl_placeholder
,
4540 build_simple_mem_ref (y1
));
4541 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4542 SET_DECL_VALUE_EXPR (placeholder
,
4543 y3
? build_simple_mem_ref (y3
)
4545 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4546 x
= lang_hooks
.decls
.omp_clause_default_ctor
4547 (c
, build_simple_mem_ref (y1
),
4548 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4550 gimplify_and_add (x
, ilist
);
4551 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4553 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4554 lower_omp (&tseq
, ctx
);
4555 gimple_seq_add_seq (ilist
, tseq
);
4557 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4560 SET_DECL_VALUE_EXPR (decl_placeholder
,
4561 build_simple_mem_ref (y2
));
4562 SET_DECL_VALUE_EXPR (placeholder
,
4563 build_simple_mem_ref (y4
));
4564 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4565 lower_omp (&tseq
, ctx
);
4566 gimple_seq_add_seq (dlist
, tseq
);
4567 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4569 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4570 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4571 x
= lang_hooks
.decls
.omp_clause_dtor
4572 (c
, build_simple_mem_ref (y2
));
4575 gimple_seq tseq
= NULL
;
4577 gimplify_stmt (&dtor
, &tseq
);
4578 gimple_seq_add_seq (dlist
, tseq
);
4583 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4584 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4586 /* reduction(-:var) sums up the partial results, so it
4587 acts identically to reduction(+:var). */
4588 if (code
== MINUS_EXPR
)
4591 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4594 x
= build2 (code
, TREE_TYPE (type
),
4595 build_simple_mem_ref (y4
),
4596 build_simple_mem_ref (y2
));
4597 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4601 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4602 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4603 gimple_seq_add_stmt (ilist
, g
);
4606 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4607 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4608 gimple_seq_add_stmt (ilist
, g
);
4610 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4611 build_int_cst (TREE_TYPE (i
), 1));
4612 gimple_seq_add_stmt (ilist
, g
);
4613 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4614 gimple_seq_add_stmt (ilist
, g
);
4615 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4618 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4619 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4620 gimple_seq_add_stmt (dlist
, g
);
4623 g
= gimple_build_assign
4624 (y4
, POINTER_PLUS_EXPR
, y4
,
4625 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4626 gimple_seq_add_stmt (dlist
, g
);
4628 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4629 build_int_cst (TREE_TYPE (i2
), 1));
4630 gimple_seq_add_stmt (dlist
, g
);
4631 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4632 gimple_seq_add_stmt (dlist
, g
);
4633 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4637 else if (is_variable_sized (var
))
4639 /* For variable sized types, we need to allocate the
4640 actual storage here. Call alloca and store the
4641 result in the pointer decl that we created elsewhere. */
4645 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4650 ptr
= DECL_VALUE_EXPR (new_var
);
4651 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4652 ptr
= TREE_OPERAND (ptr
, 0);
4653 gcc_assert (DECL_P (ptr
));
4654 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4656 /* void *tmp = __builtin_alloca */
4657 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4658 stmt
= gimple_build_call (atmp
, 2, x
,
4659 size_int (DECL_ALIGN (var
)));
4660 tmp
= create_tmp_var_raw (ptr_type_node
);
4661 gimple_add_tmp_var (tmp
);
4662 gimple_call_set_lhs (stmt
, tmp
);
4664 gimple_seq_add_stmt (ilist
, stmt
);
4666 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4667 gimplify_assign (ptr
, x
, ilist
);
4670 else if (is_reference (var
))
4672 /* For references that are being privatized for Fortran,
4673 allocate new backing storage for the new pointer
4674 variable. This allows us to avoid changing all the
4675 code that expects a pointer to something that expects
4676 a direct variable. */
4680 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4681 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4683 x
= build_receiver_ref (var
, false, ctx
);
4684 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4686 else if (TREE_CONSTANT (x
))
4688 /* For reduction in SIMD loop, defer adding the
4689 initialization of the reference, because if we decide
4690 to use SIMD array for it, the initilization could cause
4692 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4696 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4698 gimple_add_tmp_var (x
);
4699 TREE_ADDRESSABLE (x
) = 1;
4700 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4706 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4707 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4708 tree al
= size_int (TYPE_ALIGN (rtype
));
4709 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4714 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4715 gimplify_assign (new_var
, x
, ilist
);
4718 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4720 else if (c_kind
== OMP_CLAUSE_REDUCTION
4721 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4729 switch (OMP_CLAUSE_CODE (c
))
4731 case OMP_CLAUSE_SHARED
:
4732 /* Ignore shared directives in teams construct. */
4733 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4735 /* Shared global vars are just accessed directly. */
4736 if (is_global_var (new_var
))
4738 /* For taskloop firstprivate/lastprivate, represented
4739 as firstprivate and shared clause on the task, new_var
4740 is the firstprivate var. */
4741 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4743 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4744 needs to be delayed until after fixup_child_record_type so
4745 that we get the correct type during the dereference. */
4746 by_ref
= use_pointer_for_field (var
, ctx
);
4747 x
= build_receiver_ref (var
, by_ref
, ctx
);
4748 SET_DECL_VALUE_EXPR (new_var
, x
);
4749 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4751 /* ??? If VAR is not passed by reference, and the variable
4752 hasn't been initialized yet, then we'll get a warning for
4753 the store into the omp_data_s structure. Ideally, we'd be
4754 able to notice this and not store anything at all, but
4755 we're generating code too early. Suppress the warning. */
4757 TREE_NO_WARNING (var
) = 1;
4760 case OMP_CLAUSE_LASTPRIVATE
:
4761 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4765 case OMP_CLAUSE_PRIVATE
:
4766 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4767 x
= build_outer_var_ref (var
, ctx
);
4768 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4770 if (is_task_ctx (ctx
))
4771 x
= build_receiver_ref (var
, false, ctx
);
4773 x
= build_outer_var_ref (var
, ctx
);
4779 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4780 (c
, unshare_expr (new_var
), x
);
4783 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4784 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4785 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4786 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4787 idx
, lane
, ivar
, lvar
))
4790 x
= lang_hooks
.decls
.omp_clause_default_ctor
4791 (c
, unshare_expr (ivar
), x
);
4793 gimplify_and_add (x
, &llist
[0]);
4796 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4799 gimple_seq tseq
= NULL
;
4802 gimplify_stmt (&dtor
, &tseq
);
4803 gimple_seq_add_seq (&llist
[1], tseq
);
4810 gimplify_and_add (nx
, ilist
);
4814 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4817 gimple_seq tseq
= NULL
;
4820 gimplify_stmt (&dtor
, &tseq
);
4821 gimple_seq_add_seq (dlist
, tseq
);
4825 case OMP_CLAUSE_LINEAR
:
4826 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4827 goto do_firstprivate
;
4828 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4831 x
= build_outer_var_ref (var
, ctx
);
4834 case OMP_CLAUSE_FIRSTPRIVATE
:
4835 if (is_task_ctx (ctx
))
4837 if (is_reference (var
) || is_variable_sized (var
))
4839 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4841 || use_pointer_for_field (var
, NULL
))
4843 x
= build_receiver_ref (var
, false, ctx
);
4844 SET_DECL_VALUE_EXPR (new_var
, x
);
4845 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4850 x
= build_outer_var_ref (var
, ctx
);
4853 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4854 && gimple_omp_for_combined_into_p (ctx
->stmt
))
4856 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4857 tree stept
= TREE_TYPE (t
);
4858 tree ct
= find_omp_clause (clauses
,
4859 OMP_CLAUSE__LOOPTEMP_
);
4861 tree l
= OMP_CLAUSE_DECL (ct
);
4862 tree n1
= fd
->loop
.n1
;
4863 tree step
= fd
->loop
.step
;
4864 tree itype
= TREE_TYPE (l
);
4865 if (POINTER_TYPE_P (itype
))
4866 itype
= signed_type_for (itype
);
4867 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
4868 if (TYPE_UNSIGNED (itype
)
4869 && fd
->loop
.cond_code
== GT_EXPR
)
4870 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4871 fold_build1 (NEGATE_EXPR
, itype
, l
),
4872 fold_build1 (NEGATE_EXPR
,
4875 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
4876 t
= fold_build2 (MULT_EXPR
, stept
,
4877 fold_convert (stept
, l
), t
);
4879 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4881 x
= lang_hooks
.decls
.omp_clause_linear_ctor
4883 gimplify_and_add (x
, ilist
);
4887 if (POINTER_TYPE_P (TREE_TYPE (x
)))
4888 x
= fold_build2 (POINTER_PLUS_EXPR
,
4889 TREE_TYPE (x
), x
, t
);
4891 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4894 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4895 || TREE_ADDRESSABLE (new_var
))
4896 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4897 idx
, lane
, ivar
, lvar
))
4899 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4901 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4902 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4903 gimplify_and_add (x
, ilist
);
4904 gimple_stmt_iterator gsi
4905 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4907 = gimple_build_assign (unshare_expr (lvar
), iv
);
4908 gsi_insert_before_without_update (&gsi
, g
,
4910 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4911 enum tree_code code
= PLUS_EXPR
;
4912 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4913 code
= POINTER_PLUS_EXPR
;
4914 g
= gimple_build_assign (iv
, code
, iv
, t
);
4915 gsi_insert_before_without_update (&gsi
, g
,
4919 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4920 (c
, unshare_expr (ivar
), x
);
4921 gimplify_and_add (x
, &llist
[0]);
4922 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4925 gimple_seq tseq
= NULL
;
4928 gimplify_stmt (&dtor
, &tseq
);
4929 gimple_seq_add_seq (&llist
[1], tseq
);
4934 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4935 (c
, unshare_expr (new_var
), x
);
4936 gimplify_and_add (x
, ilist
);
4939 case OMP_CLAUSE__LOOPTEMP_
:
4940 gcc_assert (is_taskreg_ctx (ctx
));
4941 x
= build_outer_var_ref (var
, ctx
);
4942 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4943 gimplify_and_add (x
, ilist
);
4946 case OMP_CLAUSE_COPYIN
:
4947 by_ref
= use_pointer_for_field (var
, NULL
);
4948 x
= build_receiver_ref (var
, by_ref
, ctx
);
4949 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4950 append_to_statement_list (x
, ©in_seq
);
4951 copyin_by_ref
|= by_ref
;
4954 case OMP_CLAUSE_REDUCTION
:
4955 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4957 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4959 x
= build_outer_var_ref (var
, ctx
);
4961 if (is_reference (var
)
4962 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4964 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4965 SET_DECL_VALUE_EXPR (placeholder
, x
);
4966 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4967 tree new_vard
= new_var
;
4968 if (is_reference (var
))
4970 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4971 new_vard
= TREE_OPERAND (new_var
, 0);
4972 gcc_assert (DECL_P (new_vard
));
4975 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4976 idx
, lane
, ivar
, lvar
))
4978 if (new_vard
== new_var
)
4980 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4981 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4985 SET_DECL_VALUE_EXPR (new_vard
,
4986 build_fold_addr_expr (ivar
));
4987 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4989 x
= lang_hooks
.decls
.omp_clause_default_ctor
4990 (c
, unshare_expr (ivar
),
4991 build_outer_var_ref (var
, ctx
));
4993 gimplify_and_add (x
, &llist
[0]);
4994 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4996 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4997 lower_omp (&tseq
, ctx
);
4998 gimple_seq_add_seq (&llist
[0], tseq
);
5000 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5001 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5002 lower_omp (&tseq
, ctx
);
5003 gimple_seq_add_seq (&llist
[1], tseq
);
5004 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5005 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5006 if (new_vard
== new_var
)
5007 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5009 SET_DECL_VALUE_EXPR (new_vard
,
5010 build_fold_addr_expr (lvar
));
5011 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5016 gimplify_stmt (&dtor
, &tseq
);
5017 gimple_seq_add_seq (&llist
[1], tseq
);
5021 /* If this is a reference to constant size reduction var
5022 with placeholder, we haven't emitted the initializer
5023 for it because it is undesirable if SIMD arrays are used.
5024 But if they aren't used, we need to emit the deferred
5025 initialization now. */
5026 else if (is_reference (var
) && is_simd
)
5027 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5028 x
= lang_hooks
.decls
.omp_clause_default_ctor
5029 (c
, unshare_expr (new_var
),
5030 build_outer_var_ref (var
, ctx
));
5032 gimplify_and_add (x
, ilist
);
5033 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5035 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5036 lower_omp (&tseq
, ctx
);
5037 gimple_seq_add_seq (ilist
, tseq
);
5039 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5042 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5043 lower_omp (&tseq
, ctx
);
5044 gimple_seq_add_seq (dlist
, tseq
);
5045 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5047 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5052 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5053 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5054 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5056 /* reduction(-:var) sums up the partial results, so it
5057 acts identically to reduction(+:var). */
5058 if (code
== MINUS_EXPR
)
5061 tree new_vard
= new_var
;
5062 if (is_simd
&& is_reference (var
))
5064 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5065 new_vard
= TREE_OPERAND (new_var
, 0);
5066 gcc_assert (DECL_P (new_vard
));
5069 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5070 idx
, lane
, ivar
, lvar
))
5072 tree ref
= build_outer_var_ref (var
, ctx
);
5074 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5076 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5077 ref
= build_outer_var_ref (var
, ctx
);
5078 gimplify_assign (ref
, x
, &llist
[1]);
5080 if (new_vard
!= new_var
)
5082 SET_DECL_VALUE_EXPR (new_vard
,
5083 build_fold_addr_expr (lvar
));
5084 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5089 if (is_reference (var
) && is_simd
)
5090 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5091 gimplify_assign (new_var
, x
, ilist
);
5094 tree ref
= build_outer_var_ref (var
, ctx
);
5096 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5097 ref
= build_outer_var_ref (var
, ctx
);
5098 gimplify_assign (ref
, x
, dlist
);
5112 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5113 /* Don't want uninit warnings on simduid, it is always uninitialized,
5114 but we use it not for the value, but for the DECL_UID only. */
5115 TREE_NO_WARNING (uid
) = 1;
5117 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5118 gimple_call_set_lhs (g
, lane
);
5119 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5120 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5121 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5122 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5123 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5124 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5125 g
= gimple_build_assign (lane
, INTEGER_CST
,
5126 build_int_cst (unsigned_type_node
, 0));
5127 gimple_seq_add_stmt (ilist
, g
);
5128 for (int i
= 0; i
< 2; i
++)
5131 tree vf
= create_tmp_var (unsigned_type_node
);
5132 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5133 gimple_call_set_lhs (g
, vf
);
5134 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5135 gimple_seq_add_stmt (seq
, g
);
5136 tree t
= build_int_cst (unsigned_type_node
, 0);
5137 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5138 gimple_seq_add_stmt (seq
, g
);
5139 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5140 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5141 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5142 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5143 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5144 gimple_seq_add_seq (seq
, llist
[i
]);
5145 t
= build_int_cst (unsigned_type_node
, 1);
5146 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5147 gimple_seq_add_stmt (seq
, g
);
5148 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5149 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5150 gimple_seq_add_stmt (seq
, g
);
5151 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5155 /* The copyin sequence is not to be executed by the main thread, since
5156 that would result in self-copies. Perhaps not visible to scalars,
5157 but it certainly is to C++ operator=. */
5160 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5162 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5163 build_int_cst (TREE_TYPE (x
), 0));
5164 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5165 gimplify_and_add (x
, ilist
);
5168 /* If any copyin variable is passed by reference, we must ensure the
5169 master thread doesn't modify it before it is copied over in all
5170 threads. Similarly for variables in both firstprivate and
5171 lastprivate clauses we need to ensure the lastprivate copying
5172 happens after firstprivate copying in all threads. And similarly
5173 for UDRs if initializer expression refers to omp_orig. */
5174 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5176 /* Don't add any barrier for #pragma omp simd or
5177 #pragma omp distribute. */
5178 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5179 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5180 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5183 /* If max_vf is non-zero, then we can use only a vectorization factor
5184 up to the max_vf we chose. So stick it into the safelen clause. */
5187 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5188 OMP_CLAUSE_SAFELEN
);
5190 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5191 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5194 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5195 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5197 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5198 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5204 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5205 both parallel and workshare constructs. PREDICATE may be NULL if it's
5209 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5212 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5213 bool par_clauses
= false;
5214 tree simduid
= NULL
, lastlane
= NULL
;
5216 /* Early exit if there are no lastprivate or linear clauses. */
5217 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5218 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5219 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5220 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5222 if (clauses
== NULL
)
5224 /* If this was a workshare clause, see if it had been combined
5225 with its parallel. In that case, look for the clauses on the
5226 parallel statement itself. */
5227 if (is_parallel_ctx (ctx
))
5231 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5234 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5235 OMP_CLAUSE_LASTPRIVATE
);
5236 if (clauses
== NULL
)
5244 tree label_true
, arm1
, arm2
;
5246 label
= create_artificial_label (UNKNOWN_LOCATION
);
5247 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5248 arm1
= TREE_OPERAND (predicate
, 0);
5249 arm2
= TREE_OPERAND (predicate
, 1);
5250 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5251 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5252 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5254 gimple_seq_add_stmt (stmt_list
, stmt
);
5255 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5258 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5259 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5261 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5263 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5266 for (c
= clauses
; c
;)
5269 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5271 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5272 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5273 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5275 var
= OMP_CLAUSE_DECL (c
);
5276 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5277 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5278 && is_taskloop_ctx (ctx
))
5280 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5281 new_var
= lookup_decl (var
, ctx
->outer
);
5284 new_var
= lookup_decl (var
, ctx
);
5286 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5288 tree val
= DECL_VALUE_EXPR (new_var
);
5289 if (TREE_CODE (val
) == ARRAY_REF
5290 && VAR_P (TREE_OPERAND (val
, 0))
5291 && lookup_attribute ("omp simd array",
5292 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5295 if (lastlane
== NULL
)
5297 lastlane
= create_tmp_var (unsigned_type_node
);
5299 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5301 TREE_OPERAND (val
, 1));
5302 gimple_call_set_lhs (g
, lastlane
);
5303 gimple_seq_add_stmt (stmt_list
, g
);
5305 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5306 TREE_OPERAND (val
, 0), lastlane
,
5307 NULL_TREE
, NULL_TREE
);
5311 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5312 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5314 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5315 gimple_seq_add_seq (stmt_list
,
5316 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5317 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5319 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5320 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5322 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5323 gimple_seq_add_seq (stmt_list
,
5324 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5325 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5329 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5330 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5332 gcc_checking_assert (is_taskloop_ctx (ctx
));
5333 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5335 if (is_global_var (ovar
))
5339 x
= build_outer_var_ref (var
, ctx
, true);
5340 if (is_reference (var
))
5341 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5342 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5343 gimplify_and_add (x
, stmt_list
);
5345 c
= OMP_CLAUSE_CHAIN (c
);
5346 if (c
== NULL
&& !par_clauses
)
5348 /* If this was a workshare clause, see if it had been combined
5349 with its parallel. In that case, continue looking for the
5350 clauses also on the parallel statement itself. */
5351 if (is_parallel_ctx (ctx
))
5355 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5358 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5359 OMP_CLAUSE_LASTPRIVATE
);
5365 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5369 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
5370 tree tid
, tree var
, tree new_var
)
5372 /* The atomic add at the end of the sum creates unnecessary
5373 write contention on accelerators. To work around this,
5374 create an array to store the partial reductions. Later, in
5375 lower_omp_for (for openacc), the values of array will be
5378 tree t
= NULL_TREE
, array
, x
;
5379 tree type
= get_base_type (var
);
5382 /* Now insert the partial reductions into the array. */
5384 /* Find the reduction array. */
5386 tree ptype
= build_pointer_type (type
);
5388 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
5389 t
= build_receiver_ref (t
, false, ctx
->outer
);
5391 array
= create_tmp_var (ptype
);
5392 gimplify_assign (array
, t
, stmt_seqp
);
5394 tree ptr
= create_tmp_var (TREE_TYPE (array
));
5396 /* Find the reduction array. */
5398 /* testing a unary conversion. */
5399 tree offset
= create_tmp_var (sizetype
);
5400 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
5402 t
= create_tmp_var (sizetype
);
5403 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
5405 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
5406 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5408 /* Offset expression. Does the POINTER_PLUS_EXPR take care
5409 of adding sizeof(var) to the array? */
5410 ptr
= create_tmp_var (ptype
);
5411 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
5413 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5415 /* Move the local sum to gfc$sum[i]. */
5416 x
= unshare_expr (build_simple_mem_ref (ptr
));
5417 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
5420 /* Generate code to implement the REDUCTION clauses. */
5423 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5425 gimple_seq sub_seq
= NULL
;
5427 tree x
, c
, tid
= NULL_TREE
;
5430 /* SIMD reductions are handled in lower_rec_input_clauses. */
5431 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5432 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5435 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5436 update in that case, otherwise use a lock. */
5437 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5438 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5440 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5441 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5443 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5453 /* Initialize thread info for OpenACC. */
5454 if (is_gimple_omp_oacc (ctx
->stmt
))
5456 /* Get the current thread id. */
5457 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
5458 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
5459 gimple
*stmt
= gimple_build_call (call
, 0);
5460 gimple_call_set_lhs (stmt
, tid
);
5461 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5464 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5466 tree var
, ref
, new_var
, orig_var
;
5467 enum tree_code code
;
5468 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5470 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5473 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5474 if (TREE_CODE (var
) == MEM_REF
)
5476 var
= TREE_OPERAND (var
, 0);
5477 if (TREE_CODE (var
) == INDIRECT_REF
5478 || TREE_CODE (var
) == ADDR_EXPR
)
5479 var
= TREE_OPERAND (var
, 0);
5481 if (is_variable_sized (var
))
5483 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5484 var
= DECL_VALUE_EXPR (var
);
5485 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5486 var
= TREE_OPERAND (var
, 0);
5487 gcc_assert (DECL_P (var
));
5490 new_var
= lookup_decl (var
, ctx
);
5491 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5492 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5493 ref
= build_outer_var_ref (var
, ctx
);
5494 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5496 /* reduction(-:var) sums up the partial results, so it acts
5497 identically to reduction(+:var). */
5498 if (code
== MINUS_EXPR
)
5501 if (is_gimple_omp_oacc (ctx
->stmt
))
5503 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
5505 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
5507 else if (count
== 1)
5509 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5511 addr
= save_expr (addr
);
5512 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5513 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5514 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5515 gimplify_and_add (x
, stmt_seqp
);
5518 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5520 tree d
= OMP_CLAUSE_DECL (c
);
5521 tree type
= TREE_TYPE (d
);
5522 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5523 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5524 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5525 /* For ref build_outer_var_ref already performs this, so
5526 only new_var needs a dereference. */
5527 if (TREE_CODE (TREE_OPERAND (d
, 0)) == INDIRECT_REF
)
5529 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5530 gcc_assert (is_reference (var
) && var
== orig_var
);
5532 else if (TREE_CODE (TREE_OPERAND (d
, 0)) == ADDR_EXPR
)
5534 if (orig_var
== var
)
5536 new_var
= build_fold_addr_expr (new_var
);
5537 ref
= build_fold_addr_expr (ref
);
5542 gcc_assert (orig_var
== var
);
5543 if (is_reference (var
))
5544 ref
= build_fold_addr_expr (ref
);
5548 tree t
= maybe_lookup_decl (v
, ctx
);
5552 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5553 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5555 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5556 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5557 tree m
= create_tmp_var (ptype
, NULL
);
5558 gimplify_assign (m
, new_var
, stmt_seqp
);
5560 m
= create_tmp_var (ptype
, NULL
);
5561 gimplify_assign (m
, ref
, stmt_seqp
);
5563 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5564 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5565 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5566 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5567 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5568 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5569 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5571 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5572 tree decl_placeholder
5573 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5574 SET_DECL_VALUE_EXPR (placeholder
, out
);
5575 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5576 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5577 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5578 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5579 gimple_seq_add_seq (&sub_seq
,
5580 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5581 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5582 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5583 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5587 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5588 out
= unshare_expr (out
);
5589 gimplify_assign (out
, x
, &sub_seq
);
5591 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5592 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5593 gimple_seq_add_stmt (&sub_seq
, g
);
5594 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5595 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5596 gimple_seq_add_stmt (&sub_seq
, g
);
5597 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5598 build_int_cst (TREE_TYPE (i
), 1));
5599 gimple_seq_add_stmt (&sub_seq
, g
);
5600 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5601 gimple_seq_add_stmt (&sub_seq
, g
);
5602 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5604 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5606 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5608 if (is_reference (var
)
5609 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5611 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5612 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5613 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5614 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5615 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5616 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5617 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5621 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5622 ref
= build_outer_var_ref (var
, ctx
);
5623 gimplify_assign (ref
, x
, &sub_seq
);
5627 if (is_gimple_omp_oacc (ctx
->stmt
))
5630 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5632 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5634 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5636 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5638 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5642 /* Generate code to implement the COPYPRIVATE clauses. */
5645 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5650 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5652 tree var
, new_var
, ref
, x
;
5654 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5656 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5659 var
= OMP_CLAUSE_DECL (c
);
5660 by_ref
= use_pointer_for_field (var
, NULL
);
5662 ref
= build_sender_ref (var
, ctx
);
5663 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5666 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5667 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5669 gimplify_assign (ref
, x
, slist
);
5671 ref
= build_receiver_ref (var
, false, ctx
);
5674 ref
= fold_convert_loc (clause_loc
,
5675 build_pointer_type (TREE_TYPE (new_var
)),
5677 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5679 if (is_reference (var
))
5681 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5682 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5683 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5685 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5686 gimplify_and_add (x
, rlist
);
5691 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5692 and REDUCTION from the sender (aka parent) side. */
5695 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5699 int ignored_looptemp
= 0;
5700 bool is_taskloop
= false;
5702 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5703 by GOMP_taskloop. */
5704 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5706 ignored_looptemp
= 2;
5710 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5712 tree val
, ref
, x
, var
;
5713 bool by_ref
, do_in
= false, do_out
= false;
5714 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5716 switch (OMP_CLAUSE_CODE (c
))
5718 case OMP_CLAUSE_PRIVATE
:
5719 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
5722 case OMP_CLAUSE_FIRSTPRIVATE
:
5723 case OMP_CLAUSE_COPYIN
:
5724 case OMP_CLAUSE_LASTPRIVATE
:
5725 case OMP_CLAUSE_REDUCTION
:
5727 case OMP_CLAUSE_SHARED
:
5728 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5731 case OMP_CLAUSE__LOOPTEMP_
:
5732 if (ignored_looptemp
)
5742 val
= OMP_CLAUSE_DECL (c
);
5743 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5744 && TREE_CODE (val
) == MEM_REF
)
5746 val
= TREE_OPERAND (val
, 0);
5747 if (TREE_CODE (val
) == INDIRECT_REF
5748 || TREE_CODE (val
) == ADDR_EXPR
)
5749 val
= TREE_OPERAND (val
, 0);
5750 if (is_variable_sized (val
))
5754 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
5755 outer taskloop region. */
5756 omp_context
*ctx_for_o
= ctx
;
5758 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
5759 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5760 ctx_for_o
= ctx
->outer
;
5762 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
5764 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
5765 && is_global_var (var
))
5768 t
= omp_member_access_dummy_var (var
);
5771 var
= DECL_VALUE_EXPR (var
);
5772 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
5774 var
= unshare_and_remap (var
, t
, o
);
5776 var
= unshare_expr (var
);
5779 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
5781 /* Handle taskloop firstprivate/lastprivate, where the
5782 lastprivate on GIMPLE_OMP_TASK is represented as
5783 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
5784 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
5785 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
5786 if (use_pointer_for_field (val
, ctx
))
5787 var
= build_fold_addr_expr (var
);
5788 gimplify_assign (x
, var
, ilist
);
5789 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
5793 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
5794 || val
== OMP_CLAUSE_DECL (c
))
5795 && is_variable_sized (val
))
5797 by_ref
= use_pointer_for_field (val
, NULL
);
5799 switch (OMP_CLAUSE_CODE (c
))
5801 case OMP_CLAUSE_PRIVATE
:
5802 case OMP_CLAUSE_FIRSTPRIVATE
:
5803 case OMP_CLAUSE_COPYIN
:
5804 case OMP_CLAUSE__LOOPTEMP_
:
5808 case OMP_CLAUSE_LASTPRIVATE
:
5809 if (by_ref
|| is_reference (val
))
5811 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
5818 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
5823 case OMP_CLAUSE_REDUCTION
:
5825 if (val
== OMP_CLAUSE_DECL (c
))
5826 do_out
= !(by_ref
|| is_reference (val
));
5828 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
5837 ref
= build_sender_ref (val
, ctx
);
5838 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
5839 gimplify_assign (ref
, x
, ilist
);
5840 if (is_task_ctx (ctx
))
5841 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
5846 ref
= build_sender_ref (val
, ctx
);
5847 gimplify_assign (var
, ref
, olist
);
5852 /* Generate code to implement SHARED from the sender (aka parent)
5853 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
5854 list things that got automatically shared. */
5857 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
5859 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
5861 if (ctx
->record_type
== NULL
)
5864 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
5865 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
5867 ovar
= DECL_ABSTRACT_ORIGIN (f
);
5868 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
5871 nvar
= maybe_lookup_decl (ovar
, ctx
);
5872 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
5875 /* If CTX is a nested parallel directive. Find the immediately
5876 enclosing parallel or workshare construct that contains a
5877 mapping for OVAR. */
5878 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
5880 t
= omp_member_access_dummy_var (var
);
5883 var
= DECL_VALUE_EXPR (var
);
5884 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
5886 var
= unshare_and_remap (var
, t
, o
);
5888 var
= unshare_expr (var
);
5891 if (use_pointer_for_field (ovar
, ctx
))
5893 x
= build_sender_ref (ovar
, ctx
);
5894 var
= build_fold_addr_expr (var
);
5895 gimplify_assign (x
, var
, ilist
);
5899 x
= build_sender_ref (ovar
, ctx
);
5900 gimplify_assign (x
, var
, ilist
);
5902 if (!TREE_READONLY (var
)
5903 /* We don't need to receive a new reference to a result
5904 or parm decl. In fact we may not store to it as we will
5905 invalidate any pending RSO and generate wrong gimple
5907 && !((TREE_CODE (var
) == RESULT_DECL
5908 || TREE_CODE (var
) == PARM_DECL
)
5909 && DECL_BY_REFERENCE (var
)))
5911 x
= build_sender_ref (ovar
, ctx
);
5912 gimplify_assign (var
, x
, olist
);
5918 /* Emit an OpenACC head marker call, encapulating the partitioning and
5919 other information that must be processed by the target compiler.
5920 Return the maximum number of dimensions the associated loop might
5921 be partitioned over. */
5924 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
5925 gimple_seq
*seq
, omp_context
*ctx
)
5927 unsigned levels
= 0;
5929 tree gang_static
= NULL_TREE
;
5930 auto_vec
<tree
, 5> args
;
5932 args
.quick_push (build_int_cst
5933 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
5934 args
.quick_push (ddvar
);
5935 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5937 switch (OMP_CLAUSE_CODE (c
))
5939 case OMP_CLAUSE_GANG
:
5940 tag
|= OLF_DIM_GANG
;
5941 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
5942 /* static:* is represented by -1, and we can ignore it, as
5943 scheduling is always static. */
5944 if (gang_static
&& integer_minus_onep (gang_static
))
5945 gang_static
= NULL_TREE
;
5949 case OMP_CLAUSE_WORKER
:
5950 tag
|= OLF_DIM_WORKER
;
5954 case OMP_CLAUSE_VECTOR
:
5955 tag
|= OLF_DIM_VECTOR
;
5959 case OMP_CLAUSE_SEQ
:
5963 case OMP_CLAUSE_AUTO
:
5967 case OMP_CLAUSE_INDEPENDENT
:
5968 tag
|= OLF_INDEPENDENT
;
5978 if (DECL_P (gang_static
))
5979 gang_static
= build_outer_var_ref (gang_static
, ctx
);
5980 tag
|= OLF_GANG_STATIC
;
5983 /* In a parallel region, loops are implicitly INDEPENDENT. */
5984 omp_context
*tgt
= enclosing_target_ctx (ctx
);
5985 if (!tgt
|| is_oacc_parallel (tgt
))
5986 tag
|= OLF_INDEPENDENT
;
5988 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
5989 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
5993 /* Ensure at least one level. */
5997 args
.quick_push (build_int_cst (integer_type_node
, levels
));
5998 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6000 args
.quick_push (gang_static
);
6002 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6003 gimple_set_location (call
, loc
);
6004 gimple_set_lhs (call
, ddvar
);
6005 gimple_seq_add_stmt (seq
, call
);
6010 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6011 partitioning level of the enclosed region. */
6014 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6015 tree tofollow
, gimple_seq
*seq
)
6017 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6018 : IFN_UNIQUE_OACC_TAIL_MARK
);
6019 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6020 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6021 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6022 marker
, ddvar
, tofollow
);
6023 gimple_set_location (call
, loc
);
6024 gimple_set_lhs (call
, ddvar
);
6025 gimple_seq_add_stmt (seq
, call
);
6028 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6029 the loop clauses, from which we extract reductions. Initialize
6033 lower_oacc_head_tail (location_t loc
, tree clauses
,
6034 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6037 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6038 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6040 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6042 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6044 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6045 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6047 for (unsigned done
= 1; count
; count
--, done
++)
6049 gimple_seq fork_seq
= NULL
;
6050 gimple_seq join_seq
= NULL
;
6052 tree place
= build_int_cst (integer_type_node
, -1);
6053 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6054 fork_kind
, ddvar
, place
);
6055 gimple_set_location (fork
, loc
);
6056 gimple_set_lhs (fork
, ddvar
);
6058 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6059 join_kind
, ddvar
, place
);
6060 gimple_set_location (join
, loc
);
6061 gimple_set_lhs (join
, ddvar
);
6063 /* Mark the beginning of this level sequence. */
6065 lower_oacc_loop_marker (loc
, ddvar
, true,
6066 build_int_cst (integer_type_node
, count
),
6068 lower_oacc_loop_marker (loc
, ddvar
, false,
6069 build_int_cst (integer_type_node
, done
),
6072 gimple_seq_add_stmt (&fork_seq
, fork
);
6073 gimple_seq_add_stmt (&join_seq
, join
);
6075 /* Append this level to head. */
6076 gimple_seq_add_seq (head
, fork_seq
);
6077 /* Prepend it to tail. */
6078 gimple_seq_add_seq (&join_seq
, *tail
);
6084 /* Mark the end of the sequence. */
6085 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6086 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6089 /* A convenience function to build an empty GIMPLE_COND with just the
6093 gimple_build_cond_empty (tree cond
)
6095 enum tree_code pred_code
;
6098 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6099 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6102 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6105 /* Build the function calls to GOMP_parallel_start etc to actually
6106 generate the parallel operation. REGION is the parallel region
6107 being expanded. BB is the block where to insert the code. WS_ARGS
6108 will be set if this is a call to a combined parallel+workshare
6109 construct, it contains the list of additional arguments needed by
6110 the workshare construct. */
6113 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6114 gomp_parallel
*entry_stmt
,
6115 vec
<tree
, va_gc
> *ws_args
)
6117 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6118 gimple_stmt_iterator gsi
;
6120 enum built_in_function start_ix
;
6122 location_t clause_loc
;
6123 vec
<tree
, va_gc
> *args
;
6125 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6127 /* Determine what flavor of GOMP_parallel we will be
6129 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6130 if (is_combined_parallel (region
))
6132 switch (region
->inner
->type
)
6134 case GIMPLE_OMP_FOR
:
6135 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6136 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
6137 + (region
->inner
->sched_kind
6138 == OMP_CLAUSE_SCHEDULE_RUNTIME
6139 ? 3 : region
->inner
->sched_kind
));
6140 start_ix
= (enum built_in_function
)start_ix2
;
6142 case GIMPLE_OMP_SECTIONS
:
6143 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6150 /* By default, the value of NUM_THREADS is zero (selected at run time)
6151 and there is no conditional. */
6153 val
= build_int_cst (unsigned_type_node
, 0);
6154 flags
= build_int_cst (unsigned_type_node
, 0);
6156 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6158 cond
= OMP_CLAUSE_IF_EXPR (c
);
6160 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6163 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6164 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6167 clause_loc
= gimple_location (entry_stmt
);
6169 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6171 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6173 /* Ensure 'val' is of the correct type. */
6174 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6176 /* If we found the clause 'if (cond)', build either
6177 (cond != 0) or (cond ? val : 1u). */
6180 cond
= gimple_boolify (cond
);
6182 if (integer_zerop (val
))
6183 val
= fold_build2_loc (clause_loc
,
6184 EQ_EXPR
, unsigned_type_node
, cond
,
6185 build_int_cst (TREE_TYPE (cond
), 0));
6188 basic_block cond_bb
, then_bb
, else_bb
;
6189 edge e
, e_then
, e_else
;
6190 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6192 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6193 if (gimple_in_ssa_p (cfun
))
6195 tmp_then
= make_ssa_name (tmp_var
);
6196 tmp_else
= make_ssa_name (tmp_var
);
6197 tmp_join
= make_ssa_name (tmp_var
);
6206 e
= split_block_after_labels (bb
);
6211 then_bb
= create_empty_bb (cond_bb
);
6212 else_bb
= create_empty_bb (then_bb
);
6213 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6214 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6216 stmt
= gimple_build_cond_empty (cond
);
6217 gsi
= gsi_start_bb (cond_bb
);
6218 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6220 gsi
= gsi_start_bb (then_bb
);
6221 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6223 gsi
= gsi_start_bb (else_bb
);
6224 expand_omp_build_assign (&gsi
, tmp_else
,
6225 build_int_cst (unsigned_type_node
, 1),
6228 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6229 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6230 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6231 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6232 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6233 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6235 if (gimple_in_ssa_p (cfun
))
6237 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6238 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6239 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6245 gsi
= gsi_start_bb (bb
);
6246 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6247 false, GSI_CONTINUE_LINKING
);
6250 gsi
= gsi_last_bb (bb
);
6251 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6253 t1
= null_pointer_node
;
6255 t1
= build_fold_addr_expr (t
);
6256 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6258 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6259 args
->quick_push (t2
);
6260 args
->quick_push (t1
);
6261 args
->quick_push (val
);
6263 args
->splice (*ws_args
);
6264 args
->quick_push (flags
);
6266 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6267 builtin_decl_explicit (start_ix
), args
);
6269 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6270 false, GSI_CONTINUE_LINKING
);
6273 /* Insert a function call whose name is FUNC_NAME with the information from
6274 ENTRY_STMT into the basic_block BB. */
6277 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6278 vec
<tree
, va_gc
> *ws_args
)
6281 gimple_stmt_iterator gsi
;
6282 vec
<tree
, va_gc
> *args
;
6284 gcc_assert (vec_safe_length (ws_args
) == 2);
6285 tree func_name
= (*ws_args
)[0];
6286 tree grain
= (*ws_args
)[1];
6288 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6289 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6290 gcc_assert (count
!= NULL_TREE
);
6291 count
= OMP_CLAUSE_OPERAND (count
, 0);
6293 gsi
= gsi_last_bb (bb
);
6294 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6296 t1
= null_pointer_node
;
6298 t1
= build_fold_addr_expr (t
);
6299 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6301 vec_alloc (args
, 4);
6302 args
->quick_push (t2
);
6303 args
->quick_push (t1
);
6304 args
->quick_push (count
);
6305 args
->quick_push (grain
);
6306 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6308 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6309 GSI_CONTINUE_LINKING
);
6312 /* Build the function call to GOMP_task to actually
6313 generate the task operation. BB is the block where to insert the code. */
6316 expand_task_call (struct omp_region
*region
, basic_block bb
,
6317 gomp_task
*entry_stmt
)
6320 gimple_stmt_iterator gsi
;
6321 location_t loc
= gimple_location (entry_stmt
);
6323 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6325 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6326 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6327 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6328 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6329 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6330 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6333 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6334 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6335 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6337 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6338 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6339 tree num_tasks
= NULL_TREE
;
6343 gimple
*g
= last_stmt (region
->outer
->entry
);
6344 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6345 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6346 struct omp_for_data fd
;
6347 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6348 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6349 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6350 OMP_CLAUSE__LOOPTEMP_
);
6351 startvar
= OMP_CLAUSE_DECL (startvar
);
6352 endvar
= OMP_CLAUSE_DECL (endvar
);
6353 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6354 if (fd
.loop
.cond_code
== LT_EXPR
)
6355 iflags
|= GOMP_TASK_FLAG_UP
;
6356 tree tclauses
= gimple_omp_for_clauses (g
);
6357 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6359 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6362 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6365 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6366 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6369 num_tasks
= integer_zero_node
;
6371 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6372 if (ifc
== NULL_TREE
)
6373 iflags
|= GOMP_TASK_FLAG_IF
;
6374 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6375 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6376 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6379 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6381 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6383 tree cond
= boolean_true_node
;
6388 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6389 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6390 build_int_cst (unsigned_type_node
,
6392 build_int_cst (unsigned_type_node
, 0));
6393 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6397 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6402 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6403 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6404 build_int_cst (unsigned_type_node
,
6405 GOMP_TASK_FLAG_FINAL
),
6406 build_int_cst (unsigned_type_node
, 0));
6407 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6410 depend
= OMP_CLAUSE_DECL (depend
);
6412 depend
= build_int_cst (ptr_type_node
, 0);
6414 priority
= fold_convert (integer_type_node
,
6415 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6417 priority
= integer_zero_node
;
6419 gsi
= gsi_last_bb (bb
);
6420 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6422 t2
= null_pointer_node
;
6424 t2
= build_fold_addr_expr_loc (loc
, t
);
6425 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6426 t
= gimple_omp_task_copy_fn (entry_stmt
);
6428 t3
= null_pointer_node
;
6430 t3
= build_fold_addr_expr_loc (loc
, t
);
6433 t
= build_call_expr (ull
6434 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6435 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6437 gimple_omp_task_arg_size (entry_stmt
),
6438 gimple_omp_task_arg_align (entry_stmt
), flags
,
6439 num_tasks
, priority
, startvar
, endvar
, step
);
6441 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6443 gimple_omp_task_arg_size (entry_stmt
),
6444 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6447 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6448 false, GSI_CONTINUE_LINKING
);
6452 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6453 catch handler and return it. This prevents programs from violating the
6454 structured block semantics with throws. */
6457 maybe_catch_exception (gimple_seq body
)
6462 if (!flag_exceptions
)
6465 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6466 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6468 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6470 g
= gimple_build_eh_must_not_throw (decl
);
6471 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6474 return gimple_seq_alloc_with_stmt (g
);
6477 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6480 vec2chain (vec
<tree
, va_gc
> *v
)
6482 tree chain
= NULL_TREE
, t
;
6485 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6487 DECL_CHAIN (t
) = chain
;
6495 /* Remove barriers in REGION->EXIT's block. Note that this is only
6496 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6497 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6498 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6502 remove_exit_barrier (struct omp_region
*region
)
6504 gimple_stmt_iterator gsi
;
6505 basic_block exit_bb
;
6509 int any_addressable_vars
= -1;
6511 exit_bb
= region
->exit
;
6513 /* If the parallel region doesn't return, we don't have REGION->EXIT
6518 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6519 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6520 statements that can appear in between are extremely limited -- no
6521 memory operations at all. Here, we allow nothing at all, so the
6522 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6523 gsi
= gsi_last_bb (exit_bb
);
6524 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6526 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6529 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6531 gsi
= gsi_last_bb (e
->src
);
6532 if (gsi_end_p (gsi
))
6534 stmt
= gsi_stmt (gsi
);
6535 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6536 && !gimple_omp_return_nowait_p (stmt
))
6538 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6539 in many cases. If there could be tasks queued, the barrier
6540 might be needed to let the tasks run before some local
6541 variable of the parallel that the task uses as shared
6542 runs out of scope. The task can be spawned either
6543 from within current function (this would be easy to check)
6544 or from some function it calls and gets passed an address
6545 of such a variable. */
6546 if (any_addressable_vars
< 0)
6548 gomp_parallel
*parallel_stmt
6549 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6550 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6551 tree local_decls
, block
, decl
;
6554 any_addressable_vars
= 0;
6555 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6556 if (TREE_ADDRESSABLE (decl
))
6558 any_addressable_vars
= 1;
6561 for (block
= gimple_block (stmt
);
6562 !any_addressable_vars
6564 && TREE_CODE (block
) == BLOCK
;
6565 block
= BLOCK_SUPERCONTEXT (block
))
6567 for (local_decls
= BLOCK_VARS (block
);
6569 local_decls
= DECL_CHAIN (local_decls
))
6570 if (TREE_ADDRESSABLE (local_decls
))
6572 any_addressable_vars
= 1;
6575 if (block
== gimple_block (parallel_stmt
))
6579 if (!any_addressable_vars
)
6580 gimple_omp_return_set_nowait (stmt
);
6586 remove_exit_barriers (struct omp_region
*region
)
6588 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6589 remove_exit_barrier (region
);
6593 region
= region
->inner
;
6594 remove_exit_barriers (region
);
6595 while (region
->next
)
6597 region
= region
->next
;
6598 remove_exit_barriers (region
);
6603 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6604 calls. These can't be declared as const functions, but
6605 within one parallel body they are constant, so they can be
6606 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6607 which are declared const. Similarly for task body, except
6608 that in untied task omp_get_thread_num () can change at any task
6609 scheduling point. */
6612 optimize_omp_library_calls (gimple
*entry_stmt
)
6615 gimple_stmt_iterator gsi
;
6616 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6617 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6618 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6619 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6620 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6621 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6622 OMP_CLAUSE_UNTIED
) != NULL
);
6624 FOR_EACH_BB_FN (bb
, cfun
)
6625 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6627 gimple
*call
= gsi_stmt (gsi
);
6630 if (is_gimple_call (call
)
6631 && (decl
= gimple_call_fndecl (call
))
6632 && DECL_EXTERNAL (decl
)
6633 && TREE_PUBLIC (decl
)
6634 && DECL_INITIAL (decl
) == NULL
)
6638 if (DECL_NAME (decl
) == thr_num_id
)
6640 /* In #pragma omp task untied omp_get_thread_num () can change
6641 during the execution of the task region. */
6644 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6646 else if (DECL_NAME (decl
) == num_thr_id
)
6647 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6651 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6652 || gimple_call_num_args (call
) != 0)
6655 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
6658 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
6659 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
6660 TREE_TYPE (TREE_TYPE (built_in
))))
6663 gimple_call_set_fndecl (call
, built_in
);
6668 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
6672 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
6676 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
6677 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
6680 if (TREE_CODE (t
) == ADDR_EXPR
)
6681 recompute_tree_invariant_for_addr_expr (t
);
6683 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
6687 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
6690 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
6693 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
6694 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
6695 !after
, after
? GSI_CONTINUE_LINKING
6697 gimple
*stmt
= gimple_build_assign (to
, from
);
6699 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
6701 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
6702 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
6703 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
6705 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
6706 gimple_regimplify_operands (stmt
, &gsi
);
6710 /* Expand the OpenMP parallel or task directive starting at REGION. */
6713 expand_omp_taskreg (struct omp_region
*region
)
6715 basic_block entry_bb
, exit_bb
, new_bb
;
6716 struct function
*child_cfun
;
6717 tree child_fn
, block
, t
;
6718 gimple_stmt_iterator gsi
;
6719 gimple
*entry_stmt
, *stmt
;
6721 vec
<tree
, va_gc
> *ws_args
;
6723 entry_stmt
= last_stmt (region
->entry
);
6724 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
6725 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
6727 entry_bb
= region
->entry
;
6728 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
6729 exit_bb
= region
->cont
;
6731 exit_bb
= region
->exit
;
6735 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
6736 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
6737 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
6740 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
6741 and the inner statement contains the name of the built-in function
6743 ws_args
= region
->inner
->ws_args
;
6744 else if (is_combined_parallel (region
))
6745 ws_args
= region
->ws_args
;
6749 if (child_cfun
->cfg
)
6751 /* Due to inlining, it may happen that we have already outlined
6752 the region, in which case all we need to do is make the
6753 sub-graph unreachable and emit the parallel call. */
6754 edge entry_succ_e
, exit_succ_e
;
6756 entry_succ_e
= single_succ_edge (entry_bb
);
6758 gsi
= gsi_last_bb (entry_bb
);
6759 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
6760 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
6761 gsi_remove (&gsi
, true);
6766 exit_succ_e
= single_succ_edge (exit_bb
);
6767 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
6769 remove_edge_and_dominated_blocks (entry_succ_e
);
6773 unsigned srcidx
, dstidx
, num
;
6775 /* If the parallel region needs data sent from the parent
6776 function, then the very first statement (except possible
6777 tree profile counter updates) of the parallel body
6778 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
6779 &.OMP_DATA_O is passed as an argument to the child function,
6780 we need to replace it with the argument as seen by the child
6783 In most cases, this will end up being the identity assignment
6784 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
6785 a function call that has been inlined, the original PARM_DECL
6786 .OMP_DATA_I may have been converted into a different local
6787 variable. In which case, we need to keep the assignment. */
6788 if (gimple_omp_taskreg_data_arg (entry_stmt
))
6790 basic_block entry_succ_bb
6791 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
6792 : FALLTHRU_EDGE (entry_bb
)->dest
;
6794 gimple
*parcopy_stmt
= NULL
;
6796 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
6800 gcc_assert (!gsi_end_p (gsi
));
6801 stmt
= gsi_stmt (gsi
);
6802 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
6805 if (gimple_num_ops (stmt
) == 2)
6807 tree arg
= gimple_assign_rhs1 (stmt
);
6809 /* We're ignore the subcode because we're
6810 effectively doing a STRIP_NOPS. */
6812 if (TREE_CODE (arg
) == ADDR_EXPR
6813 && TREE_OPERAND (arg
, 0)
6814 == gimple_omp_taskreg_data_arg (entry_stmt
))
6816 parcopy_stmt
= stmt
;
6822 gcc_assert (parcopy_stmt
!= NULL
);
6823 arg
= DECL_ARGUMENTS (child_fn
);
6825 if (!gimple_in_ssa_p (cfun
))
6827 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
6828 gsi_remove (&gsi
, true);
6831 /* ?? Is setting the subcode really necessary ?? */
6832 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
6833 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6838 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
6839 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
6840 /* We'd like to set the rhs to the default def in the child_fn,
6841 but it's too early to create ssa names in the child_fn.
6842 Instead, we set the rhs to the parm. In
6843 move_sese_region_to_fn, we introduce a default def for the
6844 parm, map the parm to it's default def, and once we encounter
6845 this stmt, replace the parm with the default def. */
6846 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6847 update_stmt (parcopy_stmt
);
6851 /* Declare local variables needed in CHILD_CFUN. */
6852 block
= DECL_INITIAL (child_fn
);
6853 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
6854 /* The gimplifier could record temporaries in parallel/task block
6855 rather than in containing function's local_decls chain,
6856 which would mean cgraph missed finalizing them. Do it now. */
6857 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
6858 if (TREE_CODE (t
) == VAR_DECL
6860 && !DECL_EXTERNAL (t
))
6861 varpool_node::finalize_decl (t
);
6862 DECL_SAVED_TREE (child_fn
) = NULL
;
6863 /* We'll create a CFG for child_fn, so no gimple body is needed. */
6864 gimple_set_body (child_fn
, NULL
);
6865 TREE_USED (block
) = 1;
6867 /* Reset DECL_CONTEXT on function arguments. */
6868 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
6869 DECL_CONTEXT (t
) = child_fn
;
6871 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
6872 so that it can be moved to the child function. */
6873 gsi
= gsi_last_bb (entry_bb
);
6874 stmt
= gsi_stmt (gsi
);
6875 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
6876 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
6877 e
= split_block (entry_bb
, stmt
);
6878 gsi_remove (&gsi
, true);
6881 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
6882 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6885 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
6886 gcc_assert (e2
->dest
== region
->exit
);
6887 remove_edge (BRANCH_EDGE (entry_bb
));
6888 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
6889 gsi
= gsi_last_bb (region
->exit
);
6890 gcc_assert (!gsi_end_p (gsi
)
6891 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6892 gsi_remove (&gsi
, true);
6895 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
6898 gsi
= gsi_last_bb (exit_bb
);
6899 gcc_assert (!gsi_end_p (gsi
)
6900 && (gimple_code (gsi_stmt (gsi
))
6901 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
6902 stmt
= gimple_build_return (NULL
);
6903 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
6904 gsi_remove (&gsi
, true);
6907 /* Move the parallel region into CHILD_CFUN. */
6909 if (gimple_in_ssa_p (cfun
))
6911 init_tree_ssa (child_cfun
);
6912 init_ssa_operands (child_cfun
);
6913 child_cfun
->gimple_df
->in_ssa_p
= true;
6917 block
= gimple_block (entry_stmt
);
6919 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
6921 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
6924 basic_block dest_bb
= e2
->dest
;
6926 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
6928 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
6930 /* When the OMP expansion process cannot guarantee an up-to-date
6931 loop tree arrange for the child function to fixup loops. */
6932 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
6933 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
6935 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
6936 num
= vec_safe_length (child_cfun
->local_decls
);
6937 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
6939 t
= (*child_cfun
->local_decls
)[srcidx
];
6940 if (DECL_CONTEXT (t
) == cfun
->decl
)
6942 if (srcidx
!= dstidx
)
6943 (*child_cfun
->local_decls
)[dstidx
] = t
;
6947 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
6949 /* Inform the callgraph about the new function. */
6950 child_cfun
->curr_properties
= cfun
->curr_properties
;
6951 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
6952 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
6953 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
6954 node
->parallelized_function
= 1;
6955 cgraph_node::add_new_function (child_fn
, true);
6957 /* Fix the callgraph edges for child_cfun. Those for cfun will be
6958 fixed in a following pass. */
6959 push_cfun (child_cfun
);
6961 optimize_omp_library_calls (entry_stmt
);
6962 cgraph_edge::rebuild_edges ();
6964 /* Some EH regions might become dead, see PR34608. If
6965 pass_cleanup_cfg isn't the first pass to happen with the
6966 new child, these dead EH edges might cause problems.
6967 Clean them up now. */
6968 if (flag_exceptions
)
6971 bool changed
= false;
6973 FOR_EACH_BB_FN (bb
, cfun
)
6974 changed
|= gimple_purge_dead_eh_edges (bb
);
6976 cleanup_tree_cfg ();
6978 if (gimple_in_ssa_p (cfun
))
6979 update_ssa (TODO_update_ssa
);
6980 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
6981 verify_loop_structure ();
6985 /* Emit a library call to launch the children threads. */
6987 expand_cilk_for_call (new_bb
,
6988 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
6989 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
6990 expand_parallel_call (region
, new_bb
,
6991 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
6993 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
6994 if (gimple_in_ssa_p (cfun
))
6995 update_ssa (TODO_update_ssa_only_virtuals
);
6998 /* Information about members of an OpenACC collapsed loop nest. */
7000 struct oacc_collapse
7002 tree base
; /* Base value. */
7003 tree iters
; /* Number of steps. */
7004 tree step
; /* step size. */
7007 /* Helper for expand_oacc_for. Determine collapsed loop information.
7008 Fill in COUNTS array. Emit any initialization code before GSI.
7009 Return the calculated outer loop bound of BOUND_TYPE. */
7012 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7013 gimple_stmt_iterator
*gsi
,
7014 oacc_collapse
*counts
, tree bound_type
)
7016 tree total
= build_int_cst (bound_type
, 1);
7019 gcc_assert (integer_onep (fd
->loop
.step
));
7020 gcc_assert (integer_zerop (fd
->loop
.n1
));
7022 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7024 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7026 tree iter_type
= TREE_TYPE (loop
->v
);
7027 tree diff_type
= iter_type
;
7028 tree plus_type
= iter_type
;
7030 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7032 if (POINTER_TYPE_P (iter_type
))
7033 plus_type
= sizetype
;
7034 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7035 diff_type
= signed_type_for (diff_type
);
7039 tree s
= loop
->step
;
7040 bool up
= loop
->cond_code
== LT_EXPR
;
7041 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7045 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7046 true, GSI_SAME_STMT
);
7047 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7048 true, GSI_SAME_STMT
);
7050 /* Convert the step, avoiding possible unsigned->signed overflow. */
7051 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7053 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7054 s
= fold_convert (diff_type
, s
);
7056 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7057 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7058 true, GSI_SAME_STMT
);
7060 /* Determine the range, avoiding possible unsigned->signed overflow. */
7061 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7062 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7063 fold_convert (plus_type
, negating
? b
: e
),
7064 fold_convert (plus_type
, negating
? e
: b
));
7065 expr
= fold_convert (diff_type
, expr
);
7067 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7068 tree range
= force_gimple_operand_gsi
7069 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7071 /* Determine number of iterations. */
7072 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7073 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7074 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7076 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7077 true, GSI_SAME_STMT
);
7079 counts
[ix
].base
= b
;
7080 counts
[ix
].iters
= iters
;
7081 counts
[ix
].step
= s
;
7083 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7084 fold_convert (bound_type
, iters
));
7090 /* Emit initializers for collapsed loop members. IVAR is the outer
7091 loop iteration variable, from which collapsed loop iteration values
7092 are calculated. COUNTS array has been initialized by
7093 expand_oacc_collapse_inits. */
7096 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7097 gimple_stmt_iterator
*gsi
,
7098 const oacc_collapse
*counts
, tree ivar
)
7100 tree ivar_type
= TREE_TYPE (ivar
);
7102 /* The most rapidly changing iteration variable is the innermost
7104 for (int ix
= fd
->collapse
; ix
--;)
7106 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7107 const oacc_collapse
*collapse
= &counts
[ix
];
7108 tree iter_type
= TREE_TYPE (loop
->v
);
7109 tree diff_type
= TREE_TYPE (collapse
->step
);
7110 tree plus_type
= iter_type
;
7111 enum tree_code plus_code
= PLUS_EXPR
;
7114 if (POINTER_TYPE_P (iter_type
))
7116 plus_code
= POINTER_PLUS_EXPR
;
7117 plus_type
= sizetype
;
7120 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7121 fold_convert (ivar_type
, collapse
->iters
));
7122 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7124 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7125 fold_convert (plus_type
, expr
));
7126 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7127 true, GSI_SAME_STMT
);
7128 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7129 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7133 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7134 fold_convert (ivar_type
, collapse
->iters
));
7135 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7136 true, GSI_SAME_STMT
);
7142 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7143 of the combined collapse > 1 loop constructs, generate code like:
7144 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7149 count3 = (adj + N32 - N31) / STEP3;
7150 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7155 count2 = (adj + N22 - N21) / STEP2;
7156 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7161 count1 = (adj + N12 - N11) / STEP1;
7162 count = count1 * count2 * count3;
7163 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7165 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7166 of the combined loop constructs, just initialize COUNTS array
7167 from the _looptemp_ clauses. */
7169 /* NOTE: It *could* be better to moosh all of the BBs together,
7170 creating one larger BB with all the computation and the unexpected
7171 jump at the end. I.e.
7173 bool zero3, zero2, zero1, zero;
7176 count3 = (N32 - N31) /[cl] STEP3;
7178 count2 = (N22 - N21) /[cl] STEP2;
7180 count1 = (N12 - N11) /[cl] STEP1;
7181 zero = zero3 || zero2 || zero1;
7182 count = count1 * count2 * count3;
7183 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7185 After all, we expect the zero=false, and thus we expect to have to
7186 evaluate all of the comparison expressions, so short-circuiting
7187 oughtn't be a win. Since the condition isn't protecting a
7188 denominator, we're not concerned about divide-by-zero, so we can
7189 fully evaluate count even if a numerator turned out to be wrong.
7191 It seems like putting this all together would create much better
7192 scheduling opportunities, and less pressure on the chip's branch
7196 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7197 basic_block
&entry_bb
, tree
*counts
,
7198 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7199 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7200 basic_block
&l2_dom_bb
)
7202 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7206 /* Collapsed loops need work for expansion into SSA form. */
7207 gcc_assert (!gimple_in_ssa_p (cfun
));
7209 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7210 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7212 gcc_assert (fd
->ordered
== 0);
7213 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7214 isn't supposed to be handled, as the inner loop doesn't
7216 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7217 OMP_CLAUSE__LOOPTEMP_
);
7218 gcc_assert (innerc
);
7219 for (i
= 0; i
< fd
->collapse
; i
++)
7221 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7222 OMP_CLAUSE__LOOPTEMP_
);
7223 gcc_assert (innerc
);
7225 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7227 counts
[0] = NULL_TREE
;
7232 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7234 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7235 counts
[i
] = NULL_TREE
;
7236 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7237 fold_convert (itype
, fd
->loops
[i
].n1
),
7238 fold_convert (itype
, fd
->loops
[i
].n2
));
7239 if (t
&& integer_zerop (t
))
7241 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7242 counts
[i
] = build_int_cst (type
, 0);
7246 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7248 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7250 if (i
>= fd
->collapse
&& counts
[i
])
7252 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7253 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7254 fold_convert (itype
, fd
->loops
[i
].n1
),
7255 fold_convert (itype
, fd
->loops
[i
].n2
)))
7256 == NULL_TREE
|| !integer_onep (t
)))
7260 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7261 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7262 true, GSI_SAME_STMT
);
7263 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7264 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7265 true, GSI_SAME_STMT
);
7266 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7267 NULL_TREE
, NULL_TREE
);
7268 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7269 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7270 expand_omp_regimplify_p
, NULL
, NULL
)
7271 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7272 expand_omp_regimplify_p
, NULL
, NULL
))
7274 *gsi
= gsi_for_stmt (cond_stmt
);
7275 gimple_regimplify_operands (cond_stmt
, gsi
);
7277 e
= split_block (entry_bb
, cond_stmt
);
7278 basic_block
&zero_iter_bb
7279 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7280 int &first_zero_iter
7281 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7282 if (zero_iter_bb
== NULL
)
7284 gassign
*assign_stmt
;
7285 first_zero_iter
= i
;
7286 zero_iter_bb
= create_empty_bb (entry_bb
);
7287 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7288 *gsi
= gsi_after_labels (zero_iter_bb
);
7289 if (i
< fd
->collapse
)
7290 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7291 build_zero_cst (type
));
7294 counts
[i
] = create_tmp_reg (type
, ".count");
7296 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7298 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7299 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7302 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7303 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7304 e
->flags
= EDGE_TRUE_VALUE
;
7305 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7306 if (l2_dom_bb
== NULL
)
7307 l2_dom_bb
= entry_bb
;
7309 *gsi
= gsi_last_bb (entry_bb
);
7312 if (POINTER_TYPE_P (itype
))
7313 itype
= signed_type_for (itype
);
7314 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7316 t
= fold_build2 (PLUS_EXPR
, itype
,
7317 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7318 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7319 fold_convert (itype
, fd
->loops
[i
].n2
));
7320 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7321 fold_convert (itype
, fd
->loops
[i
].n1
));
7322 /* ?? We could probably use CEIL_DIV_EXPR instead of
7323 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7324 generate the same code in the end because generically we
7325 don't know that the values involved must be negative for
7327 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7328 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7329 fold_build1 (NEGATE_EXPR
, itype
, t
),
7330 fold_build1 (NEGATE_EXPR
, itype
,
7331 fold_convert (itype
,
7332 fd
->loops
[i
].step
)));
7334 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7335 fold_convert (itype
, fd
->loops
[i
].step
));
7336 t
= fold_convert (type
, t
);
7337 if (TREE_CODE (t
) == INTEGER_CST
)
7341 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7342 counts
[i
] = create_tmp_reg (type
, ".count");
7343 expand_omp_build_assign (gsi
, counts
[i
], t
);
7345 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7350 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7351 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7357 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7359 V3 = N31 + (T % count3) * STEP3;
7361 V2 = N21 + (T % count2) * STEP2;
7363 V1 = N11 + T * STEP1;
7364 if this loop doesn't have an inner loop construct combined with it.
7365 If it does have an inner loop construct combined with it and the
7366 iteration count isn't known constant, store values from counts array
7367 into its _looptemp_ temporaries instead. */
7370 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7371 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7374 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7376 /* If fd->loop.n2 is constant, then no propagation of the counts
7377 is needed, they are constant. */
7378 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7381 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7382 ? gimple_omp_taskreg_clauses (inner_stmt
)
7383 : gimple_omp_for_clauses (inner_stmt
);
7384 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7385 isn't supposed to be handled, as the inner loop doesn't
7387 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7388 gcc_assert (innerc
);
7389 for (i
= 0; i
< fd
->collapse
; i
++)
7391 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7392 OMP_CLAUSE__LOOPTEMP_
);
7393 gcc_assert (innerc
);
7396 tree tem
= OMP_CLAUSE_DECL (innerc
);
7397 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7398 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7399 false, GSI_CONTINUE_LINKING
);
7400 gassign
*stmt
= gimple_build_assign (tem
, t
);
7401 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7407 tree type
= TREE_TYPE (fd
->loop
.v
);
7408 tree tem
= create_tmp_reg (type
, ".tem");
7409 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7410 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7412 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7414 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7416 if (POINTER_TYPE_P (vtype
))
7417 itype
= signed_type_for (vtype
);
7419 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7422 t
= fold_convert (itype
, t
);
7423 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7424 fold_convert (itype
, fd
->loops
[i
].step
));
7425 if (POINTER_TYPE_P (vtype
))
7426 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7428 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7429 t
= force_gimple_operand_gsi (gsi
, t
,
7430 DECL_P (fd
->loops
[i
].v
)
7431 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7433 GSI_CONTINUE_LINKING
);
7434 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7435 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7438 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7439 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7440 false, GSI_CONTINUE_LINKING
);
7441 stmt
= gimple_build_assign (tem
, t
);
7442 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7448 /* Helper function for expand_omp_for_*. Generate code like:
7451 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7455 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7462 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7463 basic_block body_bb
)
7465 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7467 gimple_stmt_iterator gsi
;
7473 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7475 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7477 bb
= create_empty_bb (last_bb
);
7478 add_bb_to_loop (bb
, last_bb
->loop_father
);
7479 gsi
= gsi_start_bb (bb
);
7481 if (i
< fd
->collapse
- 1)
7483 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7484 e
->probability
= REG_BR_PROB_BASE
/ 8;
7486 t
= fd
->loops
[i
+ 1].n1
;
7487 t
= force_gimple_operand_gsi (&gsi
, t
,
7488 DECL_P (fd
->loops
[i
+ 1].v
)
7489 && TREE_ADDRESSABLE (fd
->loops
[i
7492 GSI_CONTINUE_LINKING
);
7493 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7494 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7499 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7501 if (POINTER_TYPE_P (vtype
))
7502 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7504 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7505 t
= force_gimple_operand_gsi (&gsi
, t
,
7506 DECL_P (fd
->loops
[i
].v
)
7507 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7508 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7509 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7510 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7514 t
= fd
->loops
[i
].n2
;
7515 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7516 false, GSI_CONTINUE_LINKING
);
7517 tree v
= fd
->loops
[i
].v
;
7518 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7519 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7520 false, GSI_CONTINUE_LINKING
);
7521 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7522 stmt
= gimple_build_cond_empty (t
);
7523 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7524 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7525 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7528 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7536 /* Expand #pragma omp ordered depend(source). */
7539 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7540 tree
*counts
, location_t loc
)
7542 enum built_in_function source_ix
7543 = fd
->iter_type
== long_integer_type_node
7544 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7546 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7547 build_fold_addr_expr (counts
[fd
->ordered
]));
7548 gimple_set_location (g
, loc
);
7549 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7552 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7555 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7556 tree
*counts
, tree c
, location_t loc
)
7558 auto_vec
<tree
, 10> args
;
7559 enum built_in_function sink_ix
7560 = fd
->iter_type
== long_integer_type_node
7561 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7562 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7564 gimple_stmt_iterator gsi2
= *gsi
;
7565 bool warned_step
= false;
7567 for (i
= 0; i
< fd
->ordered
; i
++)
7569 off
= TREE_PURPOSE (deps
);
7570 if (!integer_zerop (off
))
7572 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7573 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7574 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7575 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7576 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7577 "lexically later iteration");
7580 deps
= TREE_CHAIN (deps
);
7582 /* If all offsets corresponding to the collapsed loops are zero,
7583 this depend clause can be ignored. FIXME: but there is still a
7584 flush needed. We need to emit one __sync_synchronize () for it
7585 though (perhaps conditionally)? Solve this together with the
7586 conservative dependence folding optimization.
7587 if (i >= fd->collapse)
7590 deps
= OMP_CLAUSE_DECL (c
);
7592 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7593 edge e2
= split_block_after_labels (e1
->dest
);
7595 *gsi
= gsi_after_labels (e1
->dest
);
7596 for (i
= 0; i
< fd
->ordered
; i
++)
7598 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7599 if (POINTER_TYPE_P (itype
))
7602 deps
= TREE_CHAIN (deps
);
7603 off
= TREE_PURPOSE (deps
);
7604 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7606 if (integer_zerop (off
))
7607 t
= boolean_true_node
;
7611 tree co
= fold_convert_loc (loc
, itype
, off
);
7612 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7614 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7615 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7616 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7617 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7620 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7621 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7622 fd
->loops
[i
].v
, co
);
7624 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7625 fd
->loops
[i
].v
, co
);
7626 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7628 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7629 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7632 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7635 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7636 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7639 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
7643 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
7647 off
= fold_convert_loc (loc
, itype
, off
);
7649 if (fd
->loops
[i
].cond_code
== LT_EXPR
7650 ? !integer_onep (fd
->loops
[i
].step
)
7651 : !integer_minus_onep (fd
->loops
[i
].step
))
7653 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7654 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
7655 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7658 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
7659 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
7660 build_int_cst (itype
, 0));
7661 if (integer_zerop (t
) && !warned_step
)
7663 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
7664 "in the iteration space");
7667 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
7671 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7677 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7678 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7679 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
7681 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7682 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
7683 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7686 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
7687 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7688 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
7689 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
7690 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7693 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
7695 if (i
< fd
->collapse
- 1)
7697 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
7702 off
= unshare_expr (off
);
7703 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
7704 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
7705 true, GSI_SAME_STMT
);
7708 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
7709 gimple_set_location (g
, loc
);
7710 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7712 *gsi
= gsi_last_bb (e1
->src
);
7713 cond
= unshare_expr (cond
);
7714 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
7715 GSI_CONTINUE_LINKING
);
7716 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
7717 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
7718 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7719 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7720 e1
->flags
= EDGE_TRUE_VALUE
;
7721 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
7723 *gsi
= gsi_after_labels (e2
->dest
);
7726 /* Expand all #pragma omp ordered depend(source) and
7727 #pragma omp ordered depend(sink:...) constructs in the current
7728 #pragma omp for ordered(n) region. */
7731 expand_omp_ordered_source_sink (struct omp_region
*region
,
7732 struct omp_for_data
*fd
, tree
*counts
,
7733 basic_block cont_bb
)
7735 struct omp_region
*inner
;
7737 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
7738 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
7739 counts
[i
] = NULL_TREE
;
7740 else if (i
>= fd
->collapse
&& !cont_bb
)
7741 counts
[i
] = build_zero_cst (fd
->iter_type
);
7742 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
7743 && integer_onep (fd
->loops
[i
].step
))
7744 counts
[i
] = NULL_TREE
;
7746 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
7748 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
7749 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
7750 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
7752 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
7753 if (inner
->type
== GIMPLE_OMP_ORDERED
)
7755 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
7756 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
7757 location_t loc
= gimple_location (ord_stmt
);
7759 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7760 c
; c
= OMP_CLAUSE_CHAIN (c
))
7761 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
7764 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
7765 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7766 c
; c
= OMP_CLAUSE_CHAIN (c
))
7767 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
7768 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
7769 gsi_remove (&gsi
, true);
7773 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
7777 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
7778 basic_block cont_bb
, basic_block body_bb
,
7779 bool ordered_lastprivate
)
7781 if (fd
->ordered
== fd
->collapse
)
7786 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7787 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7789 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7790 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
7791 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
7792 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7793 size_int (i
- fd
->collapse
+ 1),
7794 NULL_TREE
, NULL_TREE
);
7795 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7800 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
7802 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
7803 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7804 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7805 fold_convert (type
, fd
->loops
[i
].n1
));
7807 expand_omp_build_assign (&gsi
, counts
[i
],
7808 build_zero_cst (fd
->iter_type
));
7809 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7810 size_int (i
- fd
->collapse
+ 1),
7811 NULL_TREE
, NULL_TREE
);
7812 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7813 if (!gsi_end_p (gsi
))
7816 gsi
= gsi_last_bb (body_bb
);
7817 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
7818 basic_block new_body
= e1
->dest
;
7819 if (body_bb
== cont_bb
)
7822 basic_block new_header
;
7823 if (EDGE_COUNT (cont_bb
->preds
) > 0)
7825 gsi
= gsi_last_bb (cont_bb
);
7826 if (POINTER_TYPE_P (type
))
7827 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
7828 fold_convert (sizetype
,
7829 fd
->loops
[i
].step
));
7831 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
7832 fold_convert (type
, fd
->loops
[i
].step
));
7833 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7836 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
7837 build_int_cst (fd
->iter_type
, 1));
7838 expand_omp_build_assign (&gsi
, counts
[i
], t
);
7843 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7844 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7845 t
= fold_convert (fd
->iter_type
, t
);
7846 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7847 true, GSI_SAME_STMT
);
7849 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7850 size_int (i
- fd
->collapse
+ 1),
7851 NULL_TREE
, NULL_TREE
);
7852 expand_omp_build_assign (&gsi
, aref
, t
);
7854 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
7855 new_header
= e2
->dest
;
7858 new_header
= cont_bb
;
7859 gsi
= gsi_after_labels (new_header
);
7860 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
7861 true, GSI_SAME_STMT
);
7863 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
7864 true, NULL_TREE
, true, GSI_SAME_STMT
);
7865 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
7866 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
7867 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
7870 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
7871 e3
->flags
= EDGE_FALSE_VALUE
;
7872 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7873 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
7874 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7876 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
7877 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
7881 struct loop
*loop
= alloc_loop ();
7882 loop
->header
= new_header
;
7883 loop
->latch
= e2
->src
;
7884 add_loop (loop
, body_bb
->loop_father
);
7888 /* If there are any lastprivate clauses and it is possible some loops
7889 might have zero iterations, ensure all the decls are initialized,
7890 otherwise we could crash evaluating C++ class iterators with lastprivate
7892 bool need_inits
= false;
7893 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
7896 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7897 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7898 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7899 fold_convert (type
, fd
->loops
[i
].n1
));
7903 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7904 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
7906 fold_convert (type
, fd
->loops
[i
].n1
),
7907 fold_convert (type
, fd
->loops
[i
].n2
));
7908 if (!integer_onep (this_cond
))
7916 /* A subroutine of expand_omp_for. Generate code for a parallel
7917 loop with any schedule. Given parameters:
7919 for (V = N1; V cond N2; V += STEP) BODY;
7921 where COND is "<" or ">", we generate pseudocode
7923 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
7924 if (more) goto L0; else goto L3;
7931 if (V cond iend) goto L1; else goto L2;
7933 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
7936 If this is a combined omp parallel loop, instead of the call to
7937 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
7938 If this is gimple_omp_for_combined_p loop, then instead of assigning
7939 V and iend in L0 we assign the first two _looptemp_ clause decls of the
7940 inner GIMPLE_OMP_FOR and V += STEP; and
7941 if (V cond iend) goto L1; else goto L2; are removed.
7943 For collapsed loops, given parameters:
7945 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7946 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7947 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7950 we generate pseudocode
7952 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
7957 count3 = (adj + N32 - N31) / STEP3;
7958 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
7963 count2 = (adj + N22 - N21) / STEP2;
7964 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
7969 count1 = (adj + N12 - N11) / STEP1;
7970 count = count1 * count2 * count3;
7975 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
7976 if (more) goto L0; else goto L3;
7980 V3 = N31 + (T % count3) * STEP3;
7982 V2 = N21 + (T % count2) * STEP2;
7984 V1 = N11 + T * STEP1;
7989 if (V < iend) goto L10; else goto L2;
7992 if (V3 cond3 N32) goto L1; else goto L11;
7996 if (V2 cond2 N22) goto L1; else goto L12;
8002 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8008 expand_omp_for_generic (struct omp_region
*region
,
8009 struct omp_for_data
*fd
,
8010 enum built_in_function start_fn
,
8011 enum built_in_function next_fn
,
8014 tree type
, istart0
, iend0
, iend
;
8015 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8016 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8017 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8018 gimple_stmt_iterator gsi
;
8019 gassign
*assign_stmt
;
8020 bool in_combined_parallel
= is_combined_parallel (region
);
8021 bool broken_loop
= region
->cont
== NULL
;
8023 tree
*counts
= NULL
;
8025 bool ordered_lastprivate
= false;
8027 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8028 gcc_assert (fd
->iter_type
== long_integer_type_node
8029 || !in_combined_parallel
);
8031 entry_bb
= region
->entry
;
8032 cont_bb
= region
->cont
;
8034 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8035 gcc_assert (broken_loop
8036 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8037 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8038 l1_bb
= single_succ (l0_bb
);
8041 l2_bb
= create_empty_bb (cont_bb
);
8042 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8043 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8045 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8049 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8050 exit_bb
= region
->exit
;
8052 gsi
= gsi_last_bb (entry_bb
);
8054 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8056 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8057 OMP_CLAUSE_LASTPRIVATE
))
8058 ordered_lastprivate
= false;
8059 if (fd
->collapse
> 1 || fd
->ordered
)
8061 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8062 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8064 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8065 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8066 zero_iter1_bb
, first_zero_iter1
,
8067 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8071 /* Some counts[i] vars might be uninitialized if
8072 some loop has zero iterations. But the body shouldn't
8073 be executed in that case, so just avoid uninit warnings. */
8074 for (i
= first_zero_iter1
;
8075 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8076 if (SSA_VAR_P (counts
[i
]))
8077 TREE_NO_WARNING (counts
[i
]) = 1;
8079 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8081 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8082 gsi
= gsi_last_bb (entry_bb
);
8083 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8084 get_immediate_dominator (CDI_DOMINATORS
,
8089 /* Some counts[i] vars might be uninitialized if
8090 some loop has zero iterations. But the body shouldn't
8091 be executed in that case, so just avoid uninit warnings. */
8092 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8093 if (SSA_VAR_P (counts
[i
]))
8094 TREE_NO_WARNING (counts
[i
]) = 1;
8096 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8100 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8102 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8103 gsi
= gsi_last_bb (entry_bb
);
8104 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8105 get_immediate_dominator
8106 (CDI_DOMINATORS
, zero_iter2_bb
));
8109 if (fd
->collapse
== 1)
8111 counts
[0] = fd
->loop
.n2
;
8112 fd
->loop
= fd
->loops
[0];
8116 type
= TREE_TYPE (fd
->loop
.v
);
8117 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8118 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8119 TREE_ADDRESSABLE (istart0
) = 1;
8120 TREE_ADDRESSABLE (iend0
) = 1;
8122 /* See if we need to bias by LLONG_MIN. */
8123 if (fd
->iter_type
== long_long_unsigned_type_node
8124 && TREE_CODE (type
) == INTEGER_TYPE
8125 && !TYPE_UNSIGNED (type
)
8126 && fd
->ordered
== 0)
8130 if (fd
->loop
.cond_code
== LT_EXPR
)
8133 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8137 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8140 if (TREE_CODE (n1
) != INTEGER_CST
8141 || TREE_CODE (n2
) != INTEGER_CST
8142 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8143 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8146 gimple_stmt_iterator gsif
= gsi
;
8149 tree arr
= NULL_TREE
;
8150 if (in_combined_parallel
)
8152 gcc_assert (fd
->ordered
== 0);
8153 /* In a combined parallel loop, emit a call to
8154 GOMP_loop_foo_next. */
8155 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8156 build_fold_addr_expr (istart0
),
8157 build_fold_addr_expr (iend0
));
8161 tree t0
, t1
, t2
, t3
, t4
;
8162 /* If this is not a combined parallel loop, emit a call to
8163 GOMP_loop_foo_start in ENTRY_BB. */
8164 t4
= build_fold_addr_expr (iend0
);
8165 t3
= build_fold_addr_expr (istart0
);
8168 t0
= build_int_cst (unsigned_type_node
,
8169 fd
->ordered
- fd
->collapse
+ 1);
8170 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8172 - fd
->collapse
+ 1),
8174 DECL_NAMELESS (arr
) = 1;
8175 TREE_ADDRESSABLE (arr
) = 1;
8176 TREE_STATIC (arr
) = 1;
8177 vec
<constructor_elt
, va_gc
> *v
;
8178 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8181 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8184 if (idx
== 0 && fd
->collapse
> 1)
8187 c
= counts
[idx
+ fd
->collapse
- 1];
8188 tree purpose
= size_int (idx
);
8189 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8190 if (TREE_CODE (c
) != INTEGER_CST
)
8191 TREE_STATIC (arr
) = 0;
8194 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8195 if (!TREE_STATIC (arr
))
8196 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8197 void_type_node
, arr
),
8198 true, NULL_TREE
, true, GSI_SAME_STMT
);
8199 t1
= build_fold_addr_expr (arr
);
8204 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8207 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8210 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8211 OMP_CLAUSE__LOOPTEMP_
);
8212 gcc_assert (innerc
);
8213 t0
= OMP_CLAUSE_DECL (innerc
);
8214 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8215 OMP_CLAUSE__LOOPTEMP_
);
8216 gcc_assert (innerc
);
8217 t1
= OMP_CLAUSE_DECL (innerc
);
8219 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8220 && TYPE_PRECISION (TREE_TYPE (t0
))
8221 != TYPE_PRECISION (fd
->iter_type
))
8223 /* Avoid casting pointers to integer of a different size. */
8224 tree itype
= signed_type_for (type
);
8225 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8226 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8230 t1
= fold_convert (fd
->iter_type
, t1
);
8231 t0
= fold_convert (fd
->iter_type
, t0
);
8235 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8236 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8239 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8243 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8244 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8246 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8247 5, t0
, t1
, t
, t3
, t4
);
8249 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8250 6, t0
, t1
, t2
, t
, t3
, t4
);
8252 else if (fd
->ordered
)
8253 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8256 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8257 5, t0
, t1
, t2
, t3
, t4
);
8265 /* The GOMP_loop_ull_*start functions have additional boolean
8266 argument, true for < loops and false for > loops.
8267 In Fortran, the C bool type can be different from
8268 boolean_type_node. */
8269 bfn_decl
= builtin_decl_explicit (start_fn
);
8270 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8271 t5
= build_int_cst (c_bool_type
,
8272 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8275 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8276 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8277 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8278 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8281 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8282 6, t5
, t0
, t1
, t2
, t3
, t4
);
8285 if (TREE_TYPE (t
) != boolean_type_node
)
8286 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8287 t
, build_int_cst (TREE_TYPE (t
), 0));
8288 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8289 true, GSI_SAME_STMT
);
8290 if (arr
&& !TREE_STATIC (arr
))
8292 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8293 TREE_THIS_VOLATILE (clobber
) = 1;
8294 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8297 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8299 /* Remove the GIMPLE_OMP_FOR statement. */
8300 gsi_remove (&gsi
, true);
8302 if (gsi_end_p (gsif
))
8303 gsif
= gsi_after_labels (gsi_bb (gsif
));
8306 /* Iteration setup for sequential loop goes in L0_BB. */
8307 tree startvar
= fd
->loop
.v
;
8308 tree endvar
= NULL_TREE
;
8310 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8312 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8313 && gimple_omp_for_kind (inner_stmt
)
8314 == GF_OMP_FOR_KIND_SIMD
);
8315 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8316 OMP_CLAUSE__LOOPTEMP_
);
8317 gcc_assert (innerc
);
8318 startvar
= OMP_CLAUSE_DECL (innerc
);
8319 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8320 OMP_CLAUSE__LOOPTEMP_
);
8321 gcc_assert (innerc
);
8322 endvar
= OMP_CLAUSE_DECL (innerc
);
8325 gsi
= gsi_start_bb (l0_bb
);
8327 if (fd
->ordered
&& fd
->collapse
== 1)
8328 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8329 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8331 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8332 if (fd
->ordered
&& fd
->collapse
== 1)
8334 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8335 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8336 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8339 t
= fold_convert (TREE_TYPE (startvar
), t
);
8340 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8346 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8347 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8348 t
= fold_convert (TREE_TYPE (startvar
), t
);
8350 t
= force_gimple_operand_gsi (&gsi
, t
,
8352 && TREE_ADDRESSABLE (startvar
),
8353 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8354 assign_stmt
= gimple_build_assign (startvar
, t
);
8355 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8358 if (fd
->ordered
&& fd
->collapse
== 1)
8359 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8360 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8362 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8363 if (fd
->ordered
&& fd
->collapse
== 1)
8365 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8366 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8367 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8370 t
= fold_convert (TREE_TYPE (startvar
), t
);
8371 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8377 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8378 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8379 t
= fold_convert (TREE_TYPE (startvar
), t
);
8381 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8382 false, GSI_CONTINUE_LINKING
);
8385 assign_stmt
= gimple_build_assign (endvar
, iend
);
8386 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8387 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8388 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8390 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8391 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8393 /* Handle linear clause adjustments. */
8394 tree itercnt
= NULL_TREE
;
8395 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8396 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8397 c
; c
= OMP_CLAUSE_CHAIN (c
))
8398 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8399 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8401 tree d
= OMP_CLAUSE_DECL (c
);
8402 bool is_ref
= is_reference (d
);
8403 tree t
= d
, a
, dest
;
8405 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8406 tree type
= TREE_TYPE (t
);
8407 if (POINTER_TYPE_P (type
))
8409 dest
= unshare_expr (t
);
8410 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8411 expand_omp_build_assign (&gsif
, v
, t
);
8412 if (itercnt
== NULL_TREE
)
8415 tree n1
= fd
->loop
.n1
;
8416 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8419 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8421 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8423 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8425 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8426 itercnt
, fd
->loop
.step
);
8427 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8429 GSI_CONTINUE_LINKING
);
8431 a
= fold_build2 (MULT_EXPR
, type
,
8432 fold_convert (type
, itercnt
),
8433 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8434 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8435 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8436 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8437 false, GSI_CONTINUE_LINKING
);
8438 assign_stmt
= gimple_build_assign (dest
, t
);
8439 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8441 if (fd
->collapse
> 1)
8442 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8446 /* Until now, counts array contained number of iterations or
8447 variable containing it for ith loop. From now on, we need
8448 those counts only for collapsed loops, and only for the 2nd
8449 till the last collapsed one. Move those one element earlier,
8450 we'll use counts[fd->collapse - 1] for the first source/sink
8451 iteration counter and so on and counts[fd->ordered]
8452 as the array holding the current counter values for
8454 if (fd
->collapse
> 1)
8455 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8459 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8461 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8463 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8464 fold_convert (type
, fd
->loops
[i
].n1
),
8465 fold_convert (type
, fd
->loops
[i
].n2
));
8466 if (!integer_onep (this_cond
))
8469 if (i
< fd
->ordered
)
8472 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8473 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8474 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8475 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8476 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8477 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8478 make_edge (cont_bb
, l1_bb
, 0);
8479 l2_bb
= create_empty_bb (cont_bb
);
8480 broken_loop
= false;
8483 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8484 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8485 ordered_lastprivate
);
8486 if (counts
[fd
->collapse
- 1])
8488 gcc_assert (fd
->collapse
== 1);
8489 gsi
= gsi_last_bb (l0_bb
);
8490 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8492 gsi
= gsi_last_bb (cont_bb
);
8493 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8494 build_int_cst (fd
->iter_type
, 1));
8495 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8496 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8497 size_zero_node
, NULL_TREE
, NULL_TREE
);
8498 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8499 t
= counts
[fd
->collapse
- 1];
8501 else if (fd
->collapse
> 1)
8505 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8506 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8507 t
= fold_convert (fd
->iter_type
, t
);
8509 gsi
= gsi_last_bb (l0_bb
);
8510 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8511 size_zero_node
, NULL_TREE
, NULL_TREE
);
8512 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8513 false, GSI_CONTINUE_LINKING
);
8514 expand_omp_build_assign (&gsi
, aref
, t
, true);
8519 /* Code to control the increment and predicate for the sequential
8520 loop goes in the CONT_BB. */
8521 gsi
= gsi_last_bb (cont_bb
);
8522 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8523 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8524 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8525 vback
= gimple_omp_continue_control_def (cont_stmt
);
8527 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8529 if (POINTER_TYPE_P (type
))
8530 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8532 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8533 t
= force_gimple_operand_gsi (&gsi
, t
,
8535 && TREE_ADDRESSABLE (vback
),
8536 NULL_TREE
, true, GSI_SAME_STMT
);
8537 assign_stmt
= gimple_build_assign (vback
, t
);
8538 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8540 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8542 if (fd
->collapse
> 1)
8546 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8547 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8548 t
= fold_convert (fd
->iter_type
, t
);
8550 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8551 counts
[fd
->ordered
], size_zero_node
,
8552 NULL_TREE
, NULL_TREE
);
8553 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8554 true, GSI_SAME_STMT
);
8555 expand_omp_build_assign (&gsi
, aref
, t
);
8558 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8559 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8561 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8562 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8565 /* Remove GIMPLE_OMP_CONTINUE. */
8566 gsi_remove (&gsi
, true);
8568 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8569 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8571 /* Emit code to get the next parallel iteration in L2_BB. */
8572 gsi
= gsi_start_bb (l2_bb
);
8574 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8575 build_fold_addr_expr (istart0
),
8576 build_fold_addr_expr (iend0
));
8577 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8578 false, GSI_CONTINUE_LINKING
);
8579 if (TREE_TYPE (t
) != boolean_type_node
)
8580 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8581 t
, build_int_cst (TREE_TYPE (t
), 0));
8582 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8583 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8586 /* Add the loop cleanup function. */
8587 gsi
= gsi_last_bb (exit_bb
);
8588 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8589 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8590 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8591 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8593 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8594 gcall
*call_stmt
= gimple_build_call (t
, 0);
8595 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8596 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8597 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8600 tree arr
= counts
[fd
->ordered
];
8601 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8602 TREE_THIS_VOLATILE (clobber
) = 1;
8603 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8606 gsi_remove (&gsi
, true);
8608 /* Connect the new blocks. */
8609 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8610 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8616 e
= find_edge (cont_bb
, l3_bb
);
8617 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8619 phis
= phi_nodes (l3_bb
);
8620 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8622 gimple
*phi
= gsi_stmt (gsi
);
8623 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8624 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8628 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8629 e
= find_edge (cont_bb
, l1_bb
);
8632 e
= BRANCH_EDGE (cont_bb
);
8633 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8635 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8640 else if (fd
->collapse
> 1)
8643 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
8646 e
->flags
= EDGE_TRUE_VALUE
;
8649 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
8650 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
8654 e
= find_edge (cont_bb
, l2_bb
);
8655 e
->flags
= EDGE_FALLTHRU
;
8657 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
8659 if (gimple_in_ssa_p (cfun
))
8661 /* Add phis to the outer loop that connect to the phis in the inner,
8662 original loop, and move the loop entry value of the inner phi to
8663 the loop entry value of the outer phi. */
8665 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
8667 source_location locus
;
8669 gphi
*exit_phi
= psi
.phi ();
8671 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
8672 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
8674 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
8675 edge latch_to_l1
= find_edge (latch
, l1_bb
);
8677 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
8679 tree t
= gimple_phi_result (exit_phi
);
8680 tree new_res
= copy_ssa_name (t
, NULL
);
8681 nphi
= create_phi_node (new_res
, l0_bb
);
8683 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
8684 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
8685 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
8686 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
8687 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
8689 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
8690 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
8692 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
8696 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
8697 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
8698 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
8699 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
8700 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
8701 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
8702 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
8703 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
8705 /* We enter expand_omp_for_generic with a loop. This original loop may
8706 have its own loop struct, or it may be part of an outer loop struct
8707 (which may be the fake loop). */
8708 struct loop
*outer_loop
= entry_bb
->loop_father
;
8709 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
8711 add_bb_to_loop (l2_bb
, outer_loop
);
8713 /* We've added a new loop around the original loop. Allocate the
8714 corresponding loop struct. */
8715 struct loop
*new_loop
= alloc_loop ();
8716 new_loop
->header
= l0_bb
;
8717 new_loop
->latch
= l2_bb
;
8718 add_loop (new_loop
, outer_loop
);
8720 /* Allocate a loop structure for the original loop unless we already
8722 if (!orig_loop_has_loop_struct
8723 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8725 struct loop
*orig_loop
= alloc_loop ();
8726 orig_loop
->header
= l1_bb
;
8727 /* The loop may have multiple latches. */
8728 add_loop (orig_loop
, new_loop
);
8734 /* A subroutine of expand_omp_for. Generate code for a parallel
8735 loop with static schedule and no specified chunk size. Given
8738 for (V = N1; V cond N2; V += STEP) BODY;
8740 where COND is "<" or ">", we generate pseudocode
8742 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
8747 if ((__typeof (V)) -1 > 0 && cond is >)
8748 n = -(adj + N2 - N1) / -STEP;
8750 n = (adj + N2 - N1) / STEP;
8753 if (threadid < tt) goto L3; else goto L4;
8758 s0 = q * threadid + tt;
8761 if (s0 >= e0) goto L2; else goto L0;
8767 if (V cond e) goto L1;
8772 expand_omp_for_static_nochunk (struct omp_region
*region
,
8773 struct omp_for_data
*fd
,
8776 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
8777 tree type
, itype
, vmain
, vback
;
8778 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
8779 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
8781 gimple_stmt_iterator gsi
;
8783 bool broken_loop
= region
->cont
== NULL
;
8784 tree
*counts
= NULL
;
8787 itype
= type
= TREE_TYPE (fd
->loop
.v
);
8788 if (POINTER_TYPE_P (type
))
8789 itype
= signed_type_for (type
);
8791 entry_bb
= region
->entry
;
8792 cont_bb
= region
->cont
;
8793 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8794 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8795 gcc_assert (broken_loop
8796 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
8797 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8798 body_bb
= single_succ (seq_start_bb
);
8801 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
8802 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
8803 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8805 exit_bb
= region
->exit
;
8807 /* Iteration space partitioning goes in ENTRY_BB. */
8808 gsi
= gsi_last_bb (entry_bb
);
8809 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8811 if (fd
->collapse
> 1)
8813 int first_zero_iter
= -1, dummy
= -1;
8814 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
8816 counts
= XALLOCAVEC (tree
, fd
->collapse
);
8817 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8818 fin_bb
, first_zero_iter
,
8819 dummy_bb
, dummy
, l2_dom_bb
);
8822 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8823 t
= integer_one_node
;
8825 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
8826 fold_convert (type
, fd
->loop
.n1
),
8827 fold_convert (type
, fd
->loop
.n2
));
8828 if (fd
->collapse
== 1
8829 && TYPE_UNSIGNED (type
)
8830 && (t
== NULL_TREE
|| !integer_onep (t
)))
8832 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
8833 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
8834 true, GSI_SAME_STMT
);
8835 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
8836 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
8837 true, GSI_SAME_STMT
);
8838 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
8839 NULL_TREE
, NULL_TREE
);
8840 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8841 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
8842 expand_omp_regimplify_p
, NULL
, NULL
)
8843 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
8844 expand_omp_regimplify_p
, NULL
, NULL
))
8846 gsi
= gsi_for_stmt (cond_stmt
);
8847 gimple_regimplify_operands (cond_stmt
, &gsi
);
8849 ep
= split_block (entry_bb
, cond_stmt
);
8850 ep
->flags
= EDGE_TRUE_VALUE
;
8851 entry_bb
= ep
->dest
;
8852 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
8853 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
8854 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
8855 if (gimple_in_ssa_p (cfun
))
8857 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
8858 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
8859 !gsi_end_p (gpi
); gsi_next (&gpi
))
8861 gphi
*phi
= gpi
.phi ();
8862 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
8863 ep
, UNKNOWN_LOCATION
);
8866 gsi
= gsi_last_bb (entry_bb
);
8869 switch (gimple_omp_for_kind (fd
->for_stmt
))
8871 case GF_OMP_FOR_KIND_FOR
:
8872 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
8873 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8875 case GF_OMP_FOR_KIND_DISTRIBUTE
:
8876 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
8877 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
8882 nthreads
= build_call_expr (nthreads
, 0);
8883 nthreads
= fold_convert (itype
, nthreads
);
8884 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
8885 true, GSI_SAME_STMT
);
8886 threadid
= build_call_expr (threadid
, 0);
8887 threadid
= fold_convert (itype
, threadid
);
8888 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
8889 true, GSI_SAME_STMT
);
8893 step
= fd
->loop
.step
;
8894 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8896 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8897 OMP_CLAUSE__LOOPTEMP_
);
8898 gcc_assert (innerc
);
8899 n1
= OMP_CLAUSE_DECL (innerc
);
8900 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8901 OMP_CLAUSE__LOOPTEMP_
);
8902 gcc_assert (innerc
);
8903 n2
= OMP_CLAUSE_DECL (innerc
);
8905 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
8906 true, NULL_TREE
, true, GSI_SAME_STMT
);
8907 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
8908 true, NULL_TREE
, true, GSI_SAME_STMT
);
8909 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
8910 true, NULL_TREE
, true, GSI_SAME_STMT
);
8912 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
8913 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
8914 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
8915 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
8916 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
8917 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
8918 fold_build1 (NEGATE_EXPR
, itype
, t
),
8919 fold_build1 (NEGATE_EXPR
, itype
, step
));
8921 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
8922 t
= fold_convert (itype
, t
);
8923 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8925 q
= create_tmp_reg (itype
, "q");
8926 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
8927 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
8928 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
8930 tt
= create_tmp_reg (itype
, "tt");
8931 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
8932 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
8933 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
8935 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
8936 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8937 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8939 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
8940 gsi
= gsi_last_bb (second_bb
);
8941 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8943 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
8945 gassign
*assign_stmt
8946 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
8947 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8949 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
8950 gsi
= gsi_last_bb (third_bb
);
8951 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8953 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
8954 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
8955 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8957 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
8958 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8960 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
8961 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8963 /* Remove the GIMPLE_OMP_FOR statement. */
8964 gsi_remove (&gsi
, true);
8966 /* Setup code for sequential iteration goes in SEQ_START_BB. */
8967 gsi
= gsi_start_bb (seq_start_bb
);
8969 tree startvar
= fd
->loop
.v
;
8970 tree endvar
= NULL_TREE
;
8972 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8974 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
8975 ? gimple_omp_parallel_clauses (inner_stmt
)
8976 : gimple_omp_for_clauses (inner_stmt
);
8977 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
8978 gcc_assert (innerc
);
8979 startvar
= OMP_CLAUSE_DECL (innerc
);
8980 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8981 OMP_CLAUSE__LOOPTEMP_
);
8982 gcc_assert (innerc
);
8983 endvar
= OMP_CLAUSE_DECL (innerc
);
8985 t
= fold_convert (itype
, s0
);
8986 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
8987 if (POINTER_TYPE_P (type
))
8988 t
= fold_build_pointer_plus (n1
, t
);
8990 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
8991 t
= fold_convert (TREE_TYPE (startvar
), t
);
8992 t
= force_gimple_operand_gsi (&gsi
, t
,
8994 && TREE_ADDRESSABLE (startvar
),
8995 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8996 assign_stmt
= gimple_build_assign (startvar
, t
);
8997 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8999 t
= fold_convert (itype
, e0
);
9000 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9001 if (POINTER_TYPE_P (type
))
9002 t
= fold_build_pointer_plus (n1
, t
);
9004 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9005 t
= fold_convert (TREE_TYPE (startvar
), t
);
9006 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9007 false, GSI_CONTINUE_LINKING
);
9010 assign_stmt
= gimple_build_assign (endvar
, e
);
9011 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9012 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9013 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9015 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9016 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9018 /* Handle linear clause adjustments. */
9019 tree itercnt
= NULL_TREE
;
9020 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9021 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9022 c
; c
= OMP_CLAUSE_CHAIN (c
))
9023 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9024 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9026 tree d
= OMP_CLAUSE_DECL (c
);
9027 bool is_ref
= is_reference (d
);
9028 tree t
= d
, a
, dest
;
9030 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9031 if (itercnt
== NULL_TREE
)
9033 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9035 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9036 fold_convert (itype
, n1
),
9037 fold_convert (itype
, fd
->loop
.n1
));
9038 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9039 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9040 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9042 GSI_CONTINUE_LINKING
);
9047 tree type
= TREE_TYPE (t
);
9048 if (POINTER_TYPE_P (type
))
9050 a
= fold_build2 (MULT_EXPR
, type
,
9051 fold_convert (type
, itercnt
),
9052 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9053 dest
= unshare_expr (t
);
9054 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9055 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9056 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9057 false, GSI_CONTINUE_LINKING
);
9058 assign_stmt
= gimple_build_assign (dest
, t
);
9059 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9061 if (fd
->collapse
> 1)
9062 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9066 /* The code controlling the sequential loop replaces the
9067 GIMPLE_OMP_CONTINUE. */
9068 gsi
= gsi_last_bb (cont_bb
);
9069 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9070 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9071 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9072 vback
= gimple_omp_continue_control_def (cont_stmt
);
9074 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9076 if (POINTER_TYPE_P (type
))
9077 t
= fold_build_pointer_plus (vmain
, step
);
9079 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9080 t
= force_gimple_operand_gsi (&gsi
, t
,
9082 && TREE_ADDRESSABLE (vback
),
9083 NULL_TREE
, true, GSI_SAME_STMT
);
9084 assign_stmt
= gimple_build_assign (vback
, t
);
9085 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9087 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9088 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9090 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9093 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9094 gsi_remove (&gsi
, true);
9096 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9097 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9100 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9101 gsi
= gsi_last_bb (exit_bb
);
9102 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9104 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9105 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9107 gsi_remove (&gsi
, true);
9109 /* Connect all the blocks. */
9110 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9111 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9112 ep
= find_edge (entry_bb
, second_bb
);
9113 ep
->flags
= EDGE_TRUE_VALUE
;
9114 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9115 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9116 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9120 ep
= find_edge (cont_bb
, body_bb
);
9123 ep
= BRANCH_EDGE (cont_bb
);
9124 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9126 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9131 else if (fd
->collapse
> 1)
9134 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9137 ep
->flags
= EDGE_TRUE_VALUE
;
9138 find_edge (cont_bb
, fin_bb
)->flags
9139 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9142 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9143 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9144 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9146 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9147 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9148 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9149 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9151 struct loop
*loop
= body_bb
->loop_father
;
9152 if (loop
!= entry_bb
->loop_father
)
9154 gcc_assert (loop
->header
== body_bb
);
9155 gcc_assert (broken_loop
9156 || loop
->latch
== region
->cont
9157 || single_pred (loop
->latch
) == region
->cont
);
9161 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9163 loop
= alloc_loop ();
9164 loop
->header
= body_bb
;
9165 if (collapse_bb
== NULL
)
9166 loop
->latch
= cont_bb
;
9167 add_loop (loop
, body_bb
->loop_father
);
9171 /* Return phi in E->DEST with ARG on edge E. */
9174 find_phi_with_arg_on_edge (tree arg
, edge e
)
9176 basic_block bb
= e
->dest
;
9178 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9182 gphi
*phi
= gpi
.phi ();
9183 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9190 /* A subroutine of expand_omp_for. Generate code for a parallel
9191 loop with static schedule and a specified chunk size. Given
9194 for (V = N1; V cond N2; V += STEP) BODY;
9196 where COND is "<" or ">", we generate pseudocode
9198 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9203 if ((__typeof (V)) -1 > 0 && cond is >)
9204 n = -(adj + N2 - N1) / -STEP;
9206 n = (adj + N2 - N1) / STEP;
9208 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9209 here so that V is defined
9210 if the loop is not entered
9212 s0 = (trip * nthreads + threadid) * CHUNK;
9213 e0 = min(s0 + CHUNK, n);
9214 if (s0 < n) goto L1; else goto L4;
9221 if (V cond e) goto L2; else goto L3;
9229 expand_omp_for_static_chunk (struct omp_region
*region
,
9230 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9232 tree n
, s0
, e0
, e
, t
;
9233 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9234 tree type
, itype
, vmain
, vback
, vextra
;
9235 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9236 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9237 gimple_stmt_iterator gsi
;
9239 bool broken_loop
= region
->cont
== NULL
;
9240 tree
*counts
= NULL
;
9243 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9244 if (POINTER_TYPE_P (type
))
9245 itype
= signed_type_for (type
);
9247 entry_bb
= region
->entry
;
9248 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9250 iter_part_bb
= se
->dest
;
9251 cont_bb
= region
->cont
;
9252 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9253 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9254 gcc_assert (broken_loop
9255 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9256 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9257 body_bb
= single_succ (seq_start_bb
);
9260 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9261 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9262 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9263 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9265 exit_bb
= region
->exit
;
9267 /* Trip and adjustment setup goes in ENTRY_BB. */
9268 gsi
= gsi_last_bb (entry_bb
);
9269 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9271 if (fd
->collapse
> 1)
9273 int first_zero_iter
= -1, dummy
= -1;
9274 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9276 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9277 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9278 fin_bb
, first_zero_iter
,
9279 dummy_bb
, dummy
, l2_dom_bb
);
9282 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9283 t
= integer_one_node
;
9285 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9286 fold_convert (type
, fd
->loop
.n1
),
9287 fold_convert (type
, fd
->loop
.n2
));
9288 if (fd
->collapse
== 1
9289 && TYPE_UNSIGNED (type
)
9290 && (t
== NULL_TREE
|| !integer_onep (t
)))
9292 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9293 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9294 true, GSI_SAME_STMT
);
9295 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9296 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9297 true, GSI_SAME_STMT
);
9298 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9299 NULL_TREE
, NULL_TREE
);
9300 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9301 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9302 expand_omp_regimplify_p
, NULL
, NULL
)
9303 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9304 expand_omp_regimplify_p
, NULL
, NULL
))
9306 gsi
= gsi_for_stmt (cond_stmt
);
9307 gimple_regimplify_operands (cond_stmt
, &gsi
);
9309 se
= split_block (entry_bb
, cond_stmt
);
9310 se
->flags
= EDGE_TRUE_VALUE
;
9311 entry_bb
= se
->dest
;
9312 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9313 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9314 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9315 if (gimple_in_ssa_p (cfun
))
9317 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9318 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9319 !gsi_end_p (gpi
); gsi_next (&gpi
))
9321 gphi
*phi
= gpi
.phi ();
9322 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9323 se
, UNKNOWN_LOCATION
);
9326 gsi
= gsi_last_bb (entry_bb
);
9329 switch (gimple_omp_for_kind (fd
->for_stmt
))
9331 case GF_OMP_FOR_KIND_FOR
:
9332 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9333 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9335 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9336 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9337 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9342 nthreads
= build_call_expr (nthreads
, 0);
9343 nthreads
= fold_convert (itype
, nthreads
);
9344 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9345 true, GSI_SAME_STMT
);
9346 threadid
= build_call_expr (threadid
, 0);
9347 threadid
= fold_convert (itype
, threadid
);
9348 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9349 true, GSI_SAME_STMT
);
9353 step
= fd
->loop
.step
;
9354 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9356 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9357 OMP_CLAUSE__LOOPTEMP_
);
9358 gcc_assert (innerc
);
9359 n1
= OMP_CLAUSE_DECL (innerc
);
9360 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9361 OMP_CLAUSE__LOOPTEMP_
);
9362 gcc_assert (innerc
);
9363 n2
= OMP_CLAUSE_DECL (innerc
);
9365 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9366 true, NULL_TREE
, true, GSI_SAME_STMT
);
9367 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9368 true, NULL_TREE
, true, GSI_SAME_STMT
);
9369 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9370 true, NULL_TREE
, true, GSI_SAME_STMT
);
9371 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9372 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9374 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9377 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9378 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9379 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9380 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9381 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9382 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9383 fold_build1 (NEGATE_EXPR
, itype
, t
),
9384 fold_build1 (NEGATE_EXPR
, itype
, step
));
9386 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9387 t
= fold_convert (itype
, t
);
9388 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9389 true, GSI_SAME_STMT
);
9391 trip_var
= create_tmp_reg (itype
, ".trip");
9392 if (gimple_in_ssa_p (cfun
))
9394 trip_init
= make_ssa_name (trip_var
);
9395 trip_main
= make_ssa_name (trip_var
);
9396 trip_back
= make_ssa_name (trip_var
);
9400 trip_init
= trip_var
;
9401 trip_main
= trip_var
;
9402 trip_back
= trip_var
;
9405 gassign
*assign_stmt
9406 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9407 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9409 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9410 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9411 if (POINTER_TYPE_P (type
))
9412 t
= fold_build_pointer_plus (n1
, t
);
9414 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9415 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9416 true, GSI_SAME_STMT
);
9418 /* Remove the GIMPLE_OMP_FOR. */
9419 gsi_remove (&gsi
, true);
9421 gimple_stmt_iterator gsif
= gsi
;
9423 /* Iteration space partitioning goes in ITER_PART_BB. */
9424 gsi
= gsi_last_bb (iter_part_bb
);
9426 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9427 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9428 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9429 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9430 false, GSI_CONTINUE_LINKING
);
9432 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9433 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9434 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9435 false, GSI_CONTINUE_LINKING
);
9437 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9438 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9440 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9441 gsi
= gsi_start_bb (seq_start_bb
);
9443 tree startvar
= fd
->loop
.v
;
9444 tree endvar
= NULL_TREE
;
9446 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9448 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9449 ? gimple_omp_parallel_clauses (inner_stmt
)
9450 : gimple_omp_for_clauses (inner_stmt
);
9451 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9452 gcc_assert (innerc
);
9453 startvar
= OMP_CLAUSE_DECL (innerc
);
9454 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9455 OMP_CLAUSE__LOOPTEMP_
);
9456 gcc_assert (innerc
);
9457 endvar
= OMP_CLAUSE_DECL (innerc
);
9460 t
= fold_convert (itype
, s0
);
9461 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9462 if (POINTER_TYPE_P (type
))
9463 t
= fold_build_pointer_plus (n1
, t
);
9465 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9466 t
= fold_convert (TREE_TYPE (startvar
), t
);
9467 t
= force_gimple_operand_gsi (&gsi
, t
,
9469 && TREE_ADDRESSABLE (startvar
),
9470 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9471 assign_stmt
= gimple_build_assign (startvar
, t
);
9472 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9474 t
= fold_convert (itype
, e0
);
9475 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9476 if (POINTER_TYPE_P (type
))
9477 t
= fold_build_pointer_plus (n1
, t
);
9479 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9480 t
= fold_convert (TREE_TYPE (startvar
), t
);
9481 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9482 false, GSI_CONTINUE_LINKING
);
9485 assign_stmt
= gimple_build_assign (endvar
, e
);
9486 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9487 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9488 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9490 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9491 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9493 /* Handle linear clause adjustments. */
9494 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9495 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9496 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9497 c
; c
= OMP_CLAUSE_CHAIN (c
))
9498 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9499 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9501 tree d
= OMP_CLAUSE_DECL (c
);
9502 bool is_ref
= is_reference (d
);
9503 tree t
= d
, a
, dest
;
9505 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9506 tree type
= TREE_TYPE (t
);
9507 if (POINTER_TYPE_P (type
))
9509 dest
= unshare_expr (t
);
9510 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9511 expand_omp_build_assign (&gsif
, v
, t
);
9512 if (itercnt
== NULL_TREE
)
9514 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9517 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9518 fold_convert (itype
, fd
->loop
.n1
));
9519 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9522 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9525 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9526 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9528 GSI_CONTINUE_LINKING
);
9533 a
= fold_build2 (MULT_EXPR
, type
,
9534 fold_convert (type
, itercnt
),
9535 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9536 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9537 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9538 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9539 false, GSI_CONTINUE_LINKING
);
9540 assign_stmt
= gimple_build_assign (dest
, t
);
9541 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9543 if (fd
->collapse
> 1)
9544 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9548 /* The code controlling the sequential loop goes in CONT_BB,
9549 replacing the GIMPLE_OMP_CONTINUE. */
9550 gsi
= gsi_last_bb (cont_bb
);
9551 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9552 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9553 vback
= gimple_omp_continue_control_def (cont_stmt
);
9555 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9557 if (POINTER_TYPE_P (type
))
9558 t
= fold_build_pointer_plus (vmain
, step
);
9560 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9561 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9562 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9563 true, GSI_SAME_STMT
);
9564 assign_stmt
= gimple_build_assign (vback
, t
);
9565 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9567 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9568 t
= build2 (EQ_EXPR
, boolean_type_node
,
9569 build_int_cst (itype
, 0),
9570 build_int_cst (itype
, 1));
9572 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9573 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9575 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9578 /* Remove GIMPLE_OMP_CONTINUE. */
9579 gsi_remove (&gsi
, true);
9581 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9582 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9584 /* Trip update code goes into TRIP_UPDATE_BB. */
9585 gsi
= gsi_start_bb (trip_update_bb
);
9587 t
= build_int_cst (itype
, 1);
9588 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9589 assign_stmt
= gimple_build_assign (trip_back
, t
);
9590 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9593 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9594 gsi
= gsi_last_bb (exit_bb
);
9595 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9597 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9598 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9600 gsi_remove (&gsi
, true);
9602 /* Connect the new blocks. */
9603 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
9604 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
9608 se
= find_edge (cont_bb
, body_bb
);
9611 se
= BRANCH_EDGE (cont_bb
);
9612 gcc_assert (single_succ (se
->dest
) == body_bb
);
9614 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9619 else if (fd
->collapse
> 1)
9622 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9625 se
->flags
= EDGE_TRUE_VALUE
;
9626 find_edge (cont_bb
, trip_update_bb
)->flags
9627 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9629 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
9632 if (gimple_in_ssa_p (cfun
))
9640 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
9642 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
9643 remove arguments of the phi nodes in fin_bb. We need to create
9644 appropriate phi nodes in iter_part_bb instead. */
9645 se
= find_edge (iter_part_bb
, fin_bb
);
9646 re
= single_succ_edge (trip_update_bb
);
9647 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
9648 ene
= single_succ_edge (entry_bb
);
9650 psi
= gsi_start_phis (fin_bb
);
9651 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
9652 gsi_next (&psi
), ++i
)
9655 source_location locus
;
9658 t
= gimple_phi_result (phi
);
9659 gcc_assert (t
== redirect_edge_var_map_result (vm
));
9661 if (!single_pred_p (fin_bb
))
9662 t
= copy_ssa_name (t
, phi
);
9664 nphi
= create_phi_node (t
, iter_part_bb
);
9666 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
9667 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
9669 /* A special case -- fd->loop.v is not yet computed in
9670 iter_part_bb, we need to use vextra instead. */
9671 if (t
== fd
->loop
.v
)
9673 add_phi_arg (nphi
, t
, ene
, locus
);
9674 locus
= redirect_edge_var_map_location (vm
);
9675 tree back_arg
= redirect_edge_var_map_def (vm
);
9676 add_phi_arg (nphi
, back_arg
, re
, locus
);
9677 edge ce
= find_edge (cont_bb
, body_bb
);
9680 ce
= BRANCH_EDGE (cont_bb
);
9681 gcc_assert (single_succ (ce
->dest
) == body_bb
);
9682 ce
= single_succ_edge (ce
->dest
);
9684 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
9685 gcc_assert (inner_loop_phi
!= NULL
);
9686 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
9687 find_edge (seq_start_bb
, body_bb
), locus
);
9689 if (!single_pred_p (fin_bb
))
9690 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
9692 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
9693 redirect_edge_var_map_clear (re
);
9694 if (single_pred_p (fin_bb
))
9697 psi
= gsi_start_phis (fin_bb
);
9698 if (gsi_end_p (psi
))
9700 remove_phi_node (&psi
, false);
9703 /* Make phi node for trip. */
9704 phi
= create_phi_node (trip_main
, iter_part_bb
);
9705 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
9707 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
9712 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
9713 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
9714 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
9715 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9716 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9717 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
9718 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
9719 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9720 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9724 struct loop
*loop
= body_bb
->loop_father
;
9725 struct loop
*trip_loop
= alloc_loop ();
9726 trip_loop
->header
= iter_part_bb
;
9727 trip_loop
->latch
= trip_update_bb
;
9728 add_loop (trip_loop
, iter_part_bb
->loop_father
);
9730 if (loop
!= entry_bb
->loop_father
)
9732 gcc_assert (loop
->header
== body_bb
);
9733 gcc_assert (loop
->latch
== region
->cont
9734 || single_pred (loop
->latch
) == region
->cont
);
9735 trip_loop
->inner
= loop
;
9739 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9741 loop
= alloc_loop ();
9742 loop
->header
= body_bb
;
9743 if (collapse_bb
== NULL
)
9744 loop
->latch
= cont_bb
;
9745 add_loop (loop
, trip_loop
);
9750 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
9752 for (V = N1; V cond N2; V += STEP) BODY;
9754 where COND is "<" or ">" or "!=", we generate pseudocode
9756 for (ind_var = low; ind_var < high; ind_var++)
9758 V = n1 + (ind_var * STEP)
9763 In the above pseudocode, low and high are function parameters of the
9764 child function. In the function below, we are inserting a temp.
9765 variable that will be making a call to two OMP functions that will not be
9766 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
9767 with _Cilk_for). These functions are replaced with low and high
9768 by the function that handles taskreg. */
9772 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
9774 bool broken_loop
= region
->cont
== NULL
;
9775 basic_block entry_bb
= region
->entry
;
9776 basic_block cont_bb
= region
->cont
;
9778 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9779 gcc_assert (broken_loop
9780 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
9781 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
9782 basic_block l1_bb
, l2_bb
;
9786 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
9787 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9788 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
9789 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9793 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
9794 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
9795 l2_bb
= single_succ (l1_bb
);
9797 basic_block exit_bb
= region
->exit
;
9798 basic_block l2_dom_bb
= NULL
;
9800 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
9802 /* Below statements until the "tree high_val = ..." are pseudo statements
9803 used to pass information to be used by expand_omp_taskreg.
9804 low_val and high_val will be replaced by the __low and __high
9805 parameter from the child function.
9807 The call_exprs part is a place-holder, it is mainly used
9808 to distinctly identify to the top-level part that this is
9809 where we should put low and high (reasoning given in header
9813 = gimple_omp_parallel_child_fn (
9814 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
9815 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
9816 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
9818 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
9820 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
9823 gcc_assert (low_val
&& high_val
);
9825 tree type
= TREE_TYPE (low_val
);
9826 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
9827 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9829 /* Not needed in SSA form right now. */
9830 gcc_assert (!gimple_in_ssa_p (cfun
));
9831 if (l2_dom_bb
== NULL
)
9837 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
9839 /* Replace the GIMPLE_OMP_FOR statement. */
9840 gsi_replace (&gsi
, stmt
, true);
9844 /* Code to control the increment goes in the CONT_BB. */
9845 gsi
= gsi_last_bb (cont_bb
);
9846 stmt
= gsi_stmt (gsi
);
9847 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
9848 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
9849 build_one_cst (type
));
9851 /* Replace GIMPLE_OMP_CONTINUE. */
9852 gsi_replace (&gsi
, stmt
, true);
9855 /* Emit the condition in L1_BB. */
9856 gsi
= gsi_after_labels (l1_bb
);
9857 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
9858 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
9860 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
9861 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
9862 fd
->loop
.n1
, fold_convert (sizetype
, t
));
9864 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
9865 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
9866 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
9867 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
9869 /* The condition is always '<' since the runtime will fill in the low
9871 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
9872 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
9874 /* Remove GIMPLE_OMP_RETURN. */
9875 gsi
= gsi_last_bb (exit_bb
);
9876 gsi_remove (&gsi
, true);
9878 /* Connect the new blocks. */
9879 remove_edge (FALLTHRU_EDGE (entry_bb
));
9884 remove_edge (BRANCH_EDGE (entry_bb
));
9885 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
9887 e
= BRANCH_EDGE (l1_bb
);
9888 ne
= FALLTHRU_EDGE (l1_bb
);
9889 e
->flags
= EDGE_TRUE_VALUE
;
9893 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
9895 ne
= single_succ_edge (l1_bb
);
9896 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9899 ne
->flags
= EDGE_FALSE_VALUE
;
9900 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9901 ne
->probability
= REG_BR_PROB_BASE
/ 8;
9903 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
9904 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
9905 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
9909 struct loop
*loop
= alloc_loop ();
9910 loop
->header
= l1_bb
;
9911 loop
->latch
= cont_bb
;
9912 add_loop (loop
, l1_bb
->loop_father
);
9913 loop
->safelen
= INT_MAX
;
9916 /* Pick the correct library function based on the precision of the
9917 induction variable type. */
9918 tree lib_fun
= NULL_TREE
;
9919 if (TYPE_PRECISION (type
) == 32)
9920 lib_fun
= cilk_for_32_fndecl
;
9921 else if (TYPE_PRECISION (type
) == 64)
9922 lib_fun
= cilk_for_64_fndecl
;
9926 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
9928 /* WS_ARGS contains the library function flavor to call:
9929 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
9930 user-defined grain value. If the user does not define one, then zero
9931 is passed in by the parser. */
9932 vec_alloc (region
->ws_args
, 2);
9933 region
->ws_args
->quick_push (lib_fun
);
9934 region
->ws_args
->quick_push (fd
->chunk_size
);
9937 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
9938 loop. Given parameters:
9940 for (V = N1; V cond N2; V += STEP) BODY;
9942 where COND is "<" or ">", we generate pseudocode
9950 if (V cond N2) goto L0; else goto L2;
9953 For collapsed loops, given parameters:
9955 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
9956 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
9957 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
9960 we generate pseudocode
9966 count3 = (adj + N32 - N31) / STEP3;
9971 count2 = (adj + N22 - N21) / STEP2;
9976 count1 = (adj + N12 - N11) / STEP1;
9977 count = count1 * count2 * count3;
9987 V2 += (V3 cond3 N32) ? 0 : STEP2;
9988 V3 = (V3 cond3 N32) ? V3 : N31;
9989 V1 += (V2 cond2 N22) ? 0 : STEP1;
9990 V2 = (V2 cond2 N22) ? V2 : N21;
9992 if (V < count) goto L0; else goto L2;
9998 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10001 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10002 gimple_stmt_iterator gsi
;
10005 bool broken_loop
= region
->cont
== NULL
;
10007 tree
*counts
= NULL
;
10009 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10010 OMP_CLAUSE_SAFELEN
);
10011 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10012 OMP_CLAUSE__SIMDUID_
);
10015 type
= TREE_TYPE (fd
->loop
.v
);
10016 entry_bb
= region
->entry
;
10017 cont_bb
= region
->cont
;
10018 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10019 gcc_assert (broken_loop
10020 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10021 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10024 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10025 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10026 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10027 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10031 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10032 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10033 l2_bb
= single_succ (l1_bb
);
10035 exit_bb
= region
->exit
;
10038 gsi
= gsi_last_bb (entry_bb
);
10040 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10041 /* Not needed in SSA form right now. */
10042 gcc_assert (!gimple_in_ssa_p (cfun
));
10043 if (fd
->collapse
> 1)
10045 int first_zero_iter
= -1, dummy
= -1;
10046 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10048 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10049 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10050 zero_iter_bb
, first_zero_iter
,
10051 dummy_bb
, dummy
, l2_dom_bb
);
10053 if (l2_dom_bb
== NULL
)
10058 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10060 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10061 OMP_CLAUSE__LOOPTEMP_
);
10062 gcc_assert (innerc
);
10063 n1
= OMP_CLAUSE_DECL (innerc
);
10064 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10065 OMP_CLAUSE__LOOPTEMP_
);
10066 gcc_assert (innerc
);
10067 n2
= OMP_CLAUSE_DECL (innerc
);
10068 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10069 fold_convert (type
, n1
));
10070 if (fd
->collapse
> 1)
10073 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10079 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10080 fold_convert (type
, fd
->loop
.n1
));
10081 if (fd
->collapse
> 1)
10082 for (i
= 0; i
< fd
->collapse
; i
++)
10084 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10085 if (POINTER_TYPE_P (itype
))
10086 itype
= signed_type_for (itype
);
10087 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10088 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10092 /* Remove the GIMPLE_OMP_FOR statement. */
10093 gsi_remove (&gsi
, true);
10097 /* Code to control the increment goes in the CONT_BB. */
10098 gsi
= gsi_last_bb (cont_bb
);
10099 stmt
= gsi_stmt (gsi
);
10100 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10102 if (POINTER_TYPE_P (type
))
10103 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10105 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10106 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10108 if (fd
->collapse
> 1)
10110 i
= fd
->collapse
- 1;
10111 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10113 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10114 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10118 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10119 fd
->loops
[i
].step
);
10120 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10121 fd
->loops
[i
].v
, t
);
10123 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10125 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10127 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10128 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10129 if (POINTER_TYPE_P (itype2
))
10130 itype2
= signed_type_for (itype2
);
10131 t
= build3 (COND_EXPR
, itype2
,
10132 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10134 fold_convert (itype
, fd
->loops
[i
].n2
)),
10135 build_int_cst (itype2
, 0),
10136 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10137 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10138 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10140 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10141 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10143 t
= build3 (COND_EXPR
, itype
,
10144 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10146 fold_convert (itype
, fd
->loops
[i
].n2
)),
10148 fold_convert (itype
, fd
->loops
[i
].n1
));
10149 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10153 /* Remove GIMPLE_OMP_CONTINUE. */
10154 gsi_remove (&gsi
, true);
10157 /* Emit the condition in L1_BB. */
10158 gsi
= gsi_start_bb (l1_bb
);
10160 t
= fold_convert (type
, n2
);
10161 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10162 false, GSI_CONTINUE_LINKING
);
10163 tree v
= fd
->loop
.v
;
10164 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10165 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10166 false, GSI_CONTINUE_LINKING
);
10167 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10168 cond_stmt
= gimple_build_cond_empty (t
);
10169 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10170 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10172 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10175 gsi
= gsi_for_stmt (cond_stmt
);
10176 gimple_regimplify_operands (cond_stmt
, &gsi
);
10179 /* Remove GIMPLE_OMP_RETURN. */
10180 gsi
= gsi_last_bb (exit_bb
);
10181 gsi_remove (&gsi
, true);
10183 /* Connect the new blocks. */
10184 remove_edge (FALLTHRU_EDGE (entry_bb
));
10188 remove_edge (BRANCH_EDGE (entry_bb
));
10189 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10191 e
= BRANCH_EDGE (l1_bb
);
10192 ne
= FALLTHRU_EDGE (l1_bb
);
10193 e
->flags
= EDGE_TRUE_VALUE
;
10197 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10199 ne
= single_succ_edge (l1_bb
);
10200 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10203 ne
->flags
= EDGE_FALSE_VALUE
;
10204 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10205 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10207 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10208 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10209 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10213 struct loop
*loop
= alloc_loop ();
10214 loop
->header
= l1_bb
;
10215 loop
->latch
= cont_bb
;
10216 add_loop (loop
, l1_bb
->loop_father
);
10217 if (safelen
== NULL_TREE
)
10218 loop
->safelen
= INT_MAX
;
10221 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10222 if (TREE_CODE (safelen
) != INTEGER_CST
)
10224 else if (!tree_fits_uhwi_p (safelen
)
10225 || tree_to_uhwi (safelen
) > INT_MAX
)
10226 loop
->safelen
= INT_MAX
;
10228 loop
->safelen
= tree_to_uhwi (safelen
);
10229 if (loop
->safelen
== 1)
10234 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10235 cfun
->has_simduid_loops
= true;
10237 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10239 if ((flag_tree_loop_vectorize
10240 || (!global_options_set
.x_flag_tree_loop_vectorize
10241 && !global_options_set
.x_flag_tree_vectorize
))
10242 && flag_tree_loop_optimize
10243 && loop
->safelen
> 1)
10245 loop
->force_vectorize
= true;
10246 cfun
->has_force_vectorize_loops
= true;
10250 cfun
->has_simduid_loops
= true;
10253 /* Taskloop construct is represented after gimplification with
10254 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10255 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10256 which should just compute all the needed loop temporaries
10257 for GIMPLE_OMP_TASK. */
10260 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10261 struct omp_for_data
*fd
,
10262 gimple
*inner_stmt
)
10264 tree type
, bias
= NULL_TREE
;
10265 basic_block entry_bb
, cont_bb
, exit_bb
;
10266 gimple_stmt_iterator gsi
;
10267 gassign
*assign_stmt
;
10268 tree
*counts
= NULL
;
10271 gcc_assert (inner_stmt
);
10272 gcc_assert (region
->cont
);
10273 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10274 && gimple_omp_task_taskloop_p (inner_stmt
));
10275 type
= TREE_TYPE (fd
->loop
.v
);
10277 /* See if we need to bias by LLONG_MIN. */
10278 if (fd
->iter_type
== long_long_unsigned_type_node
10279 && TREE_CODE (type
) == INTEGER_TYPE
10280 && !TYPE_UNSIGNED (type
))
10284 if (fd
->loop
.cond_code
== LT_EXPR
)
10287 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10291 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10294 if (TREE_CODE (n1
) != INTEGER_CST
10295 || TREE_CODE (n2
) != INTEGER_CST
10296 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10297 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10300 entry_bb
= region
->entry
;
10301 cont_bb
= region
->cont
;
10302 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10303 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10304 exit_bb
= region
->exit
;
10306 gsi
= gsi_last_bb (entry_bb
);
10307 gimple
*for_stmt
= gsi_stmt (gsi
);
10308 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10309 if (fd
->collapse
> 1)
10311 int first_zero_iter
= -1, dummy
= -1;
10312 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10314 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10315 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10316 zero_iter_bb
, first_zero_iter
,
10317 dummy_bb
, dummy
, l2_dom_bb
);
10321 /* Some counts[i] vars might be uninitialized if
10322 some loop has zero iterations. But the body shouldn't
10323 be executed in that case, so just avoid uninit warnings. */
10324 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10325 if (SSA_VAR_P (counts
[i
]))
10326 TREE_NO_WARNING (counts
[i
]) = 1;
10328 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10329 entry_bb
= e
->dest
;
10330 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10331 gsi
= gsi_last_bb (entry_bb
);
10332 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10333 get_immediate_dominator (CDI_DOMINATORS
,
10341 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10342 && TYPE_PRECISION (TREE_TYPE (t0
))
10343 != TYPE_PRECISION (fd
->iter_type
))
10345 /* Avoid casting pointers to integer of a different size. */
10346 tree itype
= signed_type_for (type
);
10347 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10348 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10352 t1
= fold_convert (fd
->iter_type
, t1
);
10353 t0
= fold_convert (fd
->iter_type
, t0
);
10357 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10358 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10361 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10362 OMP_CLAUSE__LOOPTEMP_
);
10363 gcc_assert (innerc
);
10364 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10365 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10366 gcc_assert (innerc
);
10367 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10368 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10370 gcc_assert (innerc
);
10371 for (i
= 1; i
< fd
->collapse
; i
++)
10373 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10374 OMP_CLAUSE__LOOPTEMP_
);
10375 gcc_assert (innerc
);
10377 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10378 OMP_CLAUSE__LOOPTEMP_
);
10381 /* If needed (inner taskloop has lastprivate clause), propagate
10382 down the total number of iterations. */
10383 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10385 GSI_CONTINUE_LINKING
);
10386 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10387 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10391 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10392 GSI_CONTINUE_LINKING
);
10393 assign_stmt
= gimple_build_assign (startvar
, t0
);
10394 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10396 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10397 GSI_CONTINUE_LINKING
);
10398 assign_stmt
= gimple_build_assign (endvar
, t1
);
10399 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10400 if (fd
->collapse
> 1)
10401 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10403 /* Remove the GIMPLE_OMP_FOR statement. */
10404 gsi
= gsi_for_stmt (for_stmt
);
10405 gsi_remove (&gsi
, true);
10407 gsi
= gsi_last_bb (cont_bb
);
10408 gsi_remove (&gsi
, true);
10410 gsi
= gsi_last_bb (exit_bb
);
10411 gsi_remove (&gsi
, true);
10413 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10414 remove_edge (BRANCH_EDGE (entry_bb
));
10415 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10416 remove_edge (BRANCH_EDGE (cont_bb
));
10417 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10418 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10419 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10422 /* Taskloop construct is represented after gimplification with
10423 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10424 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10425 GOMP_taskloop{,_ull} function arranges for each task to be given just
10426 a single range of iterations. */
10429 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10430 struct omp_for_data
*fd
,
10431 gimple
*inner_stmt
)
10433 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10434 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10435 basic_block fin_bb
;
10436 gimple_stmt_iterator gsi
;
10438 bool broken_loop
= region
->cont
== NULL
;
10439 tree
*counts
= NULL
;
10442 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10443 if (POINTER_TYPE_P (type
))
10444 itype
= signed_type_for (type
);
10446 /* See if we need to bias by LLONG_MIN. */
10447 if (fd
->iter_type
== long_long_unsigned_type_node
10448 && TREE_CODE (type
) == INTEGER_TYPE
10449 && !TYPE_UNSIGNED (type
))
10453 if (fd
->loop
.cond_code
== LT_EXPR
)
10456 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10460 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10463 if (TREE_CODE (n1
) != INTEGER_CST
10464 || TREE_CODE (n2
) != INTEGER_CST
10465 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10466 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10469 entry_bb
= region
->entry
;
10470 cont_bb
= region
->cont
;
10471 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10472 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10473 gcc_assert (broken_loop
10474 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10475 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10478 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10479 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10481 exit_bb
= region
->exit
;
10483 /* Iteration space partitioning goes in ENTRY_BB. */
10484 gsi
= gsi_last_bb (entry_bb
);
10485 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10487 if (fd
->collapse
> 1)
10489 int first_zero_iter
= -1, dummy
= -1;
10490 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10492 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10493 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10494 fin_bb
, first_zero_iter
,
10495 dummy_bb
, dummy
, l2_dom_bb
);
10499 t
= integer_one_node
;
10501 step
= fd
->loop
.step
;
10502 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10503 OMP_CLAUSE__LOOPTEMP_
);
10504 gcc_assert (innerc
);
10505 n1
= OMP_CLAUSE_DECL (innerc
);
10506 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10507 gcc_assert (innerc
);
10508 n2
= OMP_CLAUSE_DECL (innerc
);
10511 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10512 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10514 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10515 true, NULL_TREE
, true, GSI_SAME_STMT
);
10516 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10517 true, NULL_TREE
, true, GSI_SAME_STMT
);
10518 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10519 true, NULL_TREE
, true, GSI_SAME_STMT
);
10521 tree startvar
= fd
->loop
.v
;
10522 tree endvar
= NULL_TREE
;
10524 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10526 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10527 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10528 gcc_assert (innerc
);
10529 startvar
= OMP_CLAUSE_DECL (innerc
);
10530 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10531 OMP_CLAUSE__LOOPTEMP_
);
10532 gcc_assert (innerc
);
10533 endvar
= OMP_CLAUSE_DECL (innerc
);
10535 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10536 t
= force_gimple_operand_gsi (&gsi
, t
,
10538 && TREE_ADDRESSABLE (startvar
),
10539 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10540 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10541 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10543 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10544 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10545 false, GSI_CONTINUE_LINKING
);
10548 assign_stmt
= gimple_build_assign (endvar
, e
);
10549 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10550 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10551 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10553 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10554 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10556 if (fd
->collapse
> 1)
10557 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10561 /* The code controlling the sequential loop replaces the
10562 GIMPLE_OMP_CONTINUE. */
10563 gsi
= gsi_last_bb (cont_bb
);
10564 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10565 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10566 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10567 vback
= gimple_omp_continue_control_def (cont_stmt
);
10569 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10571 if (POINTER_TYPE_P (type
))
10572 t
= fold_build_pointer_plus (vmain
, step
);
10574 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10575 t
= force_gimple_operand_gsi (&gsi
, t
,
10577 && TREE_ADDRESSABLE (vback
),
10578 NULL_TREE
, true, GSI_SAME_STMT
);
10579 assign_stmt
= gimple_build_assign (vback
, t
);
10580 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10582 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10583 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10585 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10588 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10589 gsi_remove (&gsi
, true);
10591 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10592 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10595 /* Remove the GIMPLE_OMP_FOR statement. */
10596 gsi
= gsi_for_stmt (fd
->for_stmt
);
10597 gsi_remove (&gsi
, true);
10599 /* Remove the GIMPLE_OMP_RETURN statement. */
10600 gsi
= gsi_last_bb (exit_bb
);
10601 gsi_remove (&gsi
, true);
10603 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10605 remove_edge (BRANCH_EDGE (entry_bb
));
10608 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
10609 region
->outer
->cont
= NULL
;
10612 /* Connect all the blocks. */
10615 ep
= find_edge (cont_bb
, body_bb
);
10616 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10621 else if (fd
->collapse
> 1)
10624 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10627 ep
->flags
= EDGE_TRUE_VALUE
;
10628 find_edge (cont_bb
, fin_bb
)->flags
10629 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10632 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10633 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10635 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10636 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10638 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
10640 struct loop
*loop
= alloc_loop ();
10641 loop
->header
= body_bb
;
10642 if (collapse_bb
== NULL
)
10643 loop
->latch
= cont_bb
;
10644 add_loop (loop
, body_bb
->loop_father
);
10648 /* A subroutine of expand_omp_for. Generate code for an OpenACC
10649 partitioned loop. The lowering here is abstracted, in that the
10650 loop parameters are passed through internal functions, which are
10651 further lowered by oacc_device_lower, once we get to the target
10652 compiler. The loop is of the form:
10654 for (V = B; V LTGT E; V += S) {BODY}
10656 where LTGT is < or >. We may have a specified chunking size, CHUNKING
10657 (constant 0 for no chunking) and we will have a GWV partitioning
10658 mask, specifying dimensions over which the loop is to be
10659 partitioned (see note below). We generate code that looks like:
10661 <entry_bb> [incoming FALL->body, BRANCH->exit]
10662 typedef signedintify (typeof (V)) T; // underlying signed integral type
10665 T DIR = LTGT == '<' ? +1 : -1;
10666 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
10667 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
10669 <head_bb> [created by splitting end of entry_bb]
10670 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
10671 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
10672 if (!(offset LTGT bound)) goto bottom_bb;
10674 <body_bb> [incoming]
10678 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
10680 if (offset LTGT bound) goto body_bb; [*]
10682 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
10684 if (chunk < chunk_max) goto head_bb;
10686 <exit_bb> [incoming]
10687 V = B + ((range -/+ 1) / S +/- 1) * S [*]
10689 [*] Needed if V live at end of loop
10691 Note: CHUNKING & GWV mask are specified explicitly here. This is a
10692 transition, and will be specified by a more general mechanism shortly.
10696 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10698 tree v
= fd
->loop
.v
;
10699 enum tree_code cond_code
= fd
->loop
.cond_code
;
10700 enum tree_code plus_code
= PLUS_EXPR
;
10702 tree chunk_size
= integer_minus_one_node
;
10703 tree gwv
= integer_zero_node
;
10704 tree iter_type
= TREE_TYPE (v
);
10705 tree diff_type
= iter_type
;
10706 tree plus_type
= iter_type
;
10707 struct oacc_collapse
*counts
= NULL
;
10709 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
10710 == GF_OMP_FOR_KIND_OACC_LOOP
);
10711 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
10712 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
10714 if (POINTER_TYPE_P (iter_type
))
10716 plus_code
= POINTER_PLUS_EXPR
;
10717 plus_type
= sizetype
;
10719 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
10720 diff_type
= signed_type_for (diff_type
);
10722 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
10723 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
10724 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
10725 basic_block bottom_bb
= NULL
;
10727 /* entry_bb has two sucessors; the branch edge is to the exit
10728 block, fallthrough edge to body. */
10729 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
10730 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
10732 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
10733 body_bb, or to a block whose only successor is the body_bb. Its
10734 fallthrough successor is the final block (same as the branch
10735 successor of the entry_bb). */
10738 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10739 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
10741 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
10742 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
10745 gcc_assert (!gimple_in_ssa_p (cfun
));
10747 /* The exit block only has entry_bb and cont_bb as predecessors. */
10748 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
10751 tree chunk_max
= NULL_TREE
;
10752 tree bound
, offset
;
10753 tree step
= create_tmp_var (diff_type
, ".step");
10754 bool up
= cond_code
== LT_EXPR
;
10755 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
10756 bool chunking
= !gimple_in_ssa_p (cfun
);;
10759 /* SSA instances. */
10760 tree offset_incr
= NULL_TREE
;
10761 tree offset_init
= NULL_TREE
;
10763 gimple_stmt_iterator gsi
;
10769 edge split
, be
, fte
;
10771 /* Split the end of entry_bb to create head_bb. */
10772 split
= split_block (entry_bb
, last_stmt (entry_bb
));
10773 basic_block head_bb
= split
->dest
;
10774 entry_bb
= split
->src
;
10776 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
10777 gsi
= gsi_last_bb (entry_bb
);
10778 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
10779 loc
= gimple_location (for_stmt
);
10781 if (gimple_in_ssa_p (cfun
))
10783 offset_init
= gimple_omp_for_index (for_stmt
, 0);
10784 gcc_assert (integer_zerop (fd
->loop
.n1
));
10785 /* The SSA parallelizer does gang parallelism. */
10786 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
10789 if (fd
->collapse
> 1)
10791 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
10792 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
10793 TREE_TYPE (fd
->loop
.n2
));
10795 if (SSA_VAR_P (fd
->loop
.n2
))
10797 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
10798 true, GSI_SAME_STMT
);
10799 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
10800 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
10805 tree b
= fd
->loop
.n1
;
10806 tree e
= fd
->loop
.n2
;
10807 tree s
= fd
->loop
.step
;
10809 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10810 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10812 /* Convert the step, avoiding possible unsigned->signed overflow. */
10813 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
10815 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
10816 s
= fold_convert (diff_type
, s
);
10818 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
10819 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10822 chunk_size
= integer_zero_node
;
10823 expr
= fold_convert (diff_type
, chunk_size
);
10824 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
10825 NULL_TREE
, true, GSI_SAME_STMT
);
10826 /* Determine the range, avoiding possible unsigned->signed overflow. */
10827 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
10828 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
10829 fold_convert (plus_type
, negating
? b
: e
),
10830 fold_convert (plus_type
, negating
? e
: b
));
10831 expr
= fold_convert (diff_type
, expr
);
10833 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
10834 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
10835 NULL_TREE
, true, GSI_SAME_STMT
);
10837 chunk_no
= build_int_cst (diff_type
, 0);
10840 gcc_assert (!gimple_in_ssa_p (cfun
));
10843 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
10844 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
10846 ass
= gimple_build_assign (chunk_no
, expr
);
10847 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
10849 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
10850 build_int_cst (integer_type_node
,
10851 IFN_GOACC_LOOP_CHUNKS
),
10852 dir
, range
, s
, chunk_size
, gwv
);
10853 gimple_call_set_lhs (call
, chunk_max
);
10854 gimple_set_location (call
, loc
);
10855 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
10858 chunk_size
= chunk_no
;
10860 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
10861 build_int_cst (integer_type_node
,
10862 IFN_GOACC_LOOP_STEP
),
10863 dir
, range
, s
, chunk_size
, gwv
);
10864 gimple_call_set_lhs (call
, step
);
10865 gimple_set_location (call
, loc
);
10866 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
10868 /* Remove the GIMPLE_OMP_FOR. */
10869 gsi_remove (&gsi
, true);
10871 /* Fixup edges from head_bb */
10872 be
= BRANCH_EDGE (head_bb
);
10873 fte
= FALLTHRU_EDGE (head_bb
);
10874 be
->flags
|= EDGE_FALSE_VALUE
;
10875 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
10877 basic_block body_bb
= fte
->dest
;
10879 if (gimple_in_ssa_p (cfun
))
10881 gsi
= gsi_last_bb (cont_bb
);
10882 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10884 offset
= gimple_omp_continue_control_use (cont_stmt
);
10885 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
10889 offset
= create_tmp_var (diff_type
, ".offset");
10890 offset_init
= offset_incr
= offset
;
10892 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
10894 /* Loop offset & bound go into head_bb. */
10895 gsi
= gsi_start_bb (head_bb
);
10897 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
10898 build_int_cst (integer_type_node
,
10899 IFN_GOACC_LOOP_OFFSET
),
10901 chunk_size
, gwv
, chunk_no
);
10902 gimple_call_set_lhs (call
, offset_init
);
10903 gimple_set_location (call
, loc
);
10904 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
10906 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
10907 build_int_cst (integer_type_node
,
10908 IFN_GOACC_LOOP_BOUND
),
10910 chunk_size
, gwv
, offset_init
);
10911 gimple_call_set_lhs (call
, bound
);
10912 gimple_set_location (call
, loc
);
10913 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
10915 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
10916 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
10917 GSI_CONTINUE_LINKING
);
10919 /* V assignment goes into body_bb. */
10920 if (!gimple_in_ssa_p (cfun
))
10922 gsi
= gsi_start_bb (body_bb
);
10924 expr
= build2 (plus_code
, iter_type
, b
,
10925 fold_convert (plus_type
, offset
));
10926 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
10927 true, GSI_SAME_STMT
);
10928 ass
= gimple_build_assign (v
, expr
);
10929 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
10930 if (fd
->collapse
> 1)
10931 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
10934 /* Loop increment goes into cont_bb. If this is not a loop, we
10935 will have spawned threads as if it was, and each one will
10936 execute one iteration. The specification is not explicit about
10937 whether such constructs are ill-formed or not, and they can
10938 occur, especially when noreturn routines are involved. */
10941 gsi
= gsi_last_bb (cont_bb
);
10942 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10943 loc
= gimple_location (cont_stmt
);
10945 /* Increment offset. */
10946 if (gimple_in_ssa_p (cfun
))
10947 expr
= build2 (plus_code
, iter_type
, offset
,
10948 fold_convert (plus_type
, step
));
10950 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
10951 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
10952 true, GSI_SAME_STMT
);
10953 ass
= gimple_build_assign (offset_incr
, expr
);
10954 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
10955 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
10956 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
10958 /* Remove the GIMPLE_OMP_CONTINUE. */
10959 gsi_remove (&gsi
, true);
10961 /* Fixup edges from cont_bb */
10962 be
= BRANCH_EDGE (cont_bb
);
10963 fte
= FALLTHRU_EDGE (cont_bb
);
10964 be
->flags
|= EDGE_TRUE_VALUE
;
10965 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
10969 /* Split the beginning of exit_bb to make bottom_bb. We
10970 need to insert a nop at the start, because splitting is
10971 after a stmt, not before. */
10972 gsi
= gsi_start_bb (exit_bb
);
10973 stmt
= gimple_build_nop ();
10974 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10975 split
= split_block (exit_bb
, stmt
);
10976 bottom_bb
= split
->src
;
10977 exit_bb
= split
->dest
;
10978 gsi
= gsi_last_bb (bottom_bb
);
10980 /* Chunk increment and test goes into bottom_bb. */
10981 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
10982 build_int_cst (diff_type
, 1));
10983 ass
= gimple_build_assign (chunk_no
, expr
);
10984 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
10986 /* Chunk test at end of bottom_bb. */
10987 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
10988 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
10989 GSI_CONTINUE_LINKING
);
10991 /* Fixup edges from bottom_bb. */
10992 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
10993 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
10997 gsi
= gsi_last_bb (exit_bb
);
10998 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
10999 loc
= gimple_location (gsi_stmt (gsi
));
11001 if (!gimple_in_ssa_p (cfun
))
11003 /* Insert the final value of V, in case it is live. This is the
11004 value for the only thread that survives past the join. */
11005 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11006 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11007 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11008 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11009 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11010 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11011 true, GSI_SAME_STMT
);
11012 ass
= gimple_build_assign (v
, expr
);
11013 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11016 /* Remove the OMP_RETURN. */
11017 gsi_remove (&gsi
, true);
11021 /* We now have one or two nested loops. Update the loop
11023 struct loop
*parent
= entry_bb
->loop_father
;
11024 struct loop
*body
= body_bb
->loop_father
;
11028 struct loop
*chunk_loop
= alloc_loop ();
11029 chunk_loop
->header
= head_bb
;
11030 chunk_loop
->latch
= bottom_bb
;
11031 add_loop (chunk_loop
, parent
);
11032 parent
= chunk_loop
;
11034 else if (parent
!= body
)
11036 gcc_assert (body
->header
== body_bb
);
11037 gcc_assert (body
->latch
== cont_bb
11038 || single_pred (body
->latch
) == cont_bb
);
11044 struct loop
*body_loop
= alloc_loop ();
11045 body_loop
->header
= body_bb
;
11046 body_loop
->latch
= cont_bb
;
11047 add_loop (body_loop
, parent
);
11052 /* Expand the OMP loop defined by REGION. */
11055 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11057 struct omp_for_data fd
;
11058 struct omp_for_data_loop
*loops
;
11061 = (struct omp_for_data_loop
*)
11062 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11063 * sizeof (struct omp_for_data_loop
));
11064 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11066 region
->sched_kind
= fd
.sched_kind
;
11068 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11069 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11070 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11073 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11074 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11075 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11078 /* If there isn't a continue then this is a degerate case where
11079 the introduction of abnormal edges during lowering will prevent
11080 original loops from being detected. Fix that up. */
11081 loops_state_set (LOOPS_NEED_FIXUP
);
11083 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11084 expand_omp_simd (region
, &fd
);
11085 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11086 expand_cilk_for (region
, &fd
);
11087 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11089 gcc_assert (!inner_stmt
);
11090 expand_oacc_for (region
, &fd
);
11092 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11094 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11095 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11097 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11099 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11100 && !fd
.have_ordered
)
11102 if (fd
.chunk_size
== NULL
)
11103 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11105 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11109 int fn_index
, start_ix
, next_ix
;
11111 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11112 == GF_OMP_FOR_KIND_FOR
);
11113 if (fd
.chunk_size
== NULL
11114 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11115 fd
.chunk_size
= integer_zero_node
;
11116 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11117 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
11118 ? 3 : fd
.sched_kind
;
11120 fn_index
+= fd
.have_ordered
* 4;
11122 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11124 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11125 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11126 if (fd
.iter_type
== long_long_unsigned_type_node
)
11128 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11129 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11130 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11131 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11133 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11134 (enum built_in_function
) next_ix
, inner_stmt
);
11137 if (gimple_in_ssa_p (cfun
))
11138 update_ssa (TODO_update_ssa_only_virtuals
);
11142 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11144 v = GOMP_sections_start (n);
11161 v = GOMP_sections_next ();
11166 If this is a combined parallel sections, replace the call to
11167 GOMP_sections_start with call to GOMP_sections_next. */
11170 expand_omp_sections (struct omp_region
*region
)
11172 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11174 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11175 gimple_stmt_iterator si
, switch_si
;
11176 gomp_sections
*sections_stmt
;
11178 gomp_continue
*cont
;
11181 struct omp_region
*inner
;
11183 bool exit_reachable
= region
->cont
!= NULL
;
11185 gcc_assert (region
->exit
!= NULL
);
11186 entry_bb
= region
->entry
;
11187 l0_bb
= single_succ (entry_bb
);
11188 l1_bb
= region
->cont
;
11189 l2_bb
= region
->exit
;
11190 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11191 l2
= gimple_block_label (l2_bb
);
11194 /* This can happen if there are reductions. */
11195 len
= EDGE_COUNT (l0_bb
->succs
);
11196 gcc_assert (len
> 0);
11197 e
= EDGE_SUCC (l0_bb
, len
- 1);
11198 si
= gsi_last_bb (e
->dest
);
11201 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11202 l2
= gimple_block_label (e
->dest
);
11204 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11206 si
= gsi_last_bb (e
->dest
);
11208 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11210 l2
= gimple_block_label (e
->dest
);
11215 if (exit_reachable
)
11216 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11218 default_bb
= create_empty_bb (l0_bb
);
11220 /* We will build a switch() with enough cases for all the
11221 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11222 and a default case to abort if something goes wrong. */
11223 len
= EDGE_COUNT (l0_bb
->succs
);
11225 /* Use vec::quick_push on label_vec throughout, since we know the size
11227 auto_vec
<tree
> label_vec (len
);
11229 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11230 GIMPLE_OMP_SECTIONS statement. */
11231 si
= gsi_last_bb (entry_bb
);
11232 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11233 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11234 vin
= gimple_omp_sections_control (sections_stmt
);
11235 if (!is_combined_parallel (region
))
11237 /* If we are not inside a combined parallel+sections region,
11238 call GOMP_sections_start. */
11239 t
= build_int_cst (unsigned_type_node
, len
- 1);
11240 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11241 stmt
= gimple_build_call (u
, 1, t
);
11245 /* Otherwise, call GOMP_sections_next. */
11246 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11247 stmt
= gimple_build_call (u
, 0);
11249 gimple_call_set_lhs (stmt
, vin
);
11250 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11251 gsi_remove (&si
, true);
11253 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11255 switch_si
= gsi_last_bb (l0_bb
);
11256 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11257 if (exit_reachable
)
11259 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11260 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11261 vmain
= gimple_omp_continue_control_use (cont
);
11262 vnext
= gimple_omp_continue_control_def (cont
);
11270 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11271 label_vec
.quick_push (t
);
11274 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11275 for (inner
= region
->inner
, casei
= 1;
11277 inner
= inner
->next
, i
++, casei
++)
11279 basic_block s_entry_bb
, s_exit_bb
;
11281 /* Skip optional reduction region. */
11282 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11289 s_entry_bb
= inner
->entry
;
11290 s_exit_bb
= inner
->exit
;
11292 t
= gimple_block_label (s_entry_bb
);
11293 u
= build_int_cst (unsigned_type_node
, casei
);
11294 u
= build_case_label (u
, NULL
, t
);
11295 label_vec
.quick_push (u
);
11297 si
= gsi_last_bb (s_entry_bb
);
11298 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11299 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11300 gsi_remove (&si
, true);
11301 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11303 if (s_exit_bb
== NULL
)
11306 si
= gsi_last_bb (s_exit_bb
);
11307 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11308 gsi_remove (&si
, true);
11310 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11313 /* Error handling code goes in DEFAULT_BB. */
11314 t
= gimple_block_label (default_bb
);
11315 u
= build_case_label (NULL
, NULL
, t
);
11316 make_edge (l0_bb
, default_bb
, 0);
11317 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11319 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11320 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11321 gsi_remove (&switch_si
, true);
11323 si
= gsi_start_bb (default_bb
);
11324 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11325 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11327 if (exit_reachable
)
11331 /* Code to get the next section goes in L1_BB. */
11332 si
= gsi_last_bb (l1_bb
);
11333 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11335 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11336 stmt
= gimple_build_call (bfn_decl
, 0);
11337 gimple_call_set_lhs (stmt
, vnext
);
11338 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11339 gsi_remove (&si
, true);
11341 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11344 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11345 si
= gsi_last_bb (l2_bb
);
11346 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11347 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11348 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11349 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11351 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11352 stmt
= gimple_build_call (t
, 0);
11353 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11354 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11355 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11356 gsi_remove (&si
, true);
11358 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11362 /* Expand code for an OpenMP single directive. We've already expanded
11363 much of the code, here we simply place the GOMP_barrier call. */
11366 expand_omp_single (struct omp_region
*region
)
11368 basic_block entry_bb
, exit_bb
;
11369 gimple_stmt_iterator si
;
11371 entry_bb
= region
->entry
;
11372 exit_bb
= region
->exit
;
11374 si
= gsi_last_bb (entry_bb
);
11375 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11376 gsi_remove (&si
, true);
11377 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11379 si
= gsi_last_bb (exit_bb
);
11380 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11382 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11383 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11385 gsi_remove (&si
, true);
11386 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11390 /* Generic expansion for OpenMP synchronization directives: master,
11391 ordered and critical. All we need to do here is remove the entry
11392 and exit markers for REGION. */
11395 expand_omp_synch (struct omp_region
*region
)
11397 basic_block entry_bb
, exit_bb
;
11398 gimple_stmt_iterator si
;
11400 entry_bb
= region
->entry
;
11401 exit_bb
= region
->exit
;
11403 si
= gsi_last_bb (entry_bb
);
11404 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11405 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11406 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11407 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11408 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11409 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11410 gsi_remove (&si
, true);
11411 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11415 si
= gsi_last_bb (exit_bb
);
11416 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11417 gsi_remove (&si
, true);
11418 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11422 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11423 operation as a normal volatile load. */
11426 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11427 tree loaded_val
, int index
)
11429 enum built_in_function tmpbase
;
11430 gimple_stmt_iterator gsi
;
11431 basic_block store_bb
;
11434 tree decl
, call
, type
, itype
;
11436 gsi
= gsi_last_bb (load_bb
);
11437 stmt
= gsi_stmt (gsi
);
11438 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11439 loc
= gimple_location (stmt
);
11441 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11442 is smaller than word size, then expand_atomic_load assumes that the load
11443 is atomic. We could avoid the builtin entirely in this case. */
11445 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11446 decl
= builtin_decl_explicit (tmpbase
);
11447 if (decl
== NULL_TREE
)
11450 type
= TREE_TYPE (loaded_val
);
11451 itype
= TREE_TYPE (TREE_TYPE (decl
));
11453 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11454 build_int_cst (NULL
,
11455 gimple_omp_atomic_seq_cst_p (stmt
)
11457 : MEMMODEL_RELAXED
));
11458 if (!useless_type_conversion_p (type
, itype
))
11459 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11460 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11462 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11463 gsi_remove (&gsi
, true);
11465 store_bb
= single_succ (load_bb
);
11466 gsi
= gsi_last_bb (store_bb
);
11467 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11468 gsi_remove (&gsi
, true);
11470 if (gimple_in_ssa_p (cfun
))
11471 update_ssa (TODO_update_ssa_no_phi
);
11476 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11477 operation as a normal volatile store. */
11480 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11481 tree loaded_val
, tree stored_val
, int index
)
11483 enum built_in_function tmpbase
;
11484 gimple_stmt_iterator gsi
;
11485 basic_block store_bb
= single_succ (load_bb
);
11488 tree decl
, call
, type
, itype
;
11489 machine_mode imode
;
11492 gsi
= gsi_last_bb (load_bb
);
11493 stmt
= gsi_stmt (gsi
);
11494 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11496 /* If the load value is needed, then this isn't a store but an exchange. */
11497 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11499 gsi
= gsi_last_bb (store_bb
);
11500 stmt
= gsi_stmt (gsi
);
11501 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11502 loc
= gimple_location (stmt
);
11504 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11505 is smaller than word size, then expand_atomic_store assumes that the store
11506 is atomic. We could avoid the builtin entirely in this case. */
11508 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11509 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
11510 decl
= builtin_decl_explicit (tmpbase
);
11511 if (decl
== NULL_TREE
)
11514 type
= TREE_TYPE (stored_val
);
11516 /* Dig out the type of the function's second argument. */
11517 itype
= TREE_TYPE (decl
);
11518 itype
= TYPE_ARG_TYPES (itype
);
11519 itype
= TREE_CHAIN (itype
);
11520 itype
= TREE_VALUE (itype
);
11521 imode
= TYPE_MODE (itype
);
11523 if (exchange
&& !can_atomic_exchange_p (imode
, true))
11526 if (!useless_type_conversion_p (itype
, type
))
11527 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
11528 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
11529 build_int_cst (NULL
,
11530 gimple_omp_atomic_seq_cst_p (stmt
)
11532 : MEMMODEL_RELAXED
));
11535 if (!useless_type_conversion_p (type
, itype
))
11536 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11537 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11540 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11541 gsi_remove (&gsi
, true);
11543 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
11544 gsi
= gsi_last_bb (load_bb
);
11545 gsi_remove (&gsi
, true);
11547 if (gimple_in_ssa_p (cfun
))
11548 update_ssa (TODO_update_ssa_no_phi
);
11553 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11554 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
11555 size of the data type, and thus usable to find the index of the builtin
11556 decl. Returns false if the expression is not of the proper form. */
11559 expand_omp_atomic_fetch_op (basic_block load_bb
,
11560 tree addr
, tree loaded_val
,
11561 tree stored_val
, int index
)
11563 enum built_in_function oldbase
, newbase
, tmpbase
;
11564 tree decl
, itype
, call
;
11566 basic_block store_bb
= single_succ (load_bb
);
11567 gimple_stmt_iterator gsi
;
11570 enum tree_code code
;
11571 bool need_old
, need_new
;
11572 machine_mode imode
;
11575 /* We expect to find the following sequences:
11578 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
11581 val = tmp OP something; (or: something OP tmp)
11582 GIMPLE_OMP_STORE (val)
11584 ???FIXME: Allow a more flexible sequence.
11585 Perhaps use data flow to pick the statements.
11589 gsi
= gsi_after_labels (store_bb
);
11590 stmt
= gsi_stmt (gsi
);
11591 loc
= gimple_location (stmt
);
11592 if (!is_gimple_assign (stmt
))
11595 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
11597 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
11598 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
11599 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
11600 gcc_checking_assert (!need_old
|| !need_new
);
11602 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
11605 /* Check for one of the supported fetch-op operations. */
11606 code
= gimple_assign_rhs_code (stmt
);
11610 case POINTER_PLUS_EXPR
:
11611 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
11612 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
11615 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
11616 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
11619 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
11620 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
11623 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
11624 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
11627 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
11628 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
11634 /* Make sure the expression is of the proper form. */
11635 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
11636 rhs
= gimple_assign_rhs2 (stmt
);
11637 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
11638 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
11639 rhs
= gimple_assign_rhs1 (stmt
);
11643 tmpbase
= ((enum built_in_function
)
11644 ((need_new
? newbase
: oldbase
) + index
+ 1));
11645 decl
= builtin_decl_explicit (tmpbase
);
11646 if (decl
== NULL_TREE
)
11648 itype
= TREE_TYPE (TREE_TYPE (decl
));
11649 imode
= TYPE_MODE (itype
);
11651 /* We could test all of the various optabs involved, but the fact of the
11652 matter is that (with the exception of i486 vs i586 and xadd) all targets
11653 that support any atomic operaton optab also implements compare-and-swap.
11654 Let optabs.c take care of expanding any compare-and-swap loop. */
11655 if (!can_compare_and_swap_p (imode
, true))
11658 gsi
= gsi_last_bb (load_bb
);
11659 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11661 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
11662 It only requires that the operation happen atomically. Thus we can
11663 use the RELAXED memory model. */
11664 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
11665 fold_convert_loc (loc
, itype
, rhs
),
11666 build_int_cst (NULL
,
11667 seq_cst
? MEMMODEL_SEQ_CST
11668 : MEMMODEL_RELAXED
));
11670 if (need_old
|| need_new
)
11672 lhs
= need_old
? loaded_val
: stored_val
;
11673 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
11674 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
11677 call
= fold_convert_loc (loc
, void_type_node
, call
);
11678 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11679 gsi_remove (&gsi
, true);
11681 gsi
= gsi_last_bb (store_bb
);
11682 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11683 gsi_remove (&gsi
, true);
11684 gsi
= gsi_last_bb (store_bb
);
11685 gsi_remove (&gsi
, true);
11687 if (gimple_in_ssa_p (cfun
))
11688 update_ssa (TODO_update_ssa_no_phi
);
11693 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
11697 newval = rhs; // with oldval replacing *addr in rhs
11698 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
11699 if (oldval != newval)
11702 INDEX is log2 of the size of the data type, and thus usable to find the
11703 index of the builtin decl. */
11706 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
11707 tree addr
, tree loaded_val
, tree stored_val
,
11710 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
11711 tree type
, itype
, cmpxchg
, iaddr
;
11712 gimple_stmt_iterator si
;
11713 basic_block loop_header
= single_succ (load_bb
);
11714 gimple
*phi
, *stmt
;
11716 enum built_in_function fncode
;
11718 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
11719 order to use the RELAXED memory model effectively. */
11720 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
11722 cmpxchg
= builtin_decl_explicit (fncode
);
11723 if (cmpxchg
== NULL_TREE
)
11725 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
11726 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
11728 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
11731 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
11732 si
= gsi_last_bb (load_bb
);
11733 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11735 /* For floating-point values, we'll need to view-convert them to integers
11736 so that we can perform the atomic compare and swap. Simplify the
11737 following code by always setting up the "i"ntegral variables. */
11738 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
11742 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
11745 = force_gimple_operand_gsi (&si
,
11746 fold_convert (TREE_TYPE (iaddr
), addr
),
11747 false, NULL_TREE
, true, GSI_SAME_STMT
);
11748 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
11749 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11750 loadedi
= create_tmp_var (itype
);
11751 if (gimple_in_ssa_p (cfun
))
11752 loadedi
= make_ssa_name (loadedi
);
11757 loadedi
= loaded_val
;
11760 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11761 tree loaddecl
= builtin_decl_explicit (fncode
);
11764 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
11765 build_call_expr (loaddecl
, 2, iaddr
,
11766 build_int_cst (NULL_TREE
,
11767 MEMMODEL_RELAXED
)));
11769 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
11770 build_int_cst (TREE_TYPE (iaddr
), 0));
11773 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
11776 /* Move the value to the LOADEDI temporary. */
11777 if (gimple_in_ssa_p (cfun
))
11779 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
11780 phi
= create_phi_node (loadedi
, loop_header
);
11781 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
11785 gsi_insert_before (&si
,
11786 gimple_build_assign (loadedi
, initial
),
11788 if (loadedi
!= loaded_val
)
11790 gimple_stmt_iterator gsi2
;
11793 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
11794 gsi2
= gsi_start_bb (loop_header
);
11795 if (gimple_in_ssa_p (cfun
))
11798 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
11799 true, GSI_SAME_STMT
);
11800 stmt
= gimple_build_assign (loaded_val
, x
);
11801 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
11805 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
11806 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
11807 true, GSI_SAME_STMT
);
11810 gsi_remove (&si
, true);
11812 si
= gsi_last_bb (store_bb
);
11813 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
11816 storedi
= stored_val
;
11819 force_gimple_operand_gsi (&si
,
11820 build1 (VIEW_CONVERT_EXPR
, itype
,
11821 stored_val
), true, NULL_TREE
, true,
11824 /* Build the compare&swap statement. */
11825 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
11826 new_storedi
= force_gimple_operand_gsi (&si
,
11827 fold_convert (TREE_TYPE (loadedi
),
11830 true, GSI_SAME_STMT
);
11832 if (gimple_in_ssa_p (cfun
))
11833 old_vali
= loadedi
;
11836 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
11837 stmt
= gimple_build_assign (old_vali
, loadedi
);
11838 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11840 stmt
= gimple_build_assign (loadedi
, new_storedi
);
11841 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11844 /* Note that we always perform the comparison as an integer, even for
11845 floating point. This allows the atomic operation to properly
11846 succeed even with NaNs and -0.0. */
11847 stmt
= gimple_build_cond_empty
11848 (build2 (NE_EXPR
, boolean_type_node
,
11849 new_storedi
, old_vali
));
11850 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11853 e
= single_succ_edge (store_bb
);
11854 e
->flags
&= ~EDGE_FALLTHRU
;
11855 e
->flags
|= EDGE_FALSE_VALUE
;
11857 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
11859 /* Copy the new value to loadedi (we already did that before the condition
11860 if we are not in SSA). */
11861 if (gimple_in_ssa_p (cfun
))
11863 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
11864 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
11867 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
11868 gsi_remove (&si
, true);
11870 struct loop
*loop
= alloc_loop ();
11871 loop
->header
= loop_header
;
11872 loop
->latch
= store_bb
;
11873 add_loop (loop
, loop_header
->loop_father
);
11875 if (gimple_in_ssa_p (cfun
))
11876 update_ssa (TODO_update_ssa_no_phi
);
11881 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
11883 GOMP_atomic_start ();
11885 GOMP_atomic_end ();
11887 The result is not globally atomic, but works so long as all parallel
11888 references are within #pragma omp atomic directives. According to
11889 responses received from omp@openmp.org, appears to be within spec.
11890 Which makes sense, since that's how several other compilers handle
11891 this situation as well.
11892 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
11893 expanding. STORED_VAL is the operand of the matching
11894 GIMPLE_OMP_ATOMIC_STORE.
11897 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
11898 loaded_val = *addr;
11901 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
11902 *addr = stored_val;
11906 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
11907 tree addr
, tree loaded_val
, tree stored_val
)
11909 gimple_stmt_iterator si
;
11913 si
= gsi_last_bb (load_bb
);
11914 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11916 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
11917 t
= build_call_expr (t
, 0);
11918 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11920 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
11921 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11922 gsi_remove (&si
, true);
11924 si
= gsi_last_bb (store_bb
);
11925 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
11927 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
11929 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11931 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
11932 t
= build_call_expr (t
, 0);
11933 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11934 gsi_remove (&si
, true);
11936 if (gimple_in_ssa_p (cfun
))
11937 update_ssa (TODO_update_ssa_no_phi
);
11941 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
11942 using expand_omp_atomic_fetch_op. If it failed, we try to
11943 call expand_omp_atomic_pipeline, and if it fails too, the
11944 ultimate fallback is wrapping the operation in a mutex
11945 (expand_omp_atomic_mutex). REGION is the atomic region built
11946 by build_omp_regions_1(). */
11949 expand_omp_atomic (struct omp_region
*region
)
11951 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
11952 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
11953 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
11954 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
11955 tree addr
= gimple_omp_atomic_load_rhs (load
);
11956 tree stored_val
= gimple_omp_atomic_store_val (store
);
11957 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
11958 HOST_WIDE_INT index
;
11960 /* Make sure the type is one of the supported sizes. */
11961 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
11962 index
= exact_log2 (index
);
11963 if (index
>= 0 && index
<= 4)
11965 unsigned int align
= TYPE_ALIGN_UNIT (type
);
11967 /* __sync builtins require strict data alignment. */
11968 if (exact_log2 (align
) >= index
)
11971 if (loaded_val
== stored_val
11972 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
11973 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
11974 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
11975 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
11978 /* Atomic store. */
11979 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
11980 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
11981 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
11982 && store_bb
== single_succ (load_bb
)
11983 && first_stmt (store_bb
) == store
11984 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
11985 stored_val
, index
))
11988 /* When possible, use specialized atomic update functions. */
11989 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
11990 && store_bb
== single_succ (load_bb
)
11991 && expand_omp_atomic_fetch_op (load_bb
, addr
,
11992 loaded_val
, stored_val
, index
))
11995 /* If we don't have specialized __sync builtins, try and implement
11996 as a compare and swap loop. */
11997 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
11998 loaded_val
, stored_val
, index
))
12003 /* The ultimate fallback is wrapping the operation in a mutex. */
12004 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12008 /* Encode an oacc launc argument. This matches the GOMP_LAUNCH_PACK
12009 macro on gomp-constants.h. We do not check for overflow. */
12012 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12016 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12019 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12020 device
, build_int_cst (unsigned_type_node
,
12021 GOMP_LAUNCH_DEVICE_SHIFT
));
12022 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12027 /* Look for compute grid dimension clauses and convert to an attribute
12028 attached to FN. This permits the target-side code to (a) massage
12029 the dimensions, (b) emit that data and (c) optimize. Non-constant
12030 dimensions are pushed onto ARGS.
12032 The attribute value is a TREE_LIST. A set of dimensions is
12033 represented as a list of INTEGER_CST. Those that are runtime
12034 expres are represented as an INTEGER_CST of zero.
12036 TOOO. Normally the attribute will just contain a single such list. If
12037 however it contains a list of lists, this will represent the use of
12038 device_type. Each member of the outer list is an assoc list of
12039 dimensions, keyed by the device type. The first entry will be the
12040 default. Well, that's the plan. */
12042 #define OACC_FN_ATTRIB "oacc function"
12044 /* Replace any existing oacc fn attribute with updated dimensions. */
12047 replace_oacc_fn_attrib (tree fn
, tree dims
)
12049 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12050 tree attribs
= DECL_ATTRIBUTES (fn
);
12052 /* If we happen to be present as the first attrib, drop it. */
12053 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12054 attribs
= TREE_CHAIN (attribs
);
12055 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12058 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12059 function attribute. Push any that are non-constant onto the ARGS
12060 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
12063 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
12065 /* Must match GOMP_DIM ordering. */
12066 static const omp_clause_code ids
[]
12067 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12068 OMP_CLAUSE_VECTOR_LENGTH
};
12070 tree dims
[GOMP_DIM_MAX
];
12071 tree attr
= NULL_TREE
;
12072 unsigned non_const
= 0;
12074 for (ix
= GOMP_DIM_MAX
; ix
--;)
12076 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12077 tree dim
= NULL_TREE
;
12080 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12082 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12084 dim
= integer_zero_node
;
12085 non_const
|= GOMP_DIM_MASK (ix
);
12087 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12090 replace_oacc_fn_attrib (fn
, attr
);
12094 /* Push a dynamic argument set. */
12095 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12096 NULL_TREE
, non_const
));
12097 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12098 if (non_const
& GOMP_DIM_MASK (ix
))
12099 args
->safe_push (dims
[ix
]);
12103 /* Retrieve the oacc function attrib and return it. Non-oacc
12104 functions will return NULL. */
12107 get_oacc_fn_attrib (tree fn
)
12109 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12112 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12115 expand_omp_target (struct omp_region
*region
)
12117 basic_block entry_bb
, exit_bb
, new_bb
;
12118 struct function
*child_cfun
;
12119 tree child_fn
, block
, t
;
12120 gimple_stmt_iterator gsi
;
12121 gomp_target
*entry_stmt
;
12124 bool offloaded
, data_region
;
12126 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
12127 new_bb
= region
->entry
;
12129 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
12130 switch (gimple_omp_target_kind (entry_stmt
))
12132 case GF_OMP_TARGET_KIND_REGION
:
12133 case GF_OMP_TARGET_KIND_UPDATE
:
12134 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12135 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12136 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12137 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12138 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12139 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12140 data_region
= false;
12142 case GF_OMP_TARGET_KIND_DATA
:
12143 case GF_OMP_TARGET_KIND_OACC_DATA
:
12144 data_region
= true;
12147 gcc_unreachable ();
12150 child_fn
= NULL_TREE
;
12154 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
12155 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
12158 /* Supported by expand_omp_taskreg, but not here. */
12159 if (child_cfun
!= NULL
)
12160 gcc_checking_assert (!child_cfun
->cfg
);
12161 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
12163 entry_bb
= region
->entry
;
12164 exit_bb
= region
->exit
;
12168 unsigned srcidx
, dstidx
, num
;
12170 /* If the offloading region needs data sent from the parent
12171 function, then the very first statement (except possible
12172 tree profile counter updates) of the offloading body
12173 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
12174 &.OMP_DATA_O is passed as an argument to the child function,
12175 we need to replace it with the argument as seen by the child
12178 In most cases, this will end up being the identity assignment
12179 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
12180 a function call that has been inlined, the original PARM_DECL
12181 .OMP_DATA_I may have been converted into a different local
12182 variable. In which case, we need to keep the assignment. */
12183 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
12186 basic_block entry_succ_bb
= single_succ (entry_bb
);
12187 gimple_stmt_iterator gsi
;
12189 gimple
*tgtcopy_stmt
= NULL
;
12190 tree sender
= TREE_VEC_ELT (data_arg
, 0);
12192 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
12194 gcc_assert (!gsi_end_p (gsi
));
12195 stmt
= gsi_stmt (gsi
);
12196 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
12199 if (gimple_num_ops (stmt
) == 2)
12201 tree arg
= gimple_assign_rhs1 (stmt
);
12203 /* We're ignoring the subcode because we're
12204 effectively doing a STRIP_NOPS. */
12206 if (TREE_CODE (arg
) == ADDR_EXPR
12207 && TREE_OPERAND (arg
, 0) == sender
)
12209 tgtcopy_stmt
= stmt
;
12215 gcc_assert (tgtcopy_stmt
!= NULL
);
12216 arg
= DECL_ARGUMENTS (child_fn
);
12218 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
12219 gsi_remove (&gsi
, true);
12222 /* Declare local variables needed in CHILD_CFUN. */
12223 block
= DECL_INITIAL (child_fn
);
12224 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
12225 /* The gimplifier could record temporaries in the offloading block
12226 rather than in containing function's local_decls chain,
12227 which would mean cgraph missed finalizing them. Do it now. */
12228 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
12229 if (TREE_CODE (t
) == VAR_DECL
12231 && !DECL_EXTERNAL (t
))
12232 varpool_node::finalize_decl (t
);
12233 DECL_SAVED_TREE (child_fn
) = NULL
;
12234 /* We'll create a CFG for child_fn, so no gimple body is needed. */
12235 gimple_set_body (child_fn
, NULL
);
12236 TREE_USED (block
) = 1;
12238 /* Reset DECL_CONTEXT on function arguments. */
12239 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
12240 DECL_CONTEXT (t
) = child_fn
;
12242 /* Split ENTRY_BB at GIMPLE_*,
12243 so that it can be moved to the child function. */
12244 gsi
= gsi_last_bb (entry_bb
);
12245 stmt
= gsi_stmt (gsi
);
12247 && gimple_code (stmt
) == gimple_code (entry_stmt
));
12248 e
= split_block (entry_bb
, stmt
);
12249 gsi_remove (&gsi
, true);
12250 entry_bb
= e
->dest
;
12251 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
12253 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
12256 gsi
= gsi_last_bb (exit_bb
);
12257 gcc_assert (!gsi_end_p (gsi
)
12258 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
12259 stmt
= gimple_build_return (NULL
);
12260 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
12261 gsi_remove (&gsi
, true);
12264 /* Move the offloading region into CHILD_CFUN. */
12266 block
= gimple_block (entry_stmt
);
12268 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
12270 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
12271 /* When the OMP expansion process cannot guarantee an up-to-date
12272 loop tree arrange for the child function to fixup loops. */
12273 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12274 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
12276 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
12277 num
= vec_safe_length (child_cfun
->local_decls
);
12278 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
12280 t
= (*child_cfun
->local_decls
)[srcidx
];
12281 if (DECL_CONTEXT (t
) == cfun
->decl
)
12283 if (srcidx
!= dstidx
)
12284 (*child_cfun
->local_decls
)[dstidx
] = t
;
12288 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
12290 /* Inform the callgraph about the new function. */
12291 child_cfun
->curr_properties
= cfun
->curr_properties
;
12292 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
12293 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
12294 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
12295 node
->parallelized_function
= 1;
12296 cgraph_node::add_new_function (child_fn
, true);
12298 #ifdef ENABLE_OFFLOADING
12299 /* Add the new function to the offload table. */
12300 vec_safe_push (offload_funcs
, child_fn
);
12303 /* Fix the callgraph edges for child_cfun. Those for cfun will be
12304 fixed in a following pass. */
12305 push_cfun (child_cfun
);
12306 cgraph_edge::rebuild_edges ();
12308 #ifdef ENABLE_OFFLOADING
12309 /* Prevent IPA from removing child_fn as unreachable, since there are no
12310 refs from the parent function to child_fn in offload LTO mode. */
12311 cgraph_node::get (child_fn
)->mark_force_output ();
12314 /* Some EH regions might become dead, see PR34608. If
12315 pass_cleanup_cfg isn't the first pass to happen with the
12316 new child, these dead EH edges might cause problems.
12317 Clean them up now. */
12318 if (flag_exceptions
)
12321 bool changed
= false;
12323 FOR_EACH_BB_FN (bb
, cfun
)
12324 changed
|= gimple_purge_dead_eh_edges (bb
);
12326 cleanup_tree_cfg ();
12328 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12329 verify_loop_structure ();
12333 /* Emit a library call to launch the offloading region, or do data
12335 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
12336 enum built_in_function start_ix
;
12337 location_t clause_loc
;
12338 unsigned int flags_i
= 0;
12340 switch (gimple_omp_target_kind (entry_stmt
))
12342 case GF_OMP_TARGET_KIND_REGION
:
12343 start_ix
= BUILT_IN_GOMP_TARGET
;
12345 case GF_OMP_TARGET_KIND_DATA
:
12346 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
12348 case GF_OMP_TARGET_KIND_UPDATE
:
12349 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
12351 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12352 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12354 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12355 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12356 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
12358 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12359 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12360 start_ix
= BUILT_IN_GOACC_PARALLEL
;
12362 case GF_OMP_TARGET_KIND_OACC_DATA
:
12363 start_ix
= BUILT_IN_GOACC_DATA_START
;
12365 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12366 start_ix
= BUILT_IN_GOACC_UPDATE
;
12368 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12369 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
12372 gcc_unreachable ();
12375 clauses
= gimple_omp_target_clauses (entry_stmt
);
12377 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
12378 library choose) and there is no conditional. */
12380 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
12382 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
12384 cond
= OMP_CLAUSE_IF_EXPR (c
);
12386 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
12389 /* Even if we pass it to all library function calls, it is currently only
12390 defined/used for the OpenMP target ones. */
12391 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
12392 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
12393 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
12394 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
12396 device
= OMP_CLAUSE_DEVICE_ID (c
);
12397 clause_loc
= OMP_CLAUSE_LOCATION (c
);
12400 clause_loc
= gimple_location (entry_stmt
);
12402 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
12404 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
12406 /* Ensure 'device' is of the correct type. */
12407 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
12409 /* If we found the clause 'if (cond)', build
12410 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
12413 cond
= gimple_boolify (cond
);
12415 basic_block cond_bb
, then_bb
, else_bb
;
12419 tmp_var
= create_tmp_var (TREE_TYPE (device
));
12421 e
= split_block_after_labels (new_bb
);
12424 gsi
= gsi_last_bb (new_bb
);
12426 e
= split_block (new_bb
, gsi_stmt (gsi
));
12432 then_bb
= create_empty_bb (cond_bb
);
12433 else_bb
= create_empty_bb (then_bb
);
12434 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
12435 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
12437 stmt
= gimple_build_cond_empty (cond
);
12438 gsi
= gsi_last_bb (cond_bb
);
12439 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12441 gsi
= gsi_start_bb (then_bb
);
12442 stmt
= gimple_build_assign (tmp_var
, device
);
12443 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12445 gsi
= gsi_start_bb (else_bb
);
12446 stmt
= gimple_build_assign (tmp_var
,
12447 build_int_cst (integer_type_node
,
12448 GOMP_DEVICE_HOST_FALLBACK
));
12449 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12451 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
12452 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
12453 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
12454 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
12455 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
12456 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
12461 gsi
= gsi_last_bb (new_bb
);
12462 t
= gimple_omp_target_data_arg (entry_stmt
);
12465 t1
= size_zero_node
;
12466 t2
= build_zero_cst (ptr_type_node
);
12472 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
12473 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
12474 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
12475 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
12476 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
12480 bool tagging
= false;
12481 /* The maximum number used by any start_ix, without varargs. */
12482 auto_vec
<tree
, 11> args
;
12483 args
.quick_push (device
);
12485 args
.quick_push (build_fold_addr_expr (child_fn
));
12486 args
.quick_push (t1
);
12487 args
.quick_push (t2
);
12488 args
.quick_push (t3
);
12489 args
.quick_push (t4
);
12492 case BUILT_IN_GOACC_DATA_START
:
12493 case BUILT_IN_GOMP_TARGET_DATA
:
12495 case BUILT_IN_GOMP_TARGET
:
12496 case BUILT_IN_GOMP_TARGET_UPDATE
:
12497 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
12498 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
12499 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
12501 depend
= OMP_CLAUSE_DECL (c
);
12503 depend
= build_int_cst (ptr_type_node
, 0);
12504 args
.quick_push (depend
);
12506 case BUILT_IN_GOACC_PARALLEL
:
12508 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
12512 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
12513 case BUILT_IN_GOACC_UPDATE
:
12515 tree t_async
= NULL_TREE
;
12517 /* If present, use the value specified by the respective
12518 clause, making sure that is of the correct type. */
12519 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
12521 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12523 OMP_CLAUSE_ASYNC_EXPR (c
));
12525 /* Default values for t_async. */
12526 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
12528 build_int_cst (integer_type_node
,
12530 if (tagging
&& t_async
)
12532 unsigned HOST_WIDE_INT i_async
;
12534 if (TREE_CODE (t_async
) == INTEGER_CST
)
12536 /* See if we can pack the async arg in to the tag's
12538 i_async
= TREE_INT_CST_LOW (t_async
);
12540 if (i_async
< GOMP_LAUNCH_OP_MAX
)
12541 t_async
= NULL_TREE
;
12544 i_async
= GOMP_LAUNCH_OP_MAX
;
12545 args
.safe_push (oacc_launch_pack
12546 (GOMP_LAUNCH_ASYNC
, NULL_TREE
, i_async
));
12549 args
.safe_push (t_async
);
12551 /* Save the argument index, and ... */
12552 unsigned t_wait_idx
= args
.length ();
12553 unsigned num_waits
= 0;
12554 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
12556 /* ... push a placeholder. */
12557 args
.safe_push (integer_zero_node
);
12559 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
12560 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
12562 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12564 OMP_CLAUSE_WAIT_EXPR (c
)));
12568 if (!tagging
|| num_waits
)
12572 /* Now that we know the number, update the placeholder. */
12574 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
12576 len
= build_int_cst (integer_type_node
, num_waits
);
12577 len
= fold_convert_loc (gimple_location (entry_stmt
),
12578 unsigned_type_node
, len
);
12579 args
[t_wait_idx
] = len
;
12584 gcc_unreachable ();
12587 /* Push terminal marker - zero. */
12588 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
12590 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
12591 gimple_set_location (g
, gimple_location (entry_stmt
));
12592 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12595 g
= gsi_stmt (gsi
);
12596 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
12597 gsi_remove (&gsi
, true);
12599 if (data_region
&& region
->exit
)
12601 gsi
= gsi_last_bb (region
->exit
);
12602 g
= gsi_stmt (gsi
);
12603 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
12604 gsi_remove (&gsi
, true);
12609 /* Expand the parallel region tree rooted at REGION. Expansion
12610 proceeds in depth-first order. Innermost regions are expanded
12611 first. This way, parallel regions that require a new function to
12612 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
12613 internal dependencies in their body. */
12616 expand_omp (struct omp_region
*region
)
12620 location_t saved_location
;
12621 gimple
*inner_stmt
= NULL
;
12623 /* First, determine whether this is a combined parallel+workshare
12625 if (region
->type
== GIMPLE_OMP_PARALLEL
)
12626 determine_parallel_type (region
);
12628 if (region
->type
== GIMPLE_OMP_FOR
12629 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
12630 inner_stmt
= last_stmt (region
->inner
->entry
);
12633 expand_omp (region
->inner
);
12635 saved_location
= input_location
;
12636 if (gimple_has_location (last_stmt (region
->entry
)))
12637 input_location
= gimple_location (last_stmt (region
->entry
));
12639 switch (region
->type
)
12641 case GIMPLE_OMP_PARALLEL
:
12642 case GIMPLE_OMP_TASK
:
12643 expand_omp_taskreg (region
);
12646 case GIMPLE_OMP_FOR
:
12647 expand_omp_for (region
, inner_stmt
);
12650 case GIMPLE_OMP_SECTIONS
:
12651 expand_omp_sections (region
);
12654 case GIMPLE_OMP_SECTION
:
12655 /* Individual omp sections are handled together with their
12656 parent GIMPLE_OMP_SECTIONS region. */
12659 case GIMPLE_OMP_SINGLE
:
12660 expand_omp_single (region
);
12663 case GIMPLE_OMP_ORDERED
:
12665 gomp_ordered
*ord_stmt
12666 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
12667 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
12668 OMP_CLAUSE_DEPEND
))
12670 /* We'll expand these when expanding corresponding
12671 worksharing region with ordered(n) clause. */
12672 gcc_assert (region
->outer
12673 && region
->outer
->type
== GIMPLE_OMP_FOR
);
12674 region
->ord_stmt
= ord_stmt
;
12679 case GIMPLE_OMP_MASTER
:
12680 case GIMPLE_OMP_TASKGROUP
:
12681 case GIMPLE_OMP_CRITICAL
:
12682 case GIMPLE_OMP_TEAMS
:
12683 expand_omp_synch (region
);
12686 case GIMPLE_OMP_ATOMIC_LOAD
:
12687 expand_omp_atomic (region
);
12690 case GIMPLE_OMP_TARGET
:
12691 expand_omp_target (region
);
12695 gcc_unreachable ();
12698 input_location
= saved_location
;
12699 region
= region
->next
;
12704 /* Helper for build_omp_regions. Scan the dominator tree starting at
12705 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
12706 true, the function ends once a single tree is built (otherwise, whole
12707 forest of OMP constructs may be built). */
12710 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
12713 gimple_stmt_iterator gsi
;
12717 gsi
= gsi_last_bb (bb
);
12718 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
12720 struct omp_region
*region
;
12721 enum gimple_code code
;
12723 stmt
= gsi_stmt (gsi
);
12724 code
= gimple_code (stmt
);
12725 if (code
== GIMPLE_OMP_RETURN
)
12727 /* STMT is the return point out of region PARENT. Mark it
12728 as the exit point and make PARENT the immediately
12729 enclosing region. */
12730 gcc_assert (parent
);
12733 parent
= parent
->outer
;
12735 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
12737 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
12738 GIMPLE_OMP_RETURN, but matches with
12739 GIMPLE_OMP_ATOMIC_LOAD. */
12740 gcc_assert (parent
);
12741 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
12744 parent
= parent
->outer
;
12746 else if (code
== GIMPLE_OMP_CONTINUE
)
12748 gcc_assert (parent
);
12751 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
12753 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
12754 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
12758 region
= new_omp_region (bb
, code
, parent
);
12760 if (code
== GIMPLE_OMP_TARGET
)
12762 switch (gimple_omp_target_kind (stmt
))
12764 case GF_OMP_TARGET_KIND_REGION
:
12765 case GF_OMP_TARGET_KIND_DATA
:
12766 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12767 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12768 case GF_OMP_TARGET_KIND_OACC_DATA
:
12770 case GF_OMP_TARGET_KIND_UPDATE
:
12771 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12772 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12773 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12774 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12775 /* ..., other than for those stand-alone directives... */
12779 gcc_unreachable ();
12782 else if (code
== GIMPLE_OMP_ORDERED
12783 && find_omp_clause (gimple_omp_ordered_clauses
12784 (as_a
<gomp_ordered
*> (stmt
)),
12785 OMP_CLAUSE_DEPEND
))
12786 /* #pragma omp ordered depend is also just a stand-alone
12789 /* ..., this directive becomes the parent for a new region. */
12795 if (single_tree
&& !parent
)
12798 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
12800 son
= next_dom_son (CDI_DOMINATORS
, son
))
12801 build_omp_regions_1 (son
, parent
, single_tree
);
12804 /* Builds the tree of OMP regions rooted at ROOT, storing it to
12805 root_omp_region. */
12808 build_omp_regions_root (basic_block root
)
12810 gcc_assert (root_omp_region
== NULL
);
12811 build_omp_regions_1 (root
, NULL
, true);
12812 gcc_assert (root_omp_region
!= NULL
);
12815 /* Expands omp construct (and its subconstructs) starting in HEAD. */
12818 omp_expand_local (basic_block head
)
12820 build_omp_regions_root (head
);
12821 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
12823 fprintf (dump_file
, "\nOMP region tree\n\n");
12824 dump_omp_region (dump_file
, root_omp_region
, 0);
12825 fprintf (dump_file
, "\n");
12828 remove_exit_barriers (root_omp_region
);
12829 expand_omp (root_omp_region
);
12831 free_omp_regions ();
12834 /* Scan the CFG and build a tree of OMP regions. Return the root of
12835 the OMP region tree. */
12838 build_omp_regions (void)
12840 gcc_assert (root_omp_region
== NULL
);
12841 calculate_dominance_info (CDI_DOMINATORS
);
12842 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
12845 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
12847 static unsigned int
12848 execute_expand_omp (void)
12850 build_omp_regions ();
12852 if (!root_omp_region
)
12857 fprintf (dump_file
, "\nOMP region tree\n\n");
12858 dump_omp_region (dump_file
, root_omp_region
, 0);
12859 fprintf (dump_file
, "\n");
12862 remove_exit_barriers (root_omp_region
);
12864 expand_omp (root_omp_region
);
12866 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12867 verify_loop_structure ();
12868 cleanup_tree_cfg ();
12870 free_omp_regions ();
12875 /* OMP expansion -- the default pass, run before creation of SSA form. */
12879 const pass_data pass_data_expand_omp
=
12881 GIMPLE_PASS
, /* type */
12882 "ompexp", /* name */
12883 OPTGROUP_NONE
, /* optinfo_flags */
12884 TV_NONE
, /* tv_id */
12885 PROP_gimple_any
, /* properties_required */
12886 PROP_gimple_eomp
, /* properties_provided */
12887 0, /* properties_destroyed */
12888 0, /* todo_flags_start */
12889 0, /* todo_flags_finish */
12892 class pass_expand_omp
: public gimple_opt_pass
12895 pass_expand_omp (gcc::context
*ctxt
)
12896 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
12899 /* opt_pass methods: */
12900 virtual unsigned int execute (function
*)
12902 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
12903 || flag_openmp_simd
!= 0)
12904 && !seen_error ());
12906 /* This pass always runs, to provide PROP_gimple_eomp.
12907 But often, there is nothing to do. */
12911 return execute_expand_omp ();
12914 }; // class pass_expand_omp
12916 } // anon namespace
12919 make_pass_expand_omp (gcc::context
*ctxt
)
12921 return new pass_expand_omp (ctxt
);
12926 const pass_data pass_data_expand_omp_ssa
=
12928 GIMPLE_PASS
, /* type */
12929 "ompexpssa", /* name */
12930 OPTGROUP_NONE
, /* optinfo_flags */
12931 TV_NONE
, /* tv_id */
12932 PROP_cfg
| PROP_ssa
, /* properties_required */
12933 PROP_gimple_eomp
, /* properties_provided */
12934 0, /* properties_destroyed */
12935 0, /* todo_flags_start */
12936 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
12939 class pass_expand_omp_ssa
: public gimple_opt_pass
12942 pass_expand_omp_ssa (gcc::context
*ctxt
)
12943 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
12946 /* opt_pass methods: */
12947 virtual bool gate (function
*fun
)
12949 return !(fun
->curr_properties
& PROP_gimple_eomp
);
12951 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
12953 }; // class pass_expand_omp_ssa
12955 } // anon namespace
12958 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
12960 return new pass_expand_omp_ssa (ctxt
);
12963 /* Routines to lower OMP directives into OMP-GIMPLE. */
12965 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
12966 convert it to gimple. */
12968 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
12972 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
12974 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
12975 gimple_seq_add_stmt (seq
, stmt
);
12979 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12980 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
12981 gimplify_assign (t
, rdest
, seq
);
12984 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12985 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
12986 gimplify_assign (t
, idest
, seq
);
12989 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
12990 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
12991 gimplify_assign (t
, rsrc
, seq
);
12994 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
12995 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
12996 gimplify_assign (t
, isrc
, seq
);
12999 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
13000 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
13003 if (op
== PLUS_EXPR
)
13005 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
13006 gimple_seq_add_stmt (seq
, stmt
);
13008 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
13009 gimple_seq_add_stmt (seq
, stmt
);
13011 else if (op
== MULT_EXPR
)
13013 /* Let x = a + ib = dest, y = c + id = src.
13014 x * y = (ac - bd) + i(ad + bc) */
13015 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
13016 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
13017 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
13018 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
13020 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
13021 gimple_seq_add_stmt (seq
, stmt
);
13023 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
13024 gimple_seq_add_stmt (seq
, stmt
);
13026 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
13027 gimple_seq_add_stmt (seq
, stmt
);
13029 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
13030 gimple_seq_add_stmt (seq
, stmt
);
13032 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
13033 gimple_seq_add_stmt (seq
, stmt
);
13035 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
13036 gimple_seq_add_stmt (seq
, stmt
);
13039 gcc_unreachable ();
13041 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
13042 gimplify_assign (dest
, result
, seq
);
13045 /* Initialize the reduction array with default values. */
13048 oacc_init_reduction_array (tree array
, tree init
, tree nthreads
,
13049 gimple_seq
*stmt_seqp
)
13051 tree type
= TREE_TYPE (TREE_TYPE (array
));
13052 tree x
, loop_header
, loop_body
, loop_exit
;
13055 /* Create for loop.
13057 let var = the original reduction variable
13058 let array = reduction variable array
13060 for (i = 0; i < nthreads; i++)
13064 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
13065 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
13066 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
13068 /* Create and initialize an index variable. */
13069 tree ix
= create_tmp_var (sizetype
);
13070 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
13073 /* Insert the loop header label here. */
13074 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
13076 /* Exit loop if ix >= nthreads. */
13077 x
= create_tmp_var (sizetype
);
13078 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
13079 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
13080 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13082 /* Insert the loop body label here. */
13083 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
13085 /* Calculate the array offset. */
13086 tree offset
= create_tmp_var (sizetype
);
13087 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
13088 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
13089 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13091 tree ptr
= create_tmp_var (TREE_TYPE (array
));
13092 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
13093 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13096 gimplify_assign (build_simple_mem_ref (ptr
), init
, stmt_seqp
);
13098 /* Increment the induction variable. */
13099 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
13100 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
13101 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13103 /* Go back to the top of the loop. */
13104 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
13106 /* Place the loop exit label here. */
13107 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
13110 /* Helper function to initialize local data for the reduction arrays.
13111 The reduction arrays need to be placed inside the calling function
13112 for accelerators, or else the host won't be able to preform the final
13116 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
13117 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
13123 /* Find the innermost OpenACC parallel context. */
13124 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
13125 && (gimple_omp_target_kind (ctx
->stmt
)
13126 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
13130 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
13131 && (gimple_omp_target_kind (octx
->stmt
)
13132 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
13134 /* Extract the clauses. */
13135 oc
= gimple_omp_target_clauses (octx
->stmt
);
13137 /* Find the last outer clause. */
13138 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
13141 /* Allocate arrays for each reduction variable. */
13142 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
13144 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
13147 tree var
= OMP_CLAUSE_DECL (c
);
13148 tree type
= get_base_type (var
);
13149 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
13153 /* Calculate size of the reduction array. */
13154 t
= create_tmp_var (TREE_TYPE (nthreads
));
13155 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
13156 fold_convert (TREE_TYPE (nthreads
),
13157 TYPE_SIZE_UNIT (type
)));
13158 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13160 size
= create_tmp_var (sizetype
);
13161 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
13163 /* Now allocate memory for it. */
13164 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
13165 stmt
= gimple_build_call (call
, 1, size
);
13166 gimple_call_set_lhs (stmt
, array
);
13167 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13169 /* Initialize array. */
13170 tree init
= omp_reduction_init_op (OMP_CLAUSE_LOCATION (c
),
13171 OMP_CLAUSE_REDUCTION_CODE (c
),
13173 oacc_init_reduction_array (array
, init
, nthreads
, stmt_seqp
);
13175 /* Map this array into the accelerator. */
13177 /* Add the reduction array to the list of clauses. */
13179 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
13180 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_TOFROM
);
13181 OMP_CLAUSE_DECL (t
) = x
;
13182 OMP_CLAUSE_CHAIN (t
) = NULL
;
13184 OMP_CLAUSE_CHAIN (oc
) = t
;
13186 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
13187 OMP_CLAUSE_SIZE (t
) = size
;
13192 /* Helper function to process the array of partial reductions. Nthreads
13193 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
13194 cannot be used here, because nthreads on the host may be different than
13195 on the accelerator. */
13198 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
13199 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
13201 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
13204 /* Create for loop.
13206 let var = the original reduction variable
13207 let array = reduction variable array
13209 for (i = 0; i < nthreads; i++)
13213 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
13214 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
13215 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
13217 /* Create and initialize an index variable. */
13218 tree ix
= create_tmp_var (sizetype
);
13219 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
13222 /* Insert the loop header label here. */
13223 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
13225 /* Exit loop if ix >= nthreads. */
13226 x
= create_tmp_var (sizetype
);
13227 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
13228 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
13229 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13231 /* Insert the loop body label here. */
13232 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
13234 /* Collapse each reduction array, one element at a time. */
13235 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
13237 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
13240 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
13242 /* reduction(-:var) sums up the partial results, so it acts
13243 identically to reduction(+:var). */
13244 if (reduction_code
== MINUS_EXPR
)
13245 reduction_code
= PLUS_EXPR
;
13247 /* Set up reduction variable var. */
13248 var
= OMP_CLAUSE_DECL (c
);
13249 type
= get_base_type (var
);
13250 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
13251 (OMP_CLAUSE_DECL (c
)), ctx
);
13253 /* Calculate the array offset. */
13254 tree offset
= create_tmp_var (sizetype
);
13255 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
13256 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
13257 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13259 tree ptr
= create_tmp_var (TREE_TYPE (array
));
13260 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
13261 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13263 /* Extract array[ix] into mem. */
13264 tree mem
= create_tmp_var (type
);
13265 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
13267 /* Find the original reduction variable. */
13268 if (is_reference (var
))
13269 var
= build_simple_mem_ref (var
);
13271 tree t
= create_tmp_var (type
);
13273 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
13274 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
13276 /* var = var op mem */
13277 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
13279 case TRUTH_ANDIF_EXPR
:
13280 case TRUTH_ORIF_EXPR
:
13281 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
13283 gimplify_and_add (t
, stmt_seqp
);
13286 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
13287 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
13291 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
13292 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
13293 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
13296 /* Increment the induction variable. */
13297 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
13298 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
13299 gimple_seq_add_stmt (stmt_seqp
, stmt
);
13301 /* Go back to the top of the loop. */
13302 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
13304 /* Place the loop exit label here. */
13305 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
13308 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
13309 scan that for reductions. */
13312 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
13313 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
13315 gimple_stmt_iterator gsi
;
13316 gimple_seq inner
= NULL
;
13318 /* A collapse clause may have inserted a new bind block. */
13319 gsi
= gsi_start (*body
);
13320 while (!gsi_end_p (gsi
))
13322 gimple
*stmt
= gsi_stmt (gsi
);
13323 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
13325 inner
= gimple_bind_body (bind_stmt
);
13327 gsi
= gsi_start (*body
);
13329 else if (dyn_cast
<gomp_for
*> (stmt
))
13335 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13337 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
13339 bool reduction_found
= false;
13341 gimple
*stmt
= gsi_stmt (gsi
);
13343 switch (gimple_code (stmt
))
13345 case GIMPLE_OMP_FOR
:
13346 clauses
= gimple_omp_for_clauses (stmt
);
13348 /* Search for a reduction clause. */
13349 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
13350 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
13352 reduction_found
= true;
13356 if (!reduction_found
)
13359 ctx
= maybe_lookup_ctx (stmt
);
13362 /* Extract the number of threads. */
13363 nthreads
= create_tmp_var (sizetype
);
13364 t
= oacc_max_threads (ctx
);
13365 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
13367 /* Determine if this is kernel will be executed on the host. */
13368 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
13369 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
13370 stmt
= gimple_build_call (call
, 0);
13371 gimple_call_set_lhs (stmt
, acc_device
);
13372 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
13374 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
13375 acc_device_host
= create_tmp_var (integer_type_node
,
13376 ".acc_device_host");
13377 gimplify_assign (acc_device_host
,
13378 build_int_cst (integer_type_node
,
13382 enter
= create_artificial_label (UNKNOWN_LOCATION
);
13383 exit
= create_artificial_label (UNKNOWN_LOCATION
);
13385 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
13387 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
13388 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
13389 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
13392 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
13394 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
13396 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
13399 // Scan for other directives which support reduction here.
13405 /* If ctx is a worksharing context inside of a cancellable parallel
13406 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
13407 and conditional branch to parallel's cancel_label to handle
13408 cancellation in the implicit barrier. */
13411 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
13413 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
13414 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
13415 if (gimple_omp_return_nowait_p (omp_return
))
13418 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
13419 && ctx
->outer
->cancellable
)
13421 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
13422 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
13423 tree lhs
= create_tmp_var (c_bool_type
);
13424 gimple_omp_return_set_lhs (omp_return
, lhs
);
13425 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
13426 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
13427 fold_convert (c_bool_type
,
13428 boolean_false_node
),
13429 ctx
->outer
->cancel_label
, fallthru_label
);
13430 gimple_seq_add_stmt (body
, g
);
13431 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
13435 /* Lower the OpenMP sections directive in the current statement in GSI_P.
13436 CTX is the enclosing OMP context for the current statement. */
13439 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13441 tree block
, control
;
13442 gimple_stmt_iterator tgsi
;
13443 gomp_sections
*stmt
;
13445 gbind
*new_stmt
, *bind
;
13446 gimple_seq ilist
, dlist
, olist
, new_body
;
13448 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
13450 push_gimplify_context ();
13454 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
13455 &ilist
, &dlist
, ctx
, NULL
);
13457 new_body
= gimple_omp_body (stmt
);
13458 gimple_omp_set_body (stmt
, NULL
);
13459 tgsi
= gsi_start (new_body
);
13460 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
13465 sec_start
= gsi_stmt (tgsi
);
13466 sctx
= maybe_lookup_ctx (sec_start
);
13469 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
13470 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
13471 GSI_CONTINUE_LINKING
);
13472 gimple_omp_set_body (sec_start
, NULL
);
13474 if (gsi_one_before_end_p (tgsi
))
13476 gimple_seq l
= NULL
;
13477 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
13479 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
13480 gimple_omp_section_set_last (sec_start
);
13483 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
13484 GSI_CONTINUE_LINKING
);
13487 block
= make_node (BLOCK
);
13488 bind
= gimple_build_bind (NULL
, new_body
, block
);
13491 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
13493 block
= make_node (BLOCK
);
13494 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
13495 gsi_replace (gsi_p
, new_stmt
, true);
13497 pop_gimplify_context (new_stmt
);
13498 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
13499 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13500 if (BLOCK_VARS (block
))
13501 TREE_USED (block
) = 1;
13504 gimple_seq_add_seq (&new_body
, ilist
);
13505 gimple_seq_add_stmt (&new_body
, stmt
);
13506 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
13507 gimple_seq_add_stmt (&new_body
, bind
);
13509 control
= create_tmp_var (unsigned_type_node
, ".section");
13510 t
= gimple_build_omp_continue (control
, control
);
13511 gimple_omp_sections_set_control (stmt
, control
);
13512 gimple_seq_add_stmt (&new_body
, t
);
13514 gimple_seq_add_seq (&new_body
, olist
);
13515 if (ctx
->cancellable
)
13516 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
13517 gimple_seq_add_seq (&new_body
, dlist
);
13519 new_body
= maybe_catch_exception (new_body
);
13521 t
= gimple_build_omp_return
13522 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
13523 OMP_CLAUSE_NOWAIT
));
13524 gimple_seq_add_stmt (&new_body
, t
);
13525 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
13527 gimple_bind_set_body (new_stmt
, new_body
);
13531 /* A subroutine of lower_omp_single. Expand the simple form of
13532 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
13534 if (GOMP_single_start ())
13536 [ GOMP_barrier (); ] -> unless 'nowait' is present.
13538 FIXME. It may be better to delay expanding the logic of this until
13539 pass_expand_omp. The expanded logic may make the job more difficult
13540 to a synchronization analysis pass. */
13543 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
13545 location_t loc
= gimple_location (single_stmt
);
13546 tree tlabel
= create_artificial_label (loc
);
13547 tree flabel
= create_artificial_label (loc
);
13548 gimple
*call
, *cond
;
13551 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
13552 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
13553 call
= gimple_build_call (decl
, 0);
13554 gimple_call_set_lhs (call
, lhs
);
13555 gimple_seq_add_stmt (pre_p
, call
);
13557 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
13558 fold_convert_loc (loc
, TREE_TYPE (lhs
),
13559 boolean_true_node
),
13561 gimple_seq_add_stmt (pre_p
, cond
);
13562 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
13563 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13564 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
13568 /* A subroutine of lower_omp_single. Expand the simple form of
13569 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
13571 #pragma omp single copyprivate (a, b, c)
13573 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
13576 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
13582 GOMP_single_copy_end (©out);
13593 FIXME. It may be better to delay expanding the logic of this until
13594 pass_expand_omp. The expanded logic may make the job more difficult
13595 to a synchronization analysis pass. */
13598 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
13601 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
13602 gimple_seq copyin_seq
;
13603 location_t loc
= gimple_location (single_stmt
);
13605 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
13607 ptr_type
= build_pointer_type (ctx
->record_type
);
13608 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
13610 l0
= create_artificial_label (loc
);
13611 l1
= create_artificial_label (loc
);
13612 l2
= create_artificial_label (loc
);
13614 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
13615 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
13616 t
= fold_convert_loc (loc
, ptr_type
, t
);
13617 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
13619 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
13620 build_int_cst (ptr_type
, 0));
13621 t
= build3 (COND_EXPR
, void_type_node
, t
,
13622 build_and_jump (&l0
), build_and_jump (&l1
));
13623 gimplify_and_add (t
, pre_p
);
13625 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
13627 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13630 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
13633 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
13634 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
13635 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
13636 gimplify_and_add (t
, pre_p
);
13638 t
= build_and_jump (&l2
);
13639 gimplify_and_add (t
, pre_p
);
13641 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
13643 gimple_seq_add_seq (pre_p
, copyin_seq
);
13645 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
13649 /* Expand code for an OpenMP single directive. */
13652 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13656 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
13658 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
13660 push_gimplify_context ();
13662 block
= make_node (BLOCK
);
13663 bind
= gimple_build_bind (NULL
, NULL
, block
);
13664 gsi_replace (gsi_p
, bind
, true);
13667 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
13668 &bind_body
, &dlist
, ctx
, NULL
);
13669 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
13671 gimple_seq_add_stmt (&bind_body
, single_stmt
);
13673 if (ctx
->record_type
)
13674 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
13676 lower_omp_single_simple (single_stmt
, &bind_body
);
13678 gimple_omp_set_body (single_stmt
, NULL
);
13680 gimple_seq_add_seq (&bind_body
, dlist
);
13682 bind_body
= maybe_catch_exception (bind_body
);
13684 t
= gimple_build_omp_return
13685 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
13686 OMP_CLAUSE_NOWAIT
));
13687 gimple_seq_add_stmt (&bind_body_tail
, t
);
13688 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
13689 if (ctx
->record_type
)
13691 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
13692 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
13693 TREE_THIS_VOLATILE (clobber
) = 1;
13694 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
13695 clobber
), GSI_SAME_STMT
);
13697 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
13698 gimple_bind_set_body (bind
, bind_body
);
13700 pop_gimplify_context (bind
);
13702 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13703 BLOCK_VARS (block
) = ctx
->block_vars
;
13704 if (BLOCK_VARS (block
))
13705 TREE_USED (block
) = 1;
13709 /* Expand code for an OpenMP master directive. */
13712 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13714 tree block
, lab
= NULL
, x
, bfn_decl
;
13715 gimple
*stmt
= gsi_stmt (*gsi_p
);
13717 location_t loc
= gimple_location (stmt
);
13720 push_gimplify_context ();
13722 block
= make_node (BLOCK
);
13723 bind
= gimple_build_bind (NULL
, NULL
, block
);
13724 gsi_replace (gsi_p
, bind
, true);
13725 gimple_bind_add_stmt (bind
, stmt
);
13727 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
13728 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
13729 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
13730 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
13732 gimplify_and_add (x
, &tseq
);
13733 gimple_bind_add_seq (bind
, tseq
);
13735 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13736 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13737 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13738 gimple_omp_set_body (stmt
, NULL
);
13740 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
13742 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13744 pop_gimplify_context (bind
);
13746 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13747 BLOCK_VARS (block
) = ctx
->block_vars
;
13751 /* Expand code for an OpenMP taskgroup directive. */
13754 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13756 gimple
*stmt
= gsi_stmt (*gsi_p
);
13759 tree block
= make_node (BLOCK
);
13761 bind
= gimple_build_bind (NULL
, NULL
, block
);
13762 gsi_replace (gsi_p
, bind
, true);
13763 gimple_bind_add_stmt (bind
, stmt
);
13765 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
13767 gimple_bind_add_stmt (bind
, x
);
13769 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13770 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13771 gimple_omp_set_body (stmt
, NULL
);
13773 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13775 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13776 BLOCK_VARS (block
) = ctx
->block_vars
;
13780 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
13783 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
13786 struct omp_for_data fd
;
13787 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
13790 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
13791 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
13792 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
13796 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13797 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
13798 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
13799 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
13801 /* Merge depend clauses from multiple adjacent
13802 #pragma omp ordered depend(sink:...) constructs
13803 into one #pragma omp ordered depend(sink:...), so that
13804 we can optimize them together. */
13805 gimple_stmt_iterator gsi
= *gsi_p
;
13807 while (!gsi_end_p (gsi
))
13809 gimple
*stmt
= gsi_stmt (gsi
);
13810 if (is_gimple_debug (stmt
)
13811 || gimple_code (stmt
) == GIMPLE_NOP
)
13816 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
13818 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
13819 c
= gimple_omp_ordered_clauses (ord_stmt2
);
13821 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
13822 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13825 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
13827 gsi_remove (&gsi
, true);
13831 /* Canonicalize sink dependence clauses into one folded clause if
13834 The basic algorithm is to create a sink vector whose first
13835 element is the GCD of all the first elements, and whose remaining
13836 elements are the minimum of the subsequent columns.
13838 We ignore dependence vectors whose first element is zero because
13839 such dependencies are known to be executed by the same thread.
13841 We take into account the direction of the loop, so a minimum
13842 becomes a maximum if the loop is iterating forwards. We also
13843 ignore sink clauses where the loop direction is unknown, or where
13844 the offsets are clearly invalid because they are not a multiple
13845 of the loop increment.
13849 #pragma omp for ordered(2)
13850 for (i=0; i < N; ++i)
13851 for (j=0; j < M; ++j)
13853 #pragma omp ordered \
13854 depend(sink:i-8,j-2) \
13855 depend(sink:i,j-1) \ // Completely ignored because i+0.
13856 depend(sink:i-4,j-3) \
13857 depend(sink:i-6,j-4)
13858 #pragma omp ordered depend(source)
13863 depend(sink:-gcd(8,4,6),-min(2,3,4))
13868 /* FIXME: Computing GCD's where the first element is zero is
13869 non-trivial in the presence of collapsed loops. Do this later. */
13870 if (fd
.collapse
> 1)
13873 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
13874 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
13875 tree folded_dep
= NULL_TREE
;
13876 /* TRUE if the first dimension's offset is negative. */
13877 bool neg_offset_p
= false;
13879 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13881 while ((c
= *list_p
) != NULL
)
13883 bool remove
= false;
13885 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
13886 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13887 goto next_ordered_clause
;
13890 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
13891 vec
&& TREE_CODE (vec
) == TREE_LIST
;
13892 vec
= TREE_CHAIN (vec
), ++i
)
13894 gcc_assert (i
< len
);
13896 /* extract_omp_for_data has canonicalized the condition. */
13897 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
13898 || fd
.loops
[i
].cond_code
== GT_EXPR
);
13899 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
13900 bool maybe_lexically_later
= true;
13902 /* While the committee makes up its mind, bail if we have any
13903 non-constant steps. */
13904 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
13905 goto lower_omp_ordered_ret
;
13907 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
13908 if (POINTER_TYPE_P (itype
))
13910 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
13911 TYPE_PRECISION (itype
),
13912 TYPE_SIGN (itype
));
13914 /* Ignore invalid offsets that are not multiples of the step. */
13915 if (!wi::multiple_of_p
13916 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
13919 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
13920 "ignoring sink clause with offset that is not "
13921 "a multiple of the loop step");
13923 goto next_ordered_clause
;
13926 /* Calculate the first dimension. The first dimension of
13927 the folded dependency vector is the GCD of the first
13928 elements, while ignoring any first elements whose offset
13932 /* Ignore dependence vectors whose first dimension is 0. */
13936 goto next_ordered_clause
;
13940 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
13942 error_at (OMP_CLAUSE_LOCATION (c
),
13943 "first offset must be in opposite direction "
13944 "of loop iterations");
13945 goto lower_omp_ordered_ret
;
13949 neg_offset_p
= forward
;
13950 /* Initialize the first time around. */
13951 if (folded_dep
== NULL_TREE
)
13954 folded_deps
[0] = offset
;
13957 folded_deps
[0] = wi::gcd (folded_deps
[0],
13961 /* Calculate minimum for the remaining dimensions. */
13964 folded_deps
[len
+ i
- 1] = offset
;
13965 if (folded_dep
== c
)
13966 folded_deps
[i
] = offset
;
13967 else if (maybe_lexically_later
13968 && !wi::eq_p (folded_deps
[i
], offset
))
13970 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
13974 for (j
= 1; j
<= i
; j
++)
13975 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
13978 maybe_lexically_later
= false;
13982 gcc_assert (i
== len
);
13986 next_ordered_clause
:
13988 *list_p
= OMP_CLAUSE_CHAIN (c
);
13990 list_p
= &OMP_CLAUSE_CHAIN (c
);
13996 folded_deps
[0] = -folded_deps
[0];
13998 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
13999 if (POINTER_TYPE_P (itype
))
14002 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14003 = wide_int_to_tree (itype
, folded_deps
[0]);
14004 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14005 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14008 lower_omp_ordered_ret
:
14010 /* Ordered without clauses is #pragma omp threads, while we want
14011 a nop instead if we remove all clauses. */
14012 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14013 gsi_replace (gsi_p
, gimple_build_nop (), true);
14017 /* Expand code for an OpenMP ordered directive. */
14020 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14023 gimple
*stmt
= gsi_stmt (*gsi_p
);
14024 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14028 = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
), OMP_CLAUSE_SIMD
);
14030 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14031 OMP_CLAUSE_DEPEND
))
14033 /* FIXME: This is needs to be moved to the expansion to verify various
14034 conditions only testable on cfg with dominators computed, and also
14035 all the depend clauses to be merged still might need to be available
14036 for the runtime checks. */
14038 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14042 push_gimplify_context ();
14044 block
= make_node (BLOCK
);
14045 bind
= gimple_build_bind (NULL
, NULL
, block
);
14046 gsi_replace (gsi_p
, bind
, true);
14047 gimple_bind_add_stmt (bind
, stmt
);
14051 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 0);
14052 cfun
->has_simduid_loops
= true;
14055 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14057 gimple_bind_add_stmt (bind
, x
);
14059 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14060 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14061 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14062 gimple_omp_set_body (stmt
, NULL
);
14065 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 0);
14067 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14069 gimple_bind_add_stmt (bind
, x
);
14071 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14073 pop_gimplify_context (bind
);
14075 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14076 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14080 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14081 substitution of a couple of function calls. But in the NAMED case,
14082 requires that languages coordinate a symbol name. It is therefore
14083 best put here in common code. */
14085 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14088 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14091 tree name
, lock
, unlock
;
14092 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14094 location_t loc
= gimple_location (stmt
);
14097 name
= gimple_omp_critical_name (stmt
);
14102 if (!critical_name_mutexes
)
14103 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14105 tree
*n
= critical_name_mutexes
->get (name
);
14110 decl
= create_tmp_var_raw (ptr_type_node
);
14112 new_str
= ACONCAT ((".gomp_critical_user_",
14113 IDENTIFIER_POINTER (name
), NULL
));
14114 DECL_NAME (decl
) = get_identifier (new_str
);
14115 TREE_PUBLIC (decl
) = 1;
14116 TREE_STATIC (decl
) = 1;
14117 DECL_COMMON (decl
) = 1;
14118 DECL_ARTIFICIAL (decl
) = 1;
14119 DECL_IGNORED_P (decl
) = 1;
14121 varpool_node::finalize_decl (decl
);
14123 critical_name_mutexes
->put (name
, decl
);
14128 /* If '#pragma omp critical' is inside offloaded region or
14129 inside function marked as offloadable, the symbol must be
14130 marked as offloadable too. */
14132 if (cgraph_node::get (current_function_decl
)->offloadable
)
14133 varpool_node::get_create (decl
)->offloadable
= 1;
14135 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14136 if (is_gimple_omp_offloaded (octx
->stmt
))
14138 varpool_node::get_create (decl
)->offloadable
= 1;
14142 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14143 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14145 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14146 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14147 build_fold_addr_expr_loc (loc
, decl
));
14151 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14152 lock
= build_call_expr_loc (loc
, lock
, 0);
14154 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14155 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14158 push_gimplify_context ();
14160 block
= make_node (BLOCK
);
14161 bind
= gimple_build_bind (NULL
, NULL
, block
);
14162 gsi_replace (gsi_p
, bind
, true);
14163 gimple_bind_add_stmt (bind
, stmt
);
14165 tbody
= gimple_bind_body (bind
);
14166 gimplify_and_add (lock
, &tbody
);
14167 gimple_bind_set_body (bind
, tbody
);
14169 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14170 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14171 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14172 gimple_omp_set_body (stmt
, NULL
);
14174 tbody
= gimple_bind_body (bind
);
14175 gimplify_and_add (unlock
, &tbody
);
14176 gimple_bind_set_body (bind
, tbody
);
14178 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14180 pop_gimplify_context (bind
);
14181 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14182 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14186 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14187 for a lastprivate clause. Given a loop control predicate of (V
14188 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14189 is appended to *DLIST, iterator initialization is appended to
14193 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14194 gimple_seq
*dlist
, struct omp_context
*ctx
)
14196 tree clauses
, cond
, vinit
;
14197 enum tree_code cond_code
;
14200 cond_code
= fd
->loop
.cond_code
;
14201 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14203 /* When possible, use a strict equality expression. This can let VRP
14204 type optimizations deduce the value and remove a copy. */
14205 if (tree_fits_shwi_p (fd
->loop
.step
))
14207 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14208 if (step
== 1 || step
== -1)
14209 cond_code
= EQ_EXPR
;
14212 tree n2
= fd
->loop
.n2
;
14213 if (fd
->collapse
> 1
14214 && TREE_CODE (n2
) != INTEGER_CST
14215 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14217 struct omp_context
*task_ctx
= NULL
;
14218 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14220 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14221 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
)
14223 struct omp_for_data outer_fd
;
14224 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14225 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14227 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14228 task_ctx
= ctx
->outer
->outer
;
14230 else if (is_task_ctx (ctx
->outer
))
14231 task_ctx
= ctx
->outer
;
14236 = find_omp_clause (gimple_omp_task_clauses (task_ctx
->stmt
),
14237 OMP_CLAUSE__LOOPTEMP_
);
14238 gcc_assert (innerc
);
14239 for (i
= 0; i
< fd
->collapse
; i
++)
14241 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14242 OMP_CLAUSE__LOOPTEMP_
);
14243 gcc_assert (innerc
);
14245 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14246 OMP_CLAUSE__LOOPTEMP_
);
14248 n2
= fold_convert (TREE_TYPE (n2
),
14249 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14253 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14255 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14257 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14258 if (!gimple_seq_empty_p (stmts
))
14260 gimple_seq_add_seq (&stmts
, *dlist
);
14263 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
14264 vinit
= fd
->loop
.n1
;
14265 if (cond_code
== EQ_EXPR
14266 && tree_fits_shwi_p (fd
->loop
.n2
)
14267 && ! integer_zerop (fd
->loop
.n2
))
14268 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
14270 vinit
= unshare_expr (vinit
);
14272 /* Initialize the iterator variable, so that threads that don't execute
14273 any iterations don't execute the lastprivate clauses by accident. */
14274 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
14279 /* Lower code for an OMP loop directive. */
14282 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14284 tree
*rhs_p
, block
;
14285 struct omp_for_data fd
, *fdp
= NULL
;
14286 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
14288 gimple_seq omp_for_body
, body
, dlist
;
14289 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
14292 push_gimplify_context ();
14294 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
14296 block
= make_node (BLOCK
);
14297 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14298 /* Replace at gsi right away, so that 'stmt' is no member
14299 of a sequence anymore as we're going to add to a different
14301 gsi_replace (gsi_p
, new_stmt
, true);
14303 /* Move declaration of temporaries in the loop body before we make
14305 omp_for_body
= gimple_omp_body (stmt
);
14306 if (!gimple_seq_empty_p (omp_for_body
)
14307 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
14310 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
14311 tree vars
= gimple_bind_vars (inner_bind
);
14312 gimple_bind_append_vars (new_stmt
, vars
);
14313 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
14314 keep them on the inner_bind and it's block. */
14315 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
14316 if (gimple_bind_block (inner_bind
))
14317 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
14320 if (gimple_omp_for_combined_into_p (stmt
))
14322 extract_omp_for_data (stmt
, &fd
, NULL
);
14325 /* We need two temporaries with fd.loop.v type (istart/iend)
14326 and then (fd.collapse - 1) temporaries with the same
14327 type for count2 ... countN-1 vars if not constant. */
14329 tree type
= fd
.iter_type
;
14330 if (fd
.collapse
> 1
14331 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
14332 count
+= fd
.collapse
- 1;
14334 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
14335 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
14336 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
14337 tree clauses
= *pc
;
14340 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
14341 OMP_CLAUSE__LOOPTEMP_
);
14342 for (i
= 0; i
< count
; i
++)
14347 gcc_assert (outerc
);
14348 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
14349 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
14350 OMP_CLAUSE__LOOPTEMP_
);
14354 temp
= create_tmp_var (type
);
14355 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
14357 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
14358 OMP_CLAUSE_DECL (*pc
) = temp
;
14359 pc
= &OMP_CLAUSE_CHAIN (*pc
);
14364 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
14367 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
14369 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
14371 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14373 /* Lower the header expressions. At this point, we can assume that
14374 the header is of the form:
14376 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
14378 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
14379 using the .omp_data_s mapping, if needed. */
14380 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
14382 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
14383 if (!is_gimple_min_invariant (*rhs_p
))
14384 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14386 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
14387 if (!is_gimple_min_invariant (*rhs_p
))
14388 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14390 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
14391 if (!is_gimple_min_invariant (*rhs_p
))
14392 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14395 /* Once lowered, extract the bounds and clauses. */
14396 extract_omp_for_data (stmt
, &fd
, NULL
);
14398 if (is_gimple_omp_oacc (ctx
->stmt
)
14399 && !ctx_in_oacc_kernels_region (ctx
))
14400 lower_oacc_head_tail (gimple_location (stmt
),
14401 gimple_omp_for_clauses (stmt
),
14402 &oacc_head
, &oacc_tail
, ctx
);
14404 /* Add OpenACC partitioning markers just before the loop */
14406 gimple_seq_add_seq (&body
, oacc_head
);
14408 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
14410 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
14411 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14412 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
14413 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
14415 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
14416 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
14417 OMP_CLAUSE_LINEAR_STEP (c
)
14418 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
14422 gimple_seq_add_stmt (&body
, stmt
);
14423 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
14425 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
14428 /* After the loop, add exit clauses. */
14429 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
14431 if (ctx
->cancellable
)
14432 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
14434 gimple_seq_add_seq (&body
, dlist
);
14436 body
= maybe_catch_exception (body
);
14438 /* Region exit marker goes at the end of the loop body. */
14439 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
14440 maybe_add_implicit_barrier_cancel (ctx
, &body
);
14442 /* Add OpenACC joining and reduction markers just after the loop. */
14444 gimple_seq_add_seq (&body
, oacc_tail
);
14446 pop_gimplify_context (new_stmt
);
14448 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14449 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
14450 if (BLOCK_VARS (block
))
14451 TREE_USED (block
) = 1;
14453 gimple_bind_set_body (new_stmt
, body
);
14454 gimple_omp_set_body (stmt
, NULL
);
14455 gimple_omp_for_set_pre_body (stmt
, NULL
);
14458 /* Callback for walk_stmts. Check if the current statement only contains
14459 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
14462 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
14463 bool *handled_ops_p
,
14464 struct walk_stmt_info
*wi
)
14466 int *info
= (int *) wi
->info
;
14467 gimple
*stmt
= gsi_stmt (*gsi_p
);
14469 *handled_ops_p
= true;
14470 switch (gimple_code (stmt
))
14474 case GIMPLE_OMP_FOR
:
14475 case GIMPLE_OMP_SECTIONS
:
14476 *info
= *info
== 0 ? 1 : -1;
14485 struct omp_taskcopy_context
14487 /* This field must be at the beginning, as we do "inheritance": Some
14488 callback functions for tree-inline.c (e.g., omp_copy_decl)
14489 receive a copy_body_data pointer that is up-casted to an
14490 omp_context pointer. */
14496 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
14498 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
14500 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
14501 return create_tmp_var (TREE_TYPE (var
));
14507 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
14509 tree name
, new_fields
= NULL
, type
, f
;
14511 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
14512 name
= DECL_NAME (TYPE_NAME (orig_type
));
14513 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
14514 TYPE_DECL
, name
, type
);
14515 TYPE_NAME (type
) = name
;
14517 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
14519 tree new_f
= copy_node (f
);
14520 DECL_CONTEXT (new_f
) = type
;
14521 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
14522 TREE_CHAIN (new_f
) = new_fields
;
14523 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14524 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14525 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
14527 new_fields
= new_f
;
14528 tcctx
->cb
.decl_map
->put (f
, new_f
);
14530 TYPE_FIELDS (type
) = nreverse (new_fields
);
14531 layout_type (type
);
14535 /* Create task copyfn. */
14538 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
14540 struct function
*child_cfun
;
14541 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
14542 tree record_type
, srecord_type
, bind
, list
;
14543 bool record_needs_remap
= false, srecord_needs_remap
= false;
14545 struct omp_taskcopy_context tcctx
;
14546 location_t loc
= gimple_location (task_stmt
);
14548 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
14549 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
14550 gcc_assert (child_cfun
->cfg
== NULL
);
14551 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
14553 /* Reset DECL_CONTEXT on function arguments. */
14554 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
14555 DECL_CONTEXT (t
) = child_fn
;
14557 /* Populate the function. */
14558 push_gimplify_context ();
14559 push_cfun (child_cfun
);
14561 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
14562 TREE_SIDE_EFFECTS (bind
) = 1;
14564 DECL_SAVED_TREE (child_fn
) = bind
;
14565 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
14567 /* Remap src and dst argument types if needed. */
14568 record_type
= ctx
->record_type
;
14569 srecord_type
= ctx
->srecord_type
;
14570 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
14571 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14573 record_needs_remap
= true;
14576 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
14577 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14579 srecord_needs_remap
= true;
14583 if (record_needs_remap
|| srecord_needs_remap
)
14585 memset (&tcctx
, '\0', sizeof (tcctx
));
14586 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
14587 tcctx
.cb
.dst_fn
= child_fn
;
14588 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
14589 gcc_checking_assert (tcctx
.cb
.src_node
);
14590 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
14591 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
14592 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
14593 tcctx
.cb
.eh_lp_nr
= 0;
14594 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
14595 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
14598 if (record_needs_remap
)
14599 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
14600 if (srecord_needs_remap
)
14601 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
14604 tcctx
.cb
.decl_map
= NULL
;
14606 arg
= DECL_ARGUMENTS (child_fn
);
14607 TREE_TYPE (arg
) = build_pointer_type (record_type
);
14608 sarg
= DECL_CHAIN (arg
);
14609 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
14611 /* First pass: initialize temporaries used in record_type and srecord_type
14612 sizes and field offsets. */
14613 if (tcctx
.cb
.decl_map
)
14614 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14615 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14619 decl
= OMP_CLAUSE_DECL (c
);
14620 p
= tcctx
.cb
.decl_map
->get (decl
);
14623 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14624 sf
= (tree
) n
->value
;
14625 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14626 src
= build_simple_mem_ref_loc (loc
, sarg
);
14627 src
= omp_build_component_ref (src
, sf
);
14628 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
14629 append_to_statement_list (t
, &list
);
14632 /* Second pass: copy shared var pointers and copy construct non-VLA
14633 firstprivate vars. */
14634 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14635 switch (OMP_CLAUSE_CODE (c
))
14637 splay_tree_key key
;
14638 case OMP_CLAUSE_SHARED
:
14639 decl
= OMP_CLAUSE_DECL (c
);
14640 key
= (splay_tree_key
) decl
;
14641 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
14642 key
= (splay_tree_key
) &DECL_UID (decl
);
14643 n
= splay_tree_lookup (ctx
->field_map
, key
);
14646 f
= (tree
) n
->value
;
14647 if (tcctx
.cb
.decl_map
)
14648 f
= *tcctx
.cb
.decl_map
->get (f
);
14649 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
14650 sf
= (tree
) n
->value
;
14651 if (tcctx
.cb
.decl_map
)
14652 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14653 src
= build_simple_mem_ref_loc (loc
, sarg
);
14654 src
= omp_build_component_ref (src
, sf
);
14655 dst
= build_simple_mem_ref_loc (loc
, arg
);
14656 dst
= omp_build_component_ref (dst
, f
);
14657 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14658 append_to_statement_list (t
, &list
);
14660 case OMP_CLAUSE_FIRSTPRIVATE
:
14661 decl
= OMP_CLAUSE_DECL (c
);
14662 if (is_variable_sized (decl
))
14664 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14667 f
= (tree
) n
->value
;
14668 if (tcctx
.cb
.decl_map
)
14669 f
= *tcctx
.cb
.decl_map
->get (f
);
14670 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14673 sf
= (tree
) n
->value
;
14674 if (tcctx
.cb
.decl_map
)
14675 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14676 src
= build_simple_mem_ref_loc (loc
, sarg
);
14677 src
= omp_build_component_ref (src
, sf
);
14678 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
14679 src
= build_simple_mem_ref_loc (loc
, src
);
14683 dst
= build_simple_mem_ref_loc (loc
, arg
);
14684 dst
= omp_build_component_ref (dst
, f
);
14685 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14686 append_to_statement_list (t
, &list
);
14688 case OMP_CLAUSE_PRIVATE
:
14689 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
14691 decl
= OMP_CLAUSE_DECL (c
);
14692 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14693 f
= (tree
) n
->value
;
14694 if (tcctx
.cb
.decl_map
)
14695 f
= *tcctx
.cb
.decl_map
->get (f
);
14696 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14699 sf
= (tree
) n
->value
;
14700 if (tcctx
.cb
.decl_map
)
14701 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14702 src
= build_simple_mem_ref_loc (loc
, sarg
);
14703 src
= omp_build_component_ref (src
, sf
);
14704 if (use_pointer_for_field (decl
, NULL
))
14705 src
= build_simple_mem_ref_loc (loc
, src
);
14709 dst
= build_simple_mem_ref_loc (loc
, arg
);
14710 dst
= omp_build_component_ref (dst
, f
);
14711 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14712 append_to_statement_list (t
, &list
);
14718 /* Last pass: handle VLA firstprivates. */
14719 if (tcctx
.cb
.decl_map
)
14720 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14721 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14725 decl
= OMP_CLAUSE_DECL (c
);
14726 if (!is_variable_sized (decl
))
14728 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14731 f
= (tree
) n
->value
;
14732 f
= *tcctx
.cb
.decl_map
->get (f
);
14733 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
14734 ind
= DECL_VALUE_EXPR (decl
);
14735 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
14736 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
14737 n
= splay_tree_lookup (ctx
->sfield_map
,
14738 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14739 sf
= (tree
) n
->value
;
14740 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14741 src
= build_simple_mem_ref_loc (loc
, sarg
);
14742 src
= omp_build_component_ref (src
, sf
);
14743 src
= build_simple_mem_ref_loc (loc
, src
);
14744 dst
= build_simple_mem_ref_loc (loc
, arg
);
14745 dst
= omp_build_component_ref (dst
, f
);
14746 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14747 append_to_statement_list (t
, &list
);
14748 n
= splay_tree_lookup (ctx
->field_map
,
14749 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14750 df
= (tree
) n
->value
;
14751 df
= *tcctx
.cb
.decl_map
->get (df
);
14752 ptr
= build_simple_mem_ref_loc (loc
, arg
);
14753 ptr
= omp_build_component_ref (ptr
, df
);
14754 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
14755 build_fold_addr_expr_loc (loc
, dst
));
14756 append_to_statement_list (t
, &list
);
14759 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
14760 append_to_statement_list (t
, &list
);
14762 if (tcctx
.cb
.decl_map
)
14763 delete tcctx
.cb
.decl_map
;
14764 pop_gimplify_context (NULL
);
14765 BIND_EXPR_BODY (bind
) = list
;
14770 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
14774 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
14776 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
14777 gcc_assert (clauses
);
14778 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14779 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
14780 switch (OMP_CLAUSE_DEPEND_KIND (c
))
14782 case OMP_CLAUSE_DEPEND_IN
:
14785 case OMP_CLAUSE_DEPEND_OUT
:
14786 case OMP_CLAUSE_DEPEND_INOUT
:
14789 case OMP_CLAUSE_DEPEND_SOURCE
:
14790 case OMP_CLAUSE_DEPEND_SINK
:
14793 gcc_unreachable ();
14795 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
14796 tree array
= create_tmp_var (type
);
14797 TREE_ADDRESSABLE (array
) = 1;
14798 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
14800 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
14801 gimple_seq_add_stmt (iseq
, g
);
14802 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
14804 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
14805 gimple_seq_add_stmt (iseq
, g
);
14806 for (i
= 0; i
< 2; i
++)
14808 if ((i
? n_in
: n_out
) == 0)
14810 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14811 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14812 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
14814 tree t
= OMP_CLAUSE_DECL (c
);
14815 t
= fold_convert (ptr_type_node
, t
);
14816 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
14817 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
14818 NULL_TREE
, NULL_TREE
);
14819 g
= gimple_build_assign (r
, t
);
14820 gimple_seq_add_stmt (iseq
, g
);
14823 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
14824 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
14825 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
14827 tree clobber
= build_constructor (type
, NULL
);
14828 TREE_THIS_VOLATILE (clobber
) = 1;
14829 g
= gimple_build_assign (array
, clobber
);
14830 gimple_seq_add_stmt (oseq
, g
);
14833 /* Lower the OpenMP parallel or task directive in the current statement
14834 in GSI_P. CTX holds context information for the directive. */
14837 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14841 gimple
*stmt
= gsi_stmt (*gsi_p
);
14842 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
14843 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
14844 location_t loc
= gimple_location (stmt
);
14846 clauses
= gimple_omp_taskreg_clauses (stmt
);
14848 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
14849 par_body
= gimple_bind_body (par_bind
);
14850 child_fn
= ctx
->cb
.dst_fn
;
14851 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
14852 && !gimple_omp_parallel_combined_p (stmt
))
14854 struct walk_stmt_info wi
;
14857 memset (&wi
, 0, sizeof (wi
));
14859 wi
.val_only
= true;
14860 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
14862 gimple_omp_parallel_set_combined_p (stmt
, true);
14864 gimple_seq dep_ilist
= NULL
;
14865 gimple_seq dep_olist
= NULL
;
14866 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
14867 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14869 push_gimplify_context ();
14870 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14871 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
14872 &dep_ilist
, &dep_olist
);
14875 if (ctx
->srecord_type
)
14876 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
14878 push_gimplify_context ();
14883 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
14884 lower_omp (&par_body
, ctx
);
14885 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
14886 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
14888 /* Declare all the variables created by mapping and the variables
14889 declared in the scope of the parallel body. */
14890 record_vars_into (ctx
->block_vars
, child_fn
);
14891 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
14893 if (ctx
->record_type
)
14896 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
14897 : ctx
->record_type
, ".omp_data_o");
14898 DECL_NAMELESS (ctx
->sender_decl
) = 1;
14899 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
14900 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
14905 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
14906 lower_send_shared_vars (&ilist
, &olist
, ctx
);
14908 if (ctx
->record_type
)
14910 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
14911 TREE_THIS_VOLATILE (clobber
) = 1;
14912 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
14916 /* Once all the expansions are done, sequence all the different
14917 fragments inside gimple_omp_body. */
14921 if (ctx
->record_type
)
14923 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14924 /* fixup_child_record_type might have changed receiver_decl's type. */
14925 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
14926 gimple_seq_add_stmt (&new_body
,
14927 gimple_build_assign (ctx
->receiver_decl
, t
));
14930 gimple_seq_add_seq (&new_body
, par_ilist
);
14931 gimple_seq_add_seq (&new_body
, par_body
);
14932 gimple_seq_add_seq (&new_body
, par_rlist
);
14933 if (ctx
->cancellable
)
14934 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14935 gimple_seq_add_seq (&new_body
, par_olist
);
14936 new_body
= maybe_catch_exception (new_body
);
14937 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
14938 gimple_seq_add_stmt (&new_body
,
14939 gimple_build_omp_continue (integer_zero_node
,
14940 integer_zero_node
));
14941 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
14942 gimple_omp_set_body (stmt
, new_body
);
14944 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
14945 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
14946 gimple_bind_add_seq (bind
, ilist
);
14947 gimple_bind_add_stmt (bind
, stmt
);
14948 gimple_bind_add_seq (bind
, olist
);
14950 pop_gimplify_context (NULL
);
14954 gimple_bind_add_seq (dep_bind
, dep_ilist
);
14955 gimple_bind_add_stmt (dep_bind
, bind
);
14956 gimple_bind_add_seq (dep_bind
, dep_olist
);
14957 pop_gimplify_context (dep_bind
);
14961 /* Lower the GIMPLE_OMP_TARGET in the current statement
14962 in GSI_P. CTX holds context information for the directive. */
14965 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14968 tree child_fn
, t
, c
;
14969 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
14970 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
14971 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
14972 location_t loc
= gimple_location (stmt
);
14973 bool offloaded
, data_region
;
14974 unsigned int map_cnt
= 0;
14975 bool has_depend
= false;
14977 offloaded
= is_gimple_omp_offloaded (stmt
);
14978 switch (gimple_omp_target_kind (stmt
))
14980 case GF_OMP_TARGET_KIND_REGION
:
14981 case GF_OMP_TARGET_KIND_UPDATE
:
14982 case GF_OMP_TARGET_KIND_ENTER_DATA
:
14983 case GF_OMP_TARGET_KIND_EXIT_DATA
:
14984 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
14985 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
14986 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
14987 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
14988 data_region
= false;
14990 case GF_OMP_TARGET_KIND_DATA
:
14991 case GF_OMP_TARGET_KIND_OACC_DATA
:
14992 data_region
= true;
14995 gcc_unreachable ();
14998 clauses
= gimple_omp_target_clauses (stmt
);
15000 gimple_seq dep_ilist
= NULL
;
15001 gimple_seq dep_olist
= NULL
;
15002 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15004 push_gimplify_context ();
15005 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15006 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15007 &dep_ilist
, &dep_olist
);
15015 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15016 tgt_body
= gimple_bind_body (tgt_bind
);
15018 else if (data_region
)
15019 tgt_body
= gimple_omp_body (stmt
);
15020 child_fn
= ctx
->cb
.dst_fn
;
15022 push_gimplify_context ();
15027 && is_gimple_omp_oacc (stmt
))
15028 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
15030 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15031 switch (OMP_CLAUSE_CODE (c
))
15037 case OMP_CLAUSE_MAP
:
15039 /* First check what we're prepared to handle in the following. */
15040 switch (OMP_CLAUSE_MAP_KIND (c
))
15042 case GOMP_MAP_ALLOC
:
15044 case GOMP_MAP_FROM
:
15045 case GOMP_MAP_TOFROM
:
15046 case GOMP_MAP_POINTER
:
15047 case GOMP_MAP_TO_PSET
:
15048 case GOMP_MAP_FORCE_DEALLOC
:
15049 case GOMP_MAP_RELEASE
:
15050 case GOMP_MAP_ALWAYS_TO
:
15051 case GOMP_MAP_ALWAYS_FROM
:
15052 case GOMP_MAP_ALWAYS_TOFROM
:
15053 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15054 case GOMP_MAP_STRUCT
:
15056 case GOMP_MAP_FORCE_ALLOC
:
15057 case GOMP_MAP_FORCE_TO
:
15058 case GOMP_MAP_FORCE_FROM
:
15059 case GOMP_MAP_FORCE_TOFROM
:
15060 case GOMP_MAP_FORCE_PRESENT
:
15061 case GOMP_MAP_FORCE_DEVICEPTR
:
15062 gcc_assert (is_gimple_omp_oacc (stmt
));
15065 gcc_unreachable ();
15069 case OMP_CLAUSE_TO
:
15070 case OMP_CLAUSE_FROM
:
15071 var
= OMP_CLAUSE_DECL (c
);
15074 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15075 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15076 && (OMP_CLAUSE_MAP_KIND (c
)
15077 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15082 if (DECL_SIZE (var
)
15083 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15085 tree var2
= DECL_VALUE_EXPR (var
);
15086 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15087 var2
= TREE_OPERAND (var2
, 0);
15088 gcc_assert (DECL_P (var2
));
15093 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
15095 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15097 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15098 && varpool_node::get_create (var
)->offloadable
)
15101 tree type
= build_pointer_type (TREE_TYPE (var
));
15102 tree new_var
= lookup_decl (var
, ctx
);
15103 x
= create_tmp_var_raw (type
, get_name (new_var
));
15104 gimple_add_tmp_var (x
);
15105 x
= build_simple_mem_ref (x
);
15106 SET_DECL_VALUE_EXPR (new_var
, x
);
15107 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15112 if (offloaded
&& OMP_CLAUSE_MAP_PRIVATE (c
))
15118 if (!maybe_lookup_field (var
, ctx
))
15123 x
= build_receiver_ref (var
, true, ctx
);
15124 tree new_var
= lookup_decl (var
, ctx
);
15126 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15127 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15128 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15129 x
= build_simple_mem_ref (x
);
15130 SET_DECL_VALUE_EXPR (new_var
, x
);
15131 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15136 case OMP_CLAUSE_FIRSTPRIVATE
:
15138 var
= OMP_CLAUSE_DECL (c
);
15139 if (!is_reference (var
)
15140 && !is_gimple_reg_type (TREE_TYPE (var
)))
15142 tree new_var
= lookup_decl (var
, ctx
);
15143 if (is_variable_sized (var
))
15145 tree pvar
= DECL_VALUE_EXPR (var
);
15146 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15147 pvar
= TREE_OPERAND (pvar
, 0);
15148 gcc_assert (DECL_P (pvar
));
15149 tree new_pvar
= lookup_decl (pvar
, ctx
);
15150 x
= build_fold_indirect_ref (new_pvar
);
15151 TREE_THIS_NOTRAP (x
) = 1;
15154 x
= build_receiver_ref (var
, true, ctx
);
15155 SET_DECL_VALUE_EXPR (new_var
, x
);
15156 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15160 case OMP_CLAUSE_PRIVATE
:
15161 var
= OMP_CLAUSE_DECL (c
);
15162 if (is_variable_sized (var
))
15164 tree new_var
= lookup_decl (var
, ctx
);
15165 tree pvar
= DECL_VALUE_EXPR (var
);
15166 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15167 pvar
= TREE_OPERAND (pvar
, 0);
15168 gcc_assert (DECL_P (pvar
));
15169 tree new_pvar
= lookup_decl (pvar
, ctx
);
15170 x
= build_fold_indirect_ref (new_pvar
);
15171 TREE_THIS_NOTRAP (x
) = 1;
15172 SET_DECL_VALUE_EXPR (new_var
, x
);
15173 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15177 case OMP_CLAUSE_USE_DEVICE_PTR
:
15178 case OMP_CLAUSE_IS_DEVICE_PTR
:
15179 var
= OMP_CLAUSE_DECL (c
);
15181 if (is_variable_sized (var
))
15183 tree new_var
= lookup_decl (var
, ctx
);
15184 tree pvar
= DECL_VALUE_EXPR (var
);
15185 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15186 pvar
= TREE_OPERAND (pvar
, 0);
15187 gcc_assert (DECL_P (pvar
));
15188 tree new_pvar
= lookup_decl (pvar
, ctx
);
15189 x
= build_fold_indirect_ref (new_pvar
);
15190 TREE_THIS_NOTRAP (x
) = 1;
15191 SET_DECL_VALUE_EXPR (new_var
, x
);
15192 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15194 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15196 tree new_var
= lookup_decl (var
, ctx
);
15197 tree type
= build_pointer_type (TREE_TYPE (var
));
15198 x
= create_tmp_var_raw (type
, get_name (new_var
));
15199 gimple_add_tmp_var (x
);
15200 x
= build_simple_mem_ref (x
);
15201 SET_DECL_VALUE_EXPR (new_var
, x
);
15202 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15209 target_nesting_level
++;
15210 lower_omp (&tgt_body
, ctx
);
15211 target_nesting_level
--;
15213 else if (data_region
)
15214 lower_omp (&tgt_body
, ctx
);
15218 /* Declare all the variables created by mapping and the variables
15219 declared in the scope of the target body. */
15220 record_vars_into (ctx
->block_vars
, child_fn
);
15221 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
15226 if (ctx
->record_type
)
15229 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
15230 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15231 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15232 t
= make_tree_vec (3);
15233 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
15234 TREE_VEC_ELT (t
, 1)
15235 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
15236 ".omp_data_sizes");
15237 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
15238 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
15239 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
15240 tree tkind_type
= short_unsigned_type_node
;
15241 int talign_shift
= 8;
15242 TREE_VEC_ELT (t
, 2)
15243 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
15244 ".omp_data_kinds");
15245 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
15246 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
15247 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
15248 gimple_omp_target_set_data_arg (stmt
, t
);
15250 vec
<constructor_elt
, va_gc
> *vsize
;
15251 vec
<constructor_elt
, va_gc
> *vkind
;
15252 vec_alloc (vsize
, map_cnt
);
15253 vec_alloc (vkind
, map_cnt
);
15254 unsigned int map_idx
= 0;
15256 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15257 switch (OMP_CLAUSE_CODE (c
))
15259 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
15260 unsigned int talign
;
15264 case OMP_CLAUSE_MAP
:
15265 case OMP_CLAUSE_TO
:
15266 case OMP_CLAUSE_FROM
:
15268 ovar
= OMP_CLAUSE_DECL (c
);
15269 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15270 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
15272 if (!DECL_P (ovar
))
15274 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15275 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
15277 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
15278 == get_base_address (ovar
));
15279 nc
= OMP_CLAUSE_CHAIN (c
);
15280 ovar
= OMP_CLAUSE_DECL (nc
);
15284 tree x
= build_sender_ref (ovar
, ctx
);
15286 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
15287 gimplify_assign (x
, v
, &ilist
);
15293 if (DECL_SIZE (ovar
)
15294 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
15296 tree ovar2
= DECL_VALUE_EXPR (ovar
);
15297 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
15298 ovar2
= TREE_OPERAND (ovar2
, 0);
15299 gcc_assert (DECL_P (ovar2
));
15302 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15303 && OMP_CLAUSE_MAP_PRIVATE (c
))
15305 if (!maybe_lookup_field ((splay_tree_key
) &DECL_UID (ovar
),
15309 else if (!maybe_lookup_field (ovar
, ctx
))
15313 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
15314 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
15315 talign
= DECL_ALIGN_UNIT (ovar
);
15318 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15319 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15320 && OMP_CLAUSE_MAP_PRIVATE (c
))
15321 x
= build_sender_ref ((splay_tree_key
) &DECL_UID (ovar
),
15324 x
= build_sender_ref (ovar
, ctx
);
15325 if (maybe_lookup_oacc_reduction (var
, ctx
))
15327 gcc_checking_assert (offloaded
15328 && is_gimple_omp_oacc (stmt
));
15329 gimplify_assign (x
, var
, &ilist
);
15331 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15332 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15333 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15334 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
15336 gcc_assert (offloaded
);
15338 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
15339 mark_addressable (avar
);
15340 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
15341 talign
= DECL_ALIGN_UNIT (avar
);
15342 avar
= build_fold_addr_expr (avar
);
15343 gimplify_assign (x
, avar
, &ilist
);
15345 else if (is_gimple_reg (var
))
15347 gcc_assert (offloaded
);
15348 tree avar
= create_tmp_var (TREE_TYPE (var
));
15349 mark_addressable (avar
);
15350 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
15351 if (GOMP_MAP_COPY_TO_P (map_kind
)
15352 || map_kind
== GOMP_MAP_POINTER
15353 || map_kind
== GOMP_MAP_TO_PSET
15354 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15355 gimplify_assign (avar
, var
, &ilist
);
15356 avar
= build_fold_addr_expr (avar
);
15357 gimplify_assign (x
, avar
, &ilist
);
15358 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
15359 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15360 && !TYPE_READONLY (TREE_TYPE (var
)))
15362 x
= unshare_expr (x
);
15363 x
= build_simple_mem_ref (x
);
15364 gimplify_assign (var
, x
, &olist
);
15369 var
= build_fold_addr_expr (var
);
15370 gimplify_assign (x
, var
, &ilist
);
15373 s
= OMP_CLAUSE_SIZE (c
);
15374 if (s
== NULL_TREE
)
15375 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15376 s
= fold_convert (size_type_node
, s
);
15377 purpose
= size_int (map_idx
++);
15378 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15379 if (TREE_CODE (s
) != INTEGER_CST
)
15380 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15382 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
15383 switch (OMP_CLAUSE_CODE (c
))
15385 case OMP_CLAUSE_MAP
:
15386 tkind
= OMP_CLAUSE_MAP_KIND (c
);
15387 tkind_zero
= tkind
;
15388 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
15391 case GOMP_MAP_ALLOC
:
15393 case GOMP_MAP_FROM
:
15394 case GOMP_MAP_TOFROM
:
15395 case GOMP_MAP_ALWAYS_TO
:
15396 case GOMP_MAP_ALWAYS_FROM
:
15397 case GOMP_MAP_ALWAYS_TOFROM
:
15398 case GOMP_MAP_RELEASE
:
15399 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
15401 case GOMP_MAP_DELETE
:
15402 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
15406 if (tkind_zero
!= tkind
)
15408 if (integer_zerop (s
))
15409 tkind
= tkind_zero
;
15410 else if (integer_nonzerop (s
))
15411 tkind_zero
= tkind
;
15414 case OMP_CLAUSE_TO
:
15415 tkind
= GOMP_MAP_TO
;
15416 tkind_zero
= tkind
;
15418 case OMP_CLAUSE_FROM
:
15419 tkind
= GOMP_MAP_FROM
;
15420 tkind_zero
= tkind
;
15423 gcc_unreachable ();
15425 gcc_checking_assert (tkind
15426 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15427 gcc_checking_assert (tkind_zero
15428 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15429 talign
= ceil_log2 (talign
);
15430 tkind
|= talign
<< talign_shift
;
15431 tkind_zero
|= talign
<< talign_shift
;
15432 gcc_checking_assert (tkind
15433 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15434 gcc_checking_assert (tkind_zero
15435 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15436 if (tkind
== tkind_zero
)
15437 x
= build_int_cstu (tkind_type
, tkind
);
15440 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
15441 x
= build3 (COND_EXPR
, tkind_type
,
15442 fold_build2 (EQ_EXPR
, boolean_type_node
,
15443 unshare_expr (s
), size_zero_node
),
15444 build_int_cstu (tkind_type
, tkind_zero
),
15445 build_int_cstu (tkind_type
, tkind
));
15447 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
15452 case OMP_CLAUSE_FIRSTPRIVATE
:
15453 ovar
= OMP_CLAUSE_DECL (c
);
15454 if (is_reference (ovar
))
15455 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15457 talign
= DECL_ALIGN_UNIT (ovar
);
15458 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15459 x
= build_sender_ref (ovar
, ctx
);
15460 tkind
= GOMP_MAP_FIRSTPRIVATE
;
15461 type
= TREE_TYPE (ovar
);
15462 if (is_reference (ovar
))
15463 type
= TREE_TYPE (type
);
15464 bool use_firstprivate_int
, force_addr
;
15465 use_firstprivate_int
= false;
15466 force_addr
= false;
15467 if ((INTEGRAL_TYPE_P (type
)
15468 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15469 || TREE_CODE (type
) == POINTER_TYPE
)
15470 use_firstprivate_int
= true;
15473 if (is_reference (var
))
15474 use_firstprivate_int
= false;
15475 else if (is_gimple_reg (var
))
15477 if (DECL_HAS_VALUE_EXPR_P (var
))
15479 tree v
= get_base_address (var
);
15480 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15482 use_firstprivate_int
= false;
15486 switch (TREE_CODE (v
))
15490 use_firstprivate_int
= false;
15499 use_firstprivate_int
= false;
15501 if (use_firstprivate_int
)
15503 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15505 if (is_reference (var
))
15506 t
= build_simple_mem_ref (var
);
15507 if (TREE_CODE (type
) != POINTER_TYPE
)
15508 t
= fold_convert (pointer_sized_int_node
, t
);
15509 t
= fold_convert (TREE_TYPE (x
), t
);
15510 gimplify_assign (x
, t
, &ilist
);
15512 else if (is_reference (var
))
15513 gimplify_assign (x
, var
, &ilist
);
15514 else if (!force_addr
&& is_gimple_reg (var
))
15516 tree avar
= create_tmp_var (TREE_TYPE (var
));
15517 mark_addressable (avar
);
15518 gimplify_assign (avar
, var
, &ilist
);
15519 avar
= build_fold_addr_expr (avar
);
15520 gimplify_assign (x
, avar
, &ilist
);
15524 var
= build_fold_addr_expr (var
);
15525 gimplify_assign (x
, var
, &ilist
);
15527 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
15529 else if (is_reference (var
))
15530 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15532 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15533 s
= fold_convert (size_type_node
, s
);
15534 purpose
= size_int (map_idx
++);
15535 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15536 if (TREE_CODE (s
) != INTEGER_CST
)
15537 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15539 gcc_checking_assert (tkind
15540 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15541 talign
= ceil_log2 (talign
);
15542 tkind
|= talign
<< talign_shift
;
15543 gcc_checking_assert (tkind
15544 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15545 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15546 build_int_cstu (tkind_type
, tkind
));
15549 case OMP_CLAUSE_USE_DEVICE_PTR
:
15550 case OMP_CLAUSE_IS_DEVICE_PTR
:
15551 ovar
= OMP_CLAUSE_DECL (c
);
15552 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15553 x
= build_sender_ref (ovar
, ctx
);
15554 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15555 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
15557 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15558 type
= TREE_TYPE (ovar
);
15559 if (TREE_CODE (type
) == ARRAY_TYPE
)
15560 var
= build_fold_addr_expr (var
);
15563 if (is_reference (ovar
))
15565 type
= TREE_TYPE (type
);
15566 if (TREE_CODE (type
) != ARRAY_TYPE
)
15567 var
= build_simple_mem_ref (var
);
15568 var
= fold_convert (TREE_TYPE (x
), var
);
15571 gimplify_assign (x
, var
, &ilist
);
15573 purpose
= size_int (map_idx
++);
15574 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15575 gcc_checking_assert (tkind
15576 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15577 gcc_checking_assert (tkind
15578 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15579 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15580 build_int_cstu (tkind_type
, tkind
));
15584 gcc_assert (map_idx
== map_cnt
);
15586 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
15587 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
15588 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
15589 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
15590 for (int i
= 1; i
<= 2; i
++)
15591 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
15593 gimple_seq initlist
= NULL
;
15594 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
15595 TREE_VEC_ELT (t
, i
)),
15596 &initlist
, true, NULL_TREE
);
15597 gimple_seq_add_seq (&ilist
, initlist
);
15599 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
15601 TREE_THIS_VOLATILE (clobber
) = 1;
15602 gimple_seq_add_stmt (&olist
,
15603 gimple_build_assign (TREE_VEC_ELT (t
, i
),
15607 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
15608 TREE_THIS_VOLATILE (clobber
) = 1;
15609 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15613 /* Once all the expansions are done, sequence all the different
15614 fragments inside gimple_omp_body. */
15619 && ctx
->record_type
)
15621 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15622 /* fixup_child_record_type might have changed receiver_decl's type. */
15623 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15624 gimple_seq_add_stmt (&new_body
,
15625 gimple_build_assign (ctx
->receiver_decl
, t
));
15628 if (offloaded
|| data_region
)
15630 tree prev
= NULL_TREE
;
15631 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15632 switch (OMP_CLAUSE_CODE (c
))
15637 case OMP_CLAUSE_FIRSTPRIVATE
:
15638 var
= OMP_CLAUSE_DECL (c
);
15639 if (is_reference (var
)
15640 || is_gimple_reg_type (TREE_TYPE (var
)))
15642 tree new_var
= lookup_decl (var
, ctx
);
15644 type
= TREE_TYPE (var
);
15645 if (is_reference (var
))
15646 type
= TREE_TYPE (type
);
15647 bool use_firstprivate_int
;
15648 use_firstprivate_int
= false;
15649 if ((INTEGRAL_TYPE_P (type
)
15650 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15651 || TREE_CODE (type
) == POINTER_TYPE
)
15652 use_firstprivate_int
= true;
15655 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
15656 if (is_reference (v
))
15657 use_firstprivate_int
= false;
15658 else if (is_gimple_reg (v
))
15660 if (DECL_HAS_VALUE_EXPR_P (v
))
15662 v
= get_base_address (v
);
15663 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15664 use_firstprivate_int
= false;
15666 switch (TREE_CODE (v
))
15670 use_firstprivate_int
= false;
15678 use_firstprivate_int
= false;
15680 if (use_firstprivate_int
)
15682 x
= build_receiver_ref (var
, false, ctx
);
15683 if (TREE_CODE (type
) != POINTER_TYPE
)
15684 x
= fold_convert (pointer_sized_int_node
, x
);
15685 x
= fold_convert (type
, x
);
15686 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15688 if (is_reference (var
))
15690 tree v
= create_tmp_var_raw (type
, get_name (var
));
15691 gimple_add_tmp_var (v
);
15692 TREE_ADDRESSABLE (v
) = 1;
15693 gimple_seq_add_stmt (&new_body
,
15694 gimple_build_assign (v
, x
));
15695 x
= build_fold_addr_expr (v
);
15697 gimple_seq_add_stmt (&new_body
,
15698 gimple_build_assign (new_var
, x
));
15702 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
15703 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15705 gimple_seq_add_stmt (&new_body
,
15706 gimple_build_assign (new_var
, x
));
15709 else if (is_variable_sized (var
))
15711 tree pvar
= DECL_VALUE_EXPR (var
);
15712 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15713 pvar
= TREE_OPERAND (pvar
, 0);
15714 gcc_assert (DECL_P (pvar
));
15715 tree new_var
= lookup_decl (pvar
, ctx
);
15716 x
= build_receiver_ref (var
, false, ctx
);
15717 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15718 gimple_seq_add_stmt (&new_body
,
15719 gimple_build_assign (new_var
, x
));
15722 case OMP_CLAUSE_PRIVATE
:
15723 var
= OMP_CLAUSE_DECL (c
);
15724 if (is_reference (var
))
15726 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15727 tree new_var
= lookup_decl (var
, ctx
);
15728 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
15729 if (TREE_CONSTANT (x
))
15731 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
15733 gimple_add_tmp_var (x
);
15734 TREE_ADDRESSABLE (x
) = 1;
15735 x
= build_fold_addr_expr_loc (clause_loc
, x
);
15740 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15741 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
15742 tree al
= size_int (TYPE_ALIGN (rtype
));
15743 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15746 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15747 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15748 gimple_seq_add_stmt (&new_body
,
15749 gimple_build_assign (new_var
, x
));
15752 case OMP_CLAUSE_USE_DEVICE_PTR
:
15753 case OMP_CLAUSE_IS_DEVICE_PTR
:
15754 var
= OMP_CLAUSE_DECL (c
);
15755 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15756 x
= build_sender_ref (var
, ctx
);
15758 x
= build_receiver_ref (var
, false, ctx
);
15759 if (is_variable_sized (var
))
15761 tree pvar
= DECL_VALUE_EXPR (var
);
15762 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15763 pvar
= TREE_OPERAND (pvar
, 0);
15764 gcc_assert (DECL_P (pvar
));
15765 tree new_var
= lookup_decl (pvar
, ctx
);
15766 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15767 gimple_seq_add_stmt (&new_body
,
15768 gimple_build_assign (new_var
, x
));
15770 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15772 tree new_var
= lookup_decl (var
, ctx
);
15773 new_var
= DECL_VALUE_EXPR (new_var
);
15774 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
15775 new_var
= TREE_OPERAND (new_var
, 0);
15776 gcc_assert (DECL_P (new_var
));
15777 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15778 gimple_seq_add_stmt (&new_body
,
15779 gimple_build_assign (new_var
, x
));
15783 tree type
= TREE_TYPE (var
);
15784 tree new_var
= lookup_decl (var
, ctx
);
15785 if (is_reference (var
))
15787 type
= TREE_TYPE (type
);
15788 if (TREE_CODE (type
) != ARRAY_TYPE
)
15790 tree v
= create_tmp_var_raw (type
, get_name (var
));
15791 gimple_add_tmp_var (v
);
15792 TREE_ADDRESSABLE (v
) = 1;
15793 x
= fold_convert (type
, x
);
15794 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15796 gimple_seq_add_stmt (&new_body
,
15797 gimple_build_assign (v
, x
));
15798 x
= build_fold_addr_expr (v
);
15801 x
= fold_convert (TREE_TYPE (new_var
), x
);
15802 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15803 gimple_seq_add_stmt (&new_body
,
15804 gimple_build_assign (new_var
, x
));
15808 /* Handle GOMP_MAP_FIRSTPRIVATE_POINTER in second pass,
15809 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
15810 are already handled. */
15811 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15812 switch (OMP_CLAUSE_CODE (c
))
15817 case OMP_CLAUSE_MAP
:
15818 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
15820 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15821 HOST_WIDE_INT offset
= 0;
15823 var
= OMP_CLAUSE_DECL (c
);
15825 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
15826 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
15828 && varpool_node::get_create (var
)->offloadable
)
15830 if (TREE_CODE (var
) == INDIRECT_REF
15831 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
15832 var
= TREE_OPERAND (var
, 0);
15833 if (TREE_CODE (var
) == COMPONENT_REF
)
15835 var
= get_addr_base_and_unit_offset (var
, &offset
);
15836 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
15838 else if (DECL_SIZE (var
)
15839 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15841 tree var2
= DECL_VALUE_EXPR (var
);
15842 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15843 var2
= TREE_OPERAND (var2
, 0);
15844 gcc_assert (DECL_P (var2
));
15847 tree new_var
= lookup_decl (var
, ctx
), x
;
15848 tree type
= TREE_TYPE (new_var
);
15850 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
15851 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
15854 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
15856 new_var
= build2 (MEM_REF
, type
,
15857 build_fold_addr_expr (new_var
),
15858 build_int_cst (build_pointer_type (type
),
15861 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
15863 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
15864 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
15865 new_var
= build2 (MEM_REF
, type
,
15866 build_fold_addr_expr (new_var
),
15867 build_int_cst (build_pointer_type (type
),
15871 is_ref
= is_reference (var
);
15872 bool ref_to_array
= false;
15875 type
= TREE_TYPE (type
);
15876 if (TREE_CODE (type
) == ARRAY_TYPE
)
15878 type
= build_pointer_type (type
);
15879 ref_to_array
= true;
15882 else if (TREE_CODE (type
) == ARRAY_TYPE
)
15884 tree decl2
= DECL_VALUE_EXPR (new_var
);
15885 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
15886 decl2
= TREE_OPERAND (decl2
, 0);
15887 gcc_assert (DECL_P (decl2
));
15889 type
= TREE_TYPE (new_var
);
15891 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
15892 x
= fold_convert_loc (clause_loc
, type
, x
);
15893 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
15895 tree bias
= OMP_CLAUSE_SIZE (c
);
15897 bias
= lookup_decl (bias
, ctx
);
15898 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
15899 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
15901 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
15902 TREE_TYPE (x
), x
, bias
);
15905 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15906 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15907 if (is_ref
&& !ref_to_array
)
15909 tree t
= create_tmp_var_raw (type
, get_name (var
));
15910 gimple_add_tmp_var (t
);
15911 TREE_ADDRESSABLE (t
) = 1;
15912 gimple_seq_add_stmt (&new_body
,
15913 gimple_build_assign (t
, x
));
15914 x
= build_fold_addr_expr_loc (clause_loc
, t
);
15916 gimple_seq_add_stmt (&new_body
,
15917 gimple_build_assign (new_var
, x
));
15920 else if (OMP_CLAUSE_CHAIN (c
)
15921 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
15923 && OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15924 == GOMP_MAP_FIRSTPRIVATE_POINTER
)
15927 case OMP_CLAUSE_PRIVATE
:
15928 var
= OMP_CLAUSE_DECL (c
);
15929 if (is_variable_sized (var
))
15931 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15932 tree new_var
= lookup_decl (var
, ctx
);
15933 tree pvar
= DECL_VALUE_EXPR (var
);
15934 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15935 pvar
= TREE_OPERAND (pvar
, 0);
15936 gcc_assert (DECL_P (pvar
));
15937 tree new_pvar
= lookup_decl (pvar
, ctx
);
15938 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15939 tree al
= size_int (DECL_ALIGN (var
));
15940 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
15941 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15942 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
15943 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15944 gimple_seq_add_stmt (&new_body
,
15945 gimple_build_assign (new_pvar
, x
));
15949 gimple_seq_add_seq (&new_body
, tgt_body
);
15951 new_body
= maybe_catch_exception (new_body
);
15953 else if (data_region
)
15954 new_body
= tgt_body
;
15955 if (offloaded
|| data_region
)
15957 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15958 gimple_omp_set_body (stmt
, new_body
);
15961 bind
= gimple_build_bind (NULL
, NULL
,
15962 tgt_bind
? gimple_bind_block (tgt_bind
)
15964 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15965 gimple_bind_add_seq (bind
, irlist
);
15966 gimple_bind_add_seq (bind
, ilist
);
15967 gimple_bind_add_stmt (bind
, stmt
);
15968 gimple_bind_add_seq (bind
, olist
);
15969 gimple_bind_add_seq (bind
, orlist
);
15971 pop_gimplify_context (NULL
);
15975 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15976 gimple_bind_add_stmt (dep_bind
, bind
);
15977 gimple_bind_add_seq (dep_bind
, dep_olist
);
15978 pop_gimplify_context (dep_bind
);
15982 /* Expand code for an OpenMP teams directive. */
15985 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15987 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
15988 push_gimplify_context ();
15990 tree block
= make_node (BLOCK
);
15991 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
15992 gsi_replace (gsi_p
, bind
, true);
15993 gimple_seq bind_body
= NULL
;
15994 gimple_seq dlist
= NULL
;
15995 gimple_seq olist
= NULL
;
15997 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15998 OMP_CLAUSE_NUM_TEAMS
);
15999 if (num_teams
== NULL_TREE
)
16000 num_teams
= build_int_cst (unsigned_type_node
, 0);
16003 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16004 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16005 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16007 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16008 OMP_CLAUSE_THREAD_LIMIT
);
16009 if (thread_limit
== NULL_TREE
)
16010 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16013 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16014 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16015 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16019 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16020 &bind_body
, &dlist
, ctx
, NULL
);
16021 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16022 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16023 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16025 location_t loc
= gimple_location (teams_stmt
);
16026 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16027 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16028 gimple_set_location (call
, loc
);
16029 gimple_seq_add_stmt (&bind_body
, call
);
16031 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16032 gimple_omp_set_body (teams_stmt
, NULL
);
16033 gimple_seq_add_seq (&bind_body
, olist
);
16034 gimple_seq_add_seq (&bind_body
, dlist
);
16035 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16036 gimple_bind_set_body (bind
, bind_body
);
16038 pop_gimplify_context (bind
);
16040 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16041 BLOCK_VARS (block
) = ctx
->block_vars
;
16042 if (BLOCK_VARS (block
))
16043 TREE_USED (block
) = 1;
16047 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16048 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16049 of OMP context, but with task_shared_vars set. */
16052 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16057 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16058 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16061 if (task_shared_vars
16063 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16066 /* If a global variable has been privatized, TREE_CONSTANT on
16067 ADDR_EXPR might be wrong. */
16068 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16069 recompute_tree_invariant_for_addr_expr (t
);
16071 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16075 /* Data to be communicated between lower_omp_regimplify_operands and
16076 lower_omp_regimplify_operands_p. */
16078 struct lower_omp_regimplify_operands_data
16084 /* Helper function for lower_omp_regimplify_operands. Find
16085 omp_member_access_dummy_var vars and adjust temporarily their
16086 DECL_VALUE_EXPRs if needed. */
16089 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16092 tree t
= omp_member_access_dummy_var (*tp
);
16095 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16096 lower_omp_regimplify_operands_data
*ldata
16097 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16098 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16101 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16102 ldata
->decls
->safe_push (*tp
);
16103 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16104 SET_DECL_VALUE_EXPR (*tp
, v
);
16107 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16111 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16112 of omp_member_access_dummy_var vars during regimplification. */
16115 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16116 gimple_stmt_iterator
*gsi_p
)
16118 auto_vec
<tree
, 10> decls
;
16121 struct walk_stmt_info wi
;
16122 memset (&wi
, '\0', sizeof (wi
));
16123 struct lower_omp_regimplify_operands_data data
;
16125 data
.decls
= &decls
;
16127 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16129 gimple_regimplify_operands (stmt
, gsi_p
);
16130 while (!decls
.is_empty ())
16132 tree t
= decls
.pop ();
16133 tree v
= decls
.pop ();
16134 SET_DECL_VALUE_EXPR (t
, v
);
16139 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16141 gimple
*stmt
= gsi_stmt (*gsi_p
);
16142 struct walk_stmt_info wi
;
16145 if (gimple_has_location (stmt
))
16146 input_location
= gimple_location (stmt
);
16148 if (task_shared_vars
)
16149 memset (&wi
, '\0', sizeof (wi
));
16151 /* If we have issued syntax errors, avoid doing any heavy lifting.
16152 Just replace the OMP directives with a NOP to avoid
16153 confusing RTL expansion. */
16154 if (seen_error () && is_gimple_omp (stmt
))
16156 gsi_replace (gsi_p
, gimple_build_nop (), true);
16160 switch (gimple_code (stmt
))
16164 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16165 if ((ctx
|| task_shared_vars
)
16166 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16167 lower_omp_regimplify_p
,
16168 ctx
? NULL
: &wi
, NULL
)
16169 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16170 lower_omp_regimplify_p
,
16171 ctx
? NULL
: &wi
, NULL
)))
16172 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16176 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16178 case GIMPLE_EH_FILTER
:
16179 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16182 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16183 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16185 case GIMPLE_TRANSACTION
:
16186 lower_omp (gimple_transaction_body_ptr (
16187 as_a
<gtransaction
*> (stmt
)),
16191 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16193 case GIMPLE_OMP_PARALLEL
:
16194 case GIMPLE_OMP_TASK
:
16195 ctx
= maybe_lookup_ctx (stmt
);
16197 if (ctx
->cancellable
)
16198 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16199 lower_omp_taskreg (gsi_p
, ctx
);
16201 case GIMPLE_OMP_FOR
:
16202 ctx
= maybe_lookup_ctx (stmt
);
16204 if (ctx
->cancellable
)
16205 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16206 lower_omp_for (gsi_p
, ctx
);
16208 case GIMPLE_OMP_SECTIONS
:
16209 ctx
= maybe_lookup_ctx (stmt
);
16211 if (ctx
->cancellable
)
16212 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16213 lower_omp_sections (gsi_p
, ctx
);
16215 case GIMPLE_OMP_SINGLE
:
16216 ctx
= maybe_lookup_ctx (stmt
);
16218 lower_omp_single (gsi_p
, ctx
);
16220 case GIMPLE_OMP_MASTER
:
16221 ctx
= maybe_lookup_ctx (stmt
);
16223 lower_omp_master (gsi_p
, ctx
);
16225 case GIMPLE_OMP_TASKGROUP
:
16226 ctx
= maybe_lookup_ctx (stmt
);
16228 lower_omp_taskgroup (gsi_p
, ctx
);
16230 case GIMPLE_OMP_ORDERED
:
16231 ctx
= maybe_lookup_ctx (stmt
);
16233 lower_omp_ordered (gsi_p
, ctx
);
16235 case GIMPLE_OMP_CRITICAL
:
16236 ctx
= maybe_lookup_ctx (stmt
);
16238 lower_omp_critical (gsi_p
, ctx
);
16240 case GIMPLE_OMP_ATOMIC_LOAD
:
16241 if ((ctx
|| task_shared_vars
)
16242 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
16243 as_a
<gomp_atomic_load
*> (stmt
)),
16244 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
16245 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16247 case GIMPLE_OMP_TARGET
:
16248 ctx
= maybe_lookup_ctx (stmt
);
16250 lower_omp_target (gsi_p
, ctx
);
16252 case GIMPLE_OMP_TEAMS
:
16253 ctx
= maybe_lookup_ctx (stmt
);
16255 lower_omp_teams (gsi_p
, ctx
);
16259 call_stmt
= as_a
<gcall
*> (stmt
);
16260 fndecl
= gimple_call_fndecl (call_stmt
);
16262 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
16263 switch (DECL_FUNCTION_CODE (fndecl
))
16265 case BUILT_IN_GOMP_BARRIER
:
16269 case BUILT_IN_GOMP_CANCEL
:
16270 case BUILT_IN_GOMP_CANCELLATION_POINT
:
16273 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
16274 cctx
= cctx
->outer
;
16275 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
16276 if (!cctx
->cancellable
)
16278 if (DECL_FUNCTION_CODE (fndecl
)
16279 == BUILT_IN_GOMP_CANCELLATION_POINT
)
16281 stmt
= gimple_build_nop ();
16282 gsi_replace (gsi_p
, stmt
, false);
16286 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
16288 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
16289 gimple_call_set_fndecl (call_stmt
, fndecl
);
16290 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
16293 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
16294 gimple_call_set_lhs (call_stmt
, lhs
);
16295 tree fallthru_label
;
16296 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
16298 g
= gimple_build_label (fallthru_label
);
16299 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16300 g
= gimple_build_cond (NE_EXPR
, lhs
,
16301 fold_convert (TREE_TYPE (lhs
),
16302 boolean_false_node
),
16303 cctx
->cancel_label
, fallthru_label
);
16304 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16311 if ((ctx
|| task_shared_vars
)
16312 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
16315 /* Just remove clobbers, this should happen only if we have
16316 "privatized" local addressable variables in SIMD regions,
16317 the clobber isn't needed in that case and gimplifying address
16318 of the ARRAY_REF into a pointer and creating MEM_REF based
16319 clobber would create worse code than we get with the clobber
16321 if (gimple_clobber_p (stmt
))
16323 gsi_replace (gsi_p
, gimple_build_nop (), true);
16326 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16333 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
16335 location_t saved_location
= input_location
;
16336 gimple_stmt_iterator gsi
;
16337 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16338 lower_omp_1 (&gsi
, ctx
);
16339 /* During gimplification, we haven't folded statments inside offloading
16340 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
16341 if (target_nesting_level
|| taskreg_nesting_level
)
16342 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16344 input_location
= saved_location
;
16347 /* Main entry point. */
16349 static unsigned int
16350 execute_lower_omp (void)
16356 /* This pass always runs, to provide PROP_gimple_lomp.
16357 But often, there is nothing to do. */
16358 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
16359 && flag_openmp_simd
== 0)
16362 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
16363 delete_omp_context
);
16365 body
= gimple_body (current_function_decl
);
16366 scan_omp (&body
, NULL
);
16367 gcc_assert (taskreg_nesting_level
== 0);
16368 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
16369 finish_taskreg_scan (ctx
);
16370 taskreg_contexts
.release ();
16372 if (all_contexts
->root
)
16374 if (task_shared_vars
)
16375 push_gimplify_context ();
16376 lower_omp (&body
, NULL
);
16377 if (task_shared_vars
)
16378 pop_gimplify_context (NULL
);
16383 splay_tree_delete (all_contexts
);
16384 all_contexts
= NULL
;
16386 BITMAP_FREE (task_shared_vars
);
16392 const pass_data pass_data_lower_omp
=
16394 GIMPLE_PASS
, /* type */
16395 "omplower", /* name */
16396 OPTGROUP_NONE
, /* optinfo_flags */
16397 TV_NONE
, /* tv_id */
16398 PROP_gimple_any
, /* properties_required */
16399 PROP_gimple_lomp
, /* properties_provided */
16400 0, /* properties_destroyed */
16401 0, /* todo_flags_start */
16402 0, /* todo_flags_finish */
16405 class pass_lower_omp
: public gimple_opt_pass
16408 pass_lower_omp (gcc::context
*ctxt
)
16409 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
16412 /* opt_pass methods: */
16413 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
16415 }; // class pass_lower_omp
16417 } // anon namespace
16420 make_pass_lower_omp (gcc::context
*ctxt
)
16422 return new pass_lower_omp (ctxt
);
16425 /* The following is a utility to diagnose structured block violations.
16426 It is not part of the "omplower" pass, as that's invoked too late. It
16427 should be invoked by the respective front ends after gimplification. */
16429 static splay_tree all_labels
;
16431 /* Check for mismatched contexts and generate an error if needed. Return
16432 true if an error is detected. */
16435 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
16436 gimple
*branch_ctx
, gimple
*label_ctx
)
16438 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
16439 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
16441 if (label_ctx
== branch_ctx
)
16444 const char* kind
= NULL
;
16449 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
16450 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
16452 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
16453 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
16454 kind
= "Cilk Plus";
16458 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
16459 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
16461 gcc_checking_assert (kind
== NULL
);
16467 gcc_checking_assert (flag_openmp
);
16472 Previously we kept track of the label's entire context in diagnose_sb_[12]
16473 so we could traverse it and issue a correct "exit" or "enter" error
16474 message upon a structured block violation.
16476 We built the context by building a list with tree_cons'ing, but there is
16477 no easy counterpart in gimple tuples. It seems like far too much work
16478 for issuing exit/enter error messages. If someone really misses the
16479 distinct error message... patches welcome.
16483 /* Try to avoid confusing the user by producing and error message
16484 with correct "exit" or "enter" verbiage. We prefer "exit"
16485 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
16486 if (branch_ctx
== NULL
)
16492 if (TREE_VALUE (label_ctx
) == branch_ctx
)
16497 label_ctx
= TREE_CHAIN (label_ctx
);
16502 error ("invalid exit from %s structured block", kind
);
16504 error ("invalid entry to %s structured block", kind
);
16507 /* If it's obvious we have an invalid entry, be specific about the error. */
16508 if (branch_ctx
== NULL
)
16509 error ("invalid entry to %s structured block", kind
);
16512 /* Otherwise, be vague and lazy, but efficient. */
16513 error ("invalid branch to/from %s structured block", kind
);
16516 gsi_replace (gsi_p
, gimple_build_nop (), false);
16520 /* Pass 1: Create a minimal tree of structured blocks, and record
16521 where each label is found. */
16524 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16525 struct walk_stmt_info
*wi
)
16527 gimple
*context
= (gimple
*) wi
->info
;
16528 gimple
*inner_context
;
16529 gimple
*stmt
= gsi_stmt (*gsi_p
);
16531 *handled_ops_p
= true;
16533 switch (gimple_code (stmt
))
16537 case GIMPLE_OMP_PARALLEL
:
16538 case GIMPLE_OMP_TASK
:
16539 case GIMPLE_OMP_SECTIONS
:
16540 case GIMPLE_OMP_SINGLE
:
16541 case GIMPLE_OMP_SECTION
:
16542 case GIMPLE_OMP_MASTER
:
16543 case GIMPLE_OMP_ORDERED
:
16544 case GIMPLE_OMP_CRITICAL
:
16545 case GIMPLE_OMP_TARGET
:
16546 case GIMPLE_OMP_TEAMS
:
16547 case GIMPLE_OMP_TASKGROUP
:
16548 /* The minimal context here is just the current OMP construct. */
16549 inner_context
= stmt
;
16550 wi
->info
= inner_context
;
16551 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16552 wi
->info
= context
;
16555 case GIMPLE_OMP_FOR
:
16556 inner_context
= stmt
;
16557 wi
->info
= inner_context
;
16558 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16560 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
16561 diagnose_sb_1
, NULL
, wi
);
16562 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16563 wi
->info
= context
;
16567 splay_tree_insert (all_labels
,
16568 (splay_tree_key
) gimple_label_label (
16569 as_a
<glabel
*> (stmt
)),
16570 (splay_tree_value
) context
);
16580 /* Pass 2: Check each branch and see if its context differs from that of
16581 the destination label's context. */
16584 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16585 struct walk_stmt_info
*wi
)
16587 gimple
*context
= (gimple
*) wi
->info
;
16589 gimple
*stmt
= gsi_stmt (*gsi_p
);
16591 *handled_ops_p
= true;
16593 switch (gimple_code (stmt
))
16597 case GIMPLE_OMP_PARALLEL
:
16598 case GIMPLE_OMP_TASK
:
16599 case GIMPLE_OMP_SECTIONS
:
16600 case GIMPLE_OMP_SINGLE
:
16601 case GIMPLE_OMP_SECTION
:
16602 case GIMPLE_OMP_MASTER
:
16603 case GIMPLE_OMP_ORDERED
:
16604 case GIMPLE_OMP_CRITICAL
:
16605 case GIMPLE_OMP_TARGET
:
16606 case GIMPLE_OMP_TEAMS
:
16607 case GIMPLE_OMP_TASKGROUP
:
16609 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16610 wi
->info
= context
;
16613 case GIMPLE_OMP_FOR
:
16615 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16617 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
16618 diagnose_sb_2
, NULL
, wi
);
16619 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16620 wi
->info
= context
;
16625 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16626 tree lab
= gimple_cond_true_label (cond_stmt
);
16629 n
= splay_tree_lookup (all_labels
,
16630 (splay_tree_key
) lab
);
16631 diagnose_sb_0 (gsi_p
, context
,
16632 n
? (gimple
*) n
->value
: NULL
);
16634 lab
= gimple_cond_false_label (cond_stmt
);
16637 n
= splay_tree_lookup (all_labels
,
16638 (splay_tree_key
) lab
);
16639 diagnose_sb_0 (gsi_p
, context
,
16640 n
? (gimple
*) n
->value
: NULL
);
16647 tree lab
= gimple_goto_dest (stmt
);
16648 if (TREE_CODE (lab
) != LABEL_DECL
)
16651 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16652 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
16656 case GIMPLE_SWITCH
:
16658 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
16660 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
16662 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
16663 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16664 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
16670 case GIMPLE_RETURN
:
16671 diagnose_sb_0 (gsi_p
, context
, NULL
);
16681 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
16684 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
16687 gimple
*last
= last_stmt (bb
);
16688 enum gimple_code code
= gimple_code (last
);
16689 struct omp_region
*cur_region
= *region
;
16690 bool fallthru
= false;
16694 case GIMPLE_OMP_PARALLEL
:
16695 case GIMPLE_OMP_TASK
:
16696 case GIMPLE_OMP_FOR
:
16697 case GIMPLE_OMP_SINGLE
:
16698 case GIMPLE_OMP_TEAMS
:
16699 case GIMPLE_OMP_MASTER
:
16700 case GIMPLE_OMP_TASKGROUP
:
16701 case GIMPLE_OMP_CRITICAL
:
16702 case GIMPLE_OMP_SECTION
:
16703 cur_region
= new_omp_region (bb
, code
, cur_region
);
16707 case GIMPLE_OMP_ORDERED
:
16708 cur_region
= new_omp_region (bb
, code
, cur_region
);
16710 if (find_omp_clause (gimple_omp_ordered_clauses
16711 (as_a
<gomp_ordered
*> (last
)),
16712 OMP_CLAUSE_DEPEND
))
16713 cur_region
= cur_region
->outer
;
16716 case GIMPLE_OMP_TARGET
:
16717 cur_region
= new_omp_region (bb
, code
, cur_region
);
16719 switch (gimple_omp_target_kind (last
))
16721 case GF_OMP_TARGET_KIND_REGION
:
16722 case GF_OMP_TARGET_KIND_DATA
:
16723 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
16724 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
16725 case GF_OMP_TARGET_KIND_OACC_DATA
:
16727 case GF_OMP_TARGET_KIND_UPDATE
:
16728 case GF_OMP_TARGET_KIND_ENTER_DATA
:
16729 case GF_OMP_TARGET_KIND_EXIT_DATA
:
16730 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
16731 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
16732 cur_region
= cur_region
->outer
;
16735 gcc_unreachable ();
16739 case GIMPLE_OMP_SECTIONS
:
16740 cur_region
= new_omp_region (bb
, code
, cur_region
);
16744 case GIMPLE_OMP_SECTIONS_SWITCH
:
16748 case GIMPLE_OMP_ATOMIC_LOAD
:
16749 case GIMPLE_OMP_ATOMIC_STORE
:
16753 case GIMPLE_OMP_RETURN
:
16754 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
16755 somewhere other than the next block. This will be
16757 cur_region
->exit
= bb
;
16758 if (cur_region
->type
== GIMPLE_OMP_TASK
)
16759 /* Add an edge corresponding to not scheduling the task
16761 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
16762 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
16763 cur_region
= cur_region
->outer
;
16766 case GIMPLE_OMP_CONTINUE
:
16767 cur_region
->cont
= bb
;
16768 switch (cur_region
->type
)
16770 case GIMPLE_OMP_FOR
:
16771 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
16772 succs edges as abnormal to prevent splitting
16774 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
16775 /* Make the loopback edge. */
16776 make_edge (bb
, single_succ (cur_region
->entry
),
16779 /* Create an edge from GIMPLE_OMP_FOR to exit, which
16780 corresponds to the case that the body of the loop
16781 is not executed at all. */
16782 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
16783 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
16787 case GIMPLE_OMP_SECTIONS
:
16788 /* Wire up the edges into and out of the nested sections. */
16790 basic_block switch_bb
= single_succ (cur_region
->entry
);
16792 struct omp_region
*i
;
16793 for (i
= cur_region
->inner
; i
; i
= i
->next
)
16795 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
16796 make_edge (switch_bb
, i
->entry
, 0);
16797 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
16800 /* Make the loopback edge to the block with
16801 GIMPLE_OMP_SECTIONS_SWITCH. */
16802 make_edge (bb
, switch_bb
, 0);
16804 /* Make the edge from the switch to exit. */
16805 make_edge (switch_bb
, bb
->next_bb
, 0);
16810 case GIMPLE_OMP_TASK
:
16815 gcc_unreachable ();
16820 gcc_unreachable ();
16823 if (*region
!= cur_region
)
16825 *region
= cur_region
;
16827 *region_idx
= cur_region
->entry
->index
;
16835 static unsigned int
16836 diagnose_omp_structured_block_errors (void)
16838 struct walk_stmt_info wi
;
16839 gimple_seq body
= gimple_body (current_function_decl
);
16841 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
16843 memset (&wi
, 0, sizeof (wi
));
16844 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
16846 memset (&wi
, 0, sizeof (wi
));
16847 wi
.want_locations
= true;
16848 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
16850 gimple_set_body (current_function_decl
, body
);
16852 splay_tree_delete (all_labels
);
16860 const pass_data pass_data_diagnose_omp_blocks
=
16862 GIMPLE_PASS
, /* type */
16863 "*diagnose_omp_blocks", /* name */
16864 OPTGROUP_NONE
, /* optinfo_flags */
16865 TV_NONE
, /* tv_id */
16866 PROP_gimple_any
, /* properties_required */
16867 0, /* properties_provided */
16868 0, /* properties_destroyed */
16869 0, /* todo_flags_start */
16870 0, /* todo_flags_finish */
16873 class pass_diagnose_omp_blocks
: public gimple_opt_pass
16876 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16877 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
16880 /* opt_pass methods: */
16881 virtual bool gate (function
*)
16883 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
16885 virtual unsigned int execute (function
*)
16887 return diagnose_omp_structured_block_errors ();
16890 }; // class pass_diagnose_omp_blocks
16892 } // anon namespace
16895 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16897 return new pass_diagnose_omp_blocks (ctxt
);
16900 /* SIMD clone supporting code. */
16902 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
16903 of arguments to reserve space for. */
16905 static struct cgraph_simd_clone
*
16906 simd_clone_struct_alloc (int nargs
)
16908 struct cgraph_simd_clone
*clone_info
;
16909 size_t len
= (sizeof (struct cgraph_simd_clone
)
16910 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
16911 clone_info
= (struct cgraph_simd_clone
*)
16912 ggc_internal_cleared_alloc (len
);
16916 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
16919 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
16920 struct cgraph_simd_clone
*from
)
16922 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
16923 + ((from
->nargs
- from
->inbranch
)
16924 * sizeof (struct cgraph_simd_clone_arg
))));
16927 /* Return vector of parameter types of function FNDECL. This uses
16928 TYPE_ARG_TYPES if available, otherwise falls back to types of
16929 DECL_ARGUMENTS types. */
16932 simd_clone_vector_of_formal_parm_types (tree fndecl
)
16934 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
16935 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
16936 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
16939 FOR_EACH_VEC_ELT (args
, i
, arg
)
16940 args
[i
] = TREE_TYPE (args
[i
]);
16944 /* Given a simd function in NODE, extract the simd specific
16945 information from the OMP clauses passed in CLAUSES, and return
16946 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
16947 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
16948 otherwise set to FALSE. */
16950 static struct cgraph_simd_clone
*
16951 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
16952 bool *inbranch_specified
)
16954 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
16957 *inbranch_specified
= false;
16959 n
= args
.length ();
16960 if (n
> 0 && args
.last () == void_type_node
)
16963 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
16964 be cloned have a distinctive artificial label in addition to "omp
16968 && lookup_attribute ("cilk simd function",
16969 DECL_ATTRIBUTES (node
->decl
)));
16971 /* Allocate one more than needed just in case this is an in-branch
16972 clone which will require a mask argument. */
16973 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
16974 clone_info
->nargs
= n
;
16975 clone_info
->cilk_elemental
= cilk_clone
;
16982 clauses
= TREE_VALUE (clauses
);
16983 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
16986 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
16988 switch (OMP_CLAUSE_CODE (t
))
16990 case OMP_CLAUSE_INBRANCH
:
16991 clone_info
->inbranch
= 1;
16992 *inbranch_specified
= true;
16994 case OMP_CLAUSE_NOTINBRANCH
:
16995 clone_info
->inbranch
= 0;
16996 *inbranch_specified
= true;
16998 case OMP_CLAUSE_SIMDLEN
:
16999 clone_info
->simdlen
17000 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
17002 case OMP_CLAUSE_LINEAR
:
17004 tree decl
= OMP_CLAUSE_DECL (t
);
17005 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
17006 int argno
= TREE_INT_CST_LOW (decl
);
17007 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
17009 clone_info
->args
[argno
].arg_type
17010 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
17011 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
17012 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
17013 && clone_info
->args
[argno
].linear_step
< n
);
17017 if (POINTER_TYPE_P (args
[argno
]))
17018 step
= fold_convert (ssizetype
, step
);
17019 if (!tree_fits_shwi_p (step
))
17021 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
17022 "ignoring large linear step");
17026 else if (integer_zerop (step
))
17028 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
17029 "ignoring zero linear step");
17035 enum cgraph_simd_clone_arg_type arg_type
;
17036 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
17037 switch (OMP_CLAUSE_LINEAR_KIND (t
))
17039 case OMP_CLAUSE_LINEAR_REF
:
17041 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
17043 case OMP_CLAUSE_LINEAR_UVAL
:
17045 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
17047 case OMP_CLAUSE_LINEAR_VAL
:
17048 case OMP_CLAUSE_LINEAR_DEFAULT
:
17050 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
17053 gcc_unreachable ();
17056 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
17057 clone_info
->args
[argno
].arg_type
= arg_type
;
17058 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
17063 case OMP_CLAUSE_UNIFORM
:
17065 tree decl
= OMP_CLAUSE_DECL (t
);
17066 int argno
= tree_to_uhwi (decl
);
17067 clone_info
->args
[argno
].arg_type
17068 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
17071 case OMP_CLAUSE_ALIGNED
:
17073 tree decl
= OMP_CLAUSE_DECL (t
);
17074 int argno
= tree_to_uhwi (decl
);
17075 clone_info
->args
[argno
].alignment
17076 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
17087 /* Given a SIMD clone in NODE, calculate the characteristic data
17088 type and return the coresponding type. The characteristic data
17089 type is computed as described in the Intel Vector ABI. */
17092 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
17093 struct cgraph_simd_clone
*clone_info
)
17095 tree type
= integer_type_node
;
17096 tree fndecl
= node
->decl
;
17098 /* a) For non-void function, the characteristic data type is the
17100 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
17101 type
= TREE_TYPE (TREE_TYPE (fndecl
));
17103 /* b) If the function has any non-uniform, non-linear parameters,
17104 then the characteristic data type is the type of the first
17108 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
17109 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
17110 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
17118 /* c) If the characteristic data type determined by a) or b) above
17119 is struct, union, or class type which is pass-by-value (except
17120 for the type that maps to the built-in complex data type), the
17121 characteristic data type is int. */
17122 if (RECORD_OR_UNION_TYPE_P (type
)
17123 && !aggregate_value_p (type
, NULL
)
17124 && TREE_CODE (type
) != COMPLEX_TYPE
)
17125 return integer_type_node
;
17127 /* d) If none of the above three classes is applicable, the
17128 characteristic data type is int. */
17132 /* e) For Intel Xeon Phi native and offload compilation, if the
17133 resulting characteristic data type is 8-bit or 16-bit integer
17134 data type, the characteristic data type is int. */
17135 /* Well, we don't handle Xeon Phi yet. */
17139 simd_clone_mangle (struct cgraph_node
*node
,
17140 struct cgraph_simd_clone
*clone_info
)
17142 char vecsize_mangle
= clone_info
->vecsize_mangle
;
17143 char mask
= clone_info
->inbranch
? 'M' : 'N';
17144 unsigned int simdlen
= clone_info
->simdlen
;
17148 gcc_assert (vecsize_mangle
&& simdlen
);
17150 pp_string (&pp
, "_ZGV");
17151 pp_character (&pp
, vecsize_mangle
);
17152 pp_character (&pp
, mask
);
17153 pp_decimal_int (&pp
, simdlen
);
17155 for (n
= 0; n
< clone_info
->nargs
; ++n
)
17157 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
17159 switch (arg
.arg_type
)
17161 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
17162 pp_character (&pp
, 'u');
17164 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
17165 pp_character (&pp
, 'l');
17166 goto mangle_linear
;
17167 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17168 pp_character (&pp
, 'R');
17169 goto mangle_linear
;
17170 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17171 pp_character (&pp
, 'L');
17172 goto mangle_linear
;
17173 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17174 pp_character (&pp
, 'U');
17175 goto mangle_linear
;
17177 gcc_assert (arg
.linear_step
!= 0);
17178 if (arg
.linear_step
> 1)
17179 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17180 else if (arg
.linear_step
< 0)
17182 pp_character (&pp
, 'n');
17183 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
17187 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17188 pp_character (&pp
, 's');
17189 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17192 pp_character (&pp
, 'v');
17196 pp_character (&pp
, 'a');
17197 pp_decimal_int (&pp
, arg
.alignment
);
17201 pp_underscore (&pp
);
17202 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
17205 pp_string (&pp
, str
);
17206 str
= pp_formatted_text (&pp
);
17208 /* If there already is a SIMD clone with the same mangled name, don't
17209 add another one. This can happen e.g. for
17210 #pragma omp declare simd
17211 #pragma omp declare simd simdlen(8)
17212 int foo (int, int);
17213 if the simdlen is assumed to be 8 for the first one, etc. */
17214 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
17215 clone
= clone
->simdclone
->next_clone
)
17216 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
17220 return get_identifier (str
);
17223 /* Create a simd clone of OLD_NODE and return it. */
17225 static struct cgraph_node
*
17226 simd_clone_create (struct cgraph_node
*old_node
)
17228 struct cgraph_node
*new_node
;
17229 if (old_node
->definition
)
17231 if (!old_node
->has_gimple_body_p ())
17233 old_node
->get_body ();
17234 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
17240 tree old_decl
= old_node
->decl
;
17241 tree new_decl
= copy_node (old_node
->decl
);
17242 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
17243 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
17244 SET_DECL_RTL (new_decl
, NULL
);
17245 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
17246 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
17247 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
17248 if (old_node
->in_other_partition
)
17249 new_node
->in_other_partition
= 1;
17250 symtab
->call_cgraph_insertion_hooks (new_node
);
17252 if (new_node
== NULL
)
17255 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
17257 /* The function cgraph_function_versioning () will force the new
17258 symbol local. Undo this, and inherit external visability from
17260 new_node
->local
.local
= old_node
->local
.local
;
17261 new_node
->externally_visible
= old_node
->externally_visible
;
17266 /* Adjust the return type of the given function to its appropriate
17267 vector counterpart. Returns a simd array to be used throughout the
17268 function as a return value. */
17271 simd_clone_adjust_return_type (struct cgraph_node
*node
)
17273 tree fndecl
= node
->decl
;
17274 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
17275 unsigned int veclen
;
17278 /* Adjust the function return type. */
17279 if (orig_rettype
== void_type_node
)
17281 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
17282 t
= TREE_TYPE (TREE_TYPE (fndecl
));
17283 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
17284 veclen
= node
->simdclone
->vecsize_int
;
17286 veclen
= node
->simdclone
->vecsize_float
;
17287 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
17288 if (veclen
> node
->simdclone
->simdlen
)
17289 veclen
= node
->simdclone
->simdlen
;
17290 if (POINTER_TYPE_P (t
))
17291 t
= pointer_sized_int_node
;
17292 if (veclen
== node
->simdclone
->simdlen
)
17293 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
17296 t
= build_vector_type (t
, veclen
);
17297 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
17299 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
17300 if (!node
->definition
)
17303 t
= DECL_RESULT (fndecl
);
17304 /* Adjust the DECL_RESULT. */
17305 gcc_assert (TREE_TYPE (t
) != void_type_node
);
17306 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
17309 tree atype
= build_array_type_nelts (orig_rettype
,
17310 node
->simdclone
->simdlen
);
17311 if (veclen
!= node
->simdclone
->simdlen
)
17312 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
17314 /* Set up a SIMD array to use as the return value. */
17315 tree retval
= create_tmp_var_raw (atype
, "retval");
17316 gimple_add_tmp_var (retval
);
17320 /* Each vector argument has a corresponding array to be used locally
17321 as part of the eventual loop. Create such temporary array and
17324 PREFIX is the prefix to be used for the temporary.
17326 TYPE is the inner element type.
17328 SIMDLEN is the number of elements. */
17331 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
17333 tree atype
= build_array_type_nelts (type
, simdlen
);
17334 tree avar
= create_tmp_var_raw (atype
, prefix
);
17335 gimple_add_tmp_var (avar
);
17339 /* Modify the function argument types to their corresponding vector
17340 counterparts if appropriate. Also, create one array for each simd
17341 argument to be used locally when using the function arguments as
17344 NODE is the function whose arguments are to be adjusted.
17346 Returns an adjustment vector that will be filled describing how the
17347 argument types will be adjusted. */
17349 static ipa_parm_adjustment_vec
17350 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
17353 ipa_parm_adjustment_vec adjustments
;
17355 if (node
->definition
)
17356 args
= ipa_get_vector_of_formal_parms (node
->decl
);
17358 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
17359 adjustments
.create (args
.length ());
17360 unsigned i
, j
, veclen
;
17361 struct ipa_parm_adjustment adj
;
17362 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
17364 memset (&adj
, 0, sizeof (adj
));
17365 tree parm
= args
[i
];
17366 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
17367 adj
.base_index
= i
;
17370 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
17371 node
->simdclone
->args
[i
].orig_type
= parm_type
;
17373 switch (node
->simdclone
->args
[i
].arg_type
)
17376 /* No adjustment necessary for scalar arguments. */
17377 adj
.op
= IPA_PARM_OP_COPY
;
17379 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17380 if (node
->definition
)
17381 node
->simdclone
->args
[i
].simd_array
17382 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17383 TREE_TYPE (parm_type
),
17384 node
->simdclone
->simdlen
);
17385 adj
.op
= IPA_PARM_OP_COPY
;
17387 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17388 case SIMD_CLONE_ARG_TYPE_VECTOR
:
17389 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
17390 veclen
= node
->simdclone
->vecsize_int
;
17392 veclen
= node
->simdclone
->vecsize_float
;
17393 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
17394 if (veclen
> node
->simdclone
->simdlen
)
17395 veclen
= node
->simdclone
->simdlen
;
17396 adj
.arg_prefix
= "simd";
17397 if (POINTER_TYPE_P (parm_type
))
17398 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17400 adj
.type
= build_vector_type (parm_type
, veclen
);
17401 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
17402 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17404 adjustments
.safe_push (adj
);
17407 memset (&adj
, 0, sizeof (adj
));
17408 adj
.op
= IPA_PARM_OP_NEW
;
17409 adj
.arg_prefix
= "simd";
17410 adj
.base_index
= i
;
17411 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
17415 if (node
->definition
)
17416 node
->simdclone
->args
[i
].simd_array
17417 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17418 parm_type
, node
->simdclone
->simdlen
);
17420 adjustments
.safe_push (adj
);
17423 if (node
->simdclone
->inbranch
)
17426 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
17429 memset (&adj
, 0, sizeof (adj
));
17430 adj
.op
= IPA_PARM_OP_NEW
;
17431 adj
.arg_prefix
= "mask";
17433 adj
.base_index
= i
;
17434 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
17435 veclen
= node
->simdclone
->vecsize_int
;
17437 veclen
= node
->simdclone
->vecsize_float
;
17438 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
17439 if (veclen
> node
->simdclone
->simdlen
)
17440 veclen
= node
->simdclone
->simdlen
;
17441 if (POINTER_TYPE_P (base_type
))
17442 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17444 adj
.type
= build_vector_type (base_type
, veclen
);
17445 adjustments
.safe_push (adj
);
17447 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17448 adjustments
.safe_push (adj
);
17450 /* We have previously allocated one extra entry for the mask. Use
17452 struct cgraph_simd_clone
*sc
= node
->simdclone
;
17454 if (node
->definition
)
17456 sc
->args
[i
].orig_arg
17457 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
17458 sc
->args
[i
].simd_array
17459 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
17461 sc
->args
[i
].orig_type
= base_type
;
17462 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
17465 if (node
->definition
)
17466 ipa_modify_formal_parameters (node
->decl
, adjustments
);
17469 tree new_arg_types
= NULL_TREE
, new_reversed
;
17470 bool last_parm_void
= false;
17471 if (args
.length () > 0 && args
.last () == void_type_node
)
17472 last_parm_void
= true;
17474 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
17475 j
= adjustments
.length ();
17476 for (i
= 0; i
< j
; i
++)
17478 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
17480 if (adj
->op
== IPA_PARM_OP_COPY
)
17481 ptype
= args
[adj
->base_index
];
17484 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
17486 new_reversed
= nreverse (new_arg_types
);
17487 if (last_parm_void
)
17490 TREE_CHAIN (new_arg_types
) = void_list_node
;
17492 new_reversed
= void_list_node
;
17495 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
17496 TYPE_ARG_TYPES (new_type
) = new_reversed
;
17497 TREE_TYPE (node
->decl
) = new_type
;
17499 adjustments
.release ();
17502 return adjustments
;
17505 /* Initialize and copy the function arguments in NODE to their
17506 corresponding local simd arrays. Returns a fresh gimple_seq with
17507 the instruction sequence generated. */
17510 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
17511 ipa_parm_adjustment_vec adjustments
)
17513 gimple_seq seq
= NULL
;
17514 unsigned i
= 0, j
= 0, k
;
17516 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
17518 arg
= DECL_CHAIN (arg
), i
++, j
++)
17520 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
17521 || POINTER_TYPE_P (TREE_TYPE (arg
)))
17524 node
->simdclone
->args
[i
].vector_arg
= arg
;
17526 tree array
= node
->simdclone
->args
[i
].simd_array
;
17527 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
17529 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17530 tree ptr
= build_fold_addr_expr (array
);
17531 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17532 build_int_cst (ptype
, 0));
17533 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17534 gimplify_and_add (t
, &seq
);
17538 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
17539 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17540 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
17542 tree ptr
= build_fold_addr_expr (array
);
17546 arg
= DECL_CHAIN (arg
);
17550 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
17551 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17552 build_int_cst (ptype
, k
* elemsize
));
17553 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17554 gimplify_and_add (t
, &seq
);
17561 /* Callback info for ipa_simd_modify_stmt_ops below. */
17563 struct modify_stmt_info
{
17564 ipa_parm_adjustment_vec adjustments
;
17566 /* True if the parent statement was modified by
17567 ipa_simd_modify_stmt_ops. */
17571 /* Callback for walk_gimple_op.
17573 Adjust operands from a given statement as specified in the
17574 adjustments vector in the callback data. */
17577 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
17579 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17580 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
17581 tree
*orig_tp
= tp
;
17582 if (TREE_CODE (*tp
) == ADDR_EXPR
)
17583 tp
= &TREE_OPERAND (*tp
, 0);
17584 struct ipa_parm_adjustment
*cand
= NULL
;
17585 if (TREE_CODE (*tp
) == PARM_DECL
)
17586 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
17590 *walk_subtrees
= 0;
17593 tree repl
= NULL_TREE
;
17595 repl
= unshare_expr (cand
->new_decl
);
17600 *walk_subtrees
= 0;
17601 bool modified
= info
->modified
;
17602 info
->modified
= false;
17603 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
17604 if (!info
->modified
)
17606 info
->modified
= modified
;
17609 info
->modified
= modified
;
17618 repl
= build_fold_addr_expr (repl
);
17620 if (is_gimple_debug (info
->stmt
))
17622 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
17623 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
17624 DECL_ARTIFICIAL (vexpr
) = 1;
17625 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
17626 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
17631 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
17632 repl
= gimple_assign_lhs (stmt
);
17634 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
17635 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17638 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
17640 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
17646 info
->modified
= true;
17650 /* Traverse the function body and perform all modifications as
17651 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
17652 modified such that the replacement/reduction value will now be an
17653 offset into the corresponding simd_array.
17655 This function will replace all function argument uses with their
17656 corresponding simd array elements, and ajust the return values
17660 ipa_simd_modify_function_body (struct cgraph_node
*node
,
17661 ipa_parm_adjustment_vec adjustments
,
17662 tree retval_array
, tree iter
)
17665 unsigned int i
, j
, l
;
17667 /* Re-use the adjustments array, but this time use it to replace
17668 every function argument use to an offset into the corresponding
17670 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
17672 if (!node
->simdclone
->args
[i
].vector_arg
)
17675 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17676 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
17677 adjustments
[j
].new_decl
17678 = build4 (ARRAY_REF
,
17680 node
->simdclone
->args
[i
].simd_array
,
17682 NULL_TREE
, NULL_TREE
);
17683 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
17684 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
17685 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
17688 l
= adjustments
.length ();
17689 for (i
= 1; i
< num_ssa_names
; i
++)
17691 tree name
= ssa_name (i
);
17693 && SSA_NAME_VAR (name
)
17694 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
17696 for (j
= 0; j
< l
; j
++)
17697 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
17698 && adjustments
[j
].new_decl
)
17701 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
17704 = copy_var_decl (adjustments
[j
].base
,
17705 DECL_NAME (adjustments
[j
].base
),
17706 TREE_TYPE (adjustments
[j
].base
));
17707 adjustments
[j
].new_ssa_base
= base_var
;
17710 base_var
= adjustments
[j
].new_ssa_base
;
17711 if (SSA_NAME_IS_DEFAULT_DEF (name
))
17713 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17714 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
17715 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
17716 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
17717 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17718 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
17719 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
17720 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17723 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17728 struct modify_stmt_info info
;
17729 info
.adjustments
= adjustments
;
17731 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
17733 gimple_stmt_iterator gsi
;
17735 gsi
= gsi_start_bb (bb
);
17736 while (!gsi_end_p (gsi
))
17738 gimple
*stmt
= gsi_stmt (gsi
);
17740 struct walk_stmt_info wi
;
17742 memset (&wi
, 0, sizeof (wi
));
17743 info
.modified
= false;
17745 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
17747 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
17749 tree retval
= gimple_return_retval (return_stmt
);
17752 gsi_remove (&gsi
, true);
17756 /* Replace `return foo' with `retval_array[iter] = foo'. */
17757 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
17758 retval_array
, iter
, NULL
, NULL
);
17759 stmt
= gimple_build_assign (ref
, retval
);
17760 gsi_replace (&gsi
, stmt
, true);
17761 info
.modified
= true;
17766 update_stmt (stmt
);
17767 if (maybe_clean_eh_stmt (stmt
))
17768 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
17775 /* Adjust the argument types in NODE to their appropriate vector
17779 simd_clone_adjust (struct cgraph_node
*node
)
17781 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
17783 targetm
.simd_clone
.adjust (node
);
17785 tree retval
= simd_clone_adjust_return_type (node
);
17786 ipa_parm_adjustment_vec adjustments
17787 = simd_clone_adjust_argument_types (node
);
17789 push_gimplify_context ();
17791 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
17793 /* Adjust all uses of vector arguments accordingly. Adjust all
17794 return values accordingly. */
17795 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
17796 tree iter1
= make_ssa_name (iter
);
17797 tree iter2
= make_ssa_name (iter
);
17798 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
17800 /* Initialize the iteration variable. */
17801 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17802 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
17803 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17804 /* Insert the SIMD array and iv initialization at function
17806 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
17808 pop_gimplify_context (NULL
);
17810 /* Create a new BB right before the original exit BB, to hold the
17811 iteration increment and the condition/branch. */
17812 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
17813 basic_block incr_bb
= create_empty_bb (orig_exit
);
17814 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
17815 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
17816 flag. Set it now to be a FALLTHRU_EDGE. */
17817 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
17818 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
17819 for (unsigned i
= 0;
17820 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
17822 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
17823 redirect_edge_succ (e
, incr_bb
);
17825 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
17826 e
->probability
= REG_BR_PROB_BASE
;
17827 gsi
= gsi_last_bb (incr_bb
);
17828 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
17829 build_int_cst (unsigned_type_node
, 1));
17830 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17832 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
17833 struct loop
*loop
= alloc_loop ();
17834 cfun
->has_force_vectorize_loops
= true;
17835 loop
->safelen
= node
->simdclone
->simdlen
;
17836 loop
->force_vectorize
= true;
17837 loop
->header
= body_bb
;
17839 /* Branch around the body if the mask applies. */
17840 if (node
->simdclone
->inbranch
)
17842 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
17844 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
17845 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
17846 tree aref
= build4 (ARRAY_REF
,
17847 TREE_TYPE (TREE_TYPE (mask_array
)),
17850 g
= gimple_build_assign (mask
, aref
);
17851 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17852 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
17853 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
17855 aref
= build1 (VIEW_CONVERT_EXPR
,
17856 build_nonstandard_integer_type (bitsize
, 0), mask
);
17857 mask
= make_ssa_name (TREE_TYPE (aref
));
17858 g
= gimple_build_assign (mask
, aref
);
17859 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17862 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
17864 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17865 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
17866 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
17869 /* Generate the condition. */
17870 g
= gimple_build_cond (LT_EXPR
,
17872 build_int_cst (unsigned_type_node
,
17873 node
->simdclone
->simdlen
),
17875 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17876 e
= split_block (incr_bb
, gsi_stmt (gsi
));
17877 basic_block latch_bb
= e
->dest
;
17878 basic_block new_exit_bb
;
17879 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
17880 loop
->latch
= latch_bb
;
17882 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
17884 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
17885 /* The successor of incr_bb is already pointing to latch_bb; just
17887 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
17888 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
17890 gphi
*phi
= create_phi_node (iter1
, body_bb
);
17891 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
17892 edge latch_edge
= single_succ_edge (latch_bb
);
17893 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
17895 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17897 /* Generate the new return. */
17898 gsi
= gsi_last_bb (new_exit_bb
);
17900 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
17901 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
17902 retval
= TREE_OPERAND (retval
, 0);
17905 retval
= build1 (VIEW_CONVERT_EXPR
,
17906 TREE_TYPE (TREE_TYPE (node
->decl
)),
17908 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
17909 false, GSI_CONTINUE_LINKING
);
17911 g
= gimple_build_return (retval
);
17912 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17914 /* Handle aligned clauses by replacing default defs of the aligned
17915 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
17916 lhs. Handle linear by adding PHIs. */
17917 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
17918 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
17919 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
17920 || !is_gimple_reg_type
17921 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
17923 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17924 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
17925 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
17928 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
17929 gimple_add_tmp_var (iter1
);
17931 gsi
= gsi_after_labels (entry_bb
);
17932 g
= gimple_build_assign (iter1
, orig_arg
);
17933 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17934 gsi
= gsi_after_labels (body_bb
);
17935 g
= gimple_build_assign (orig_arg
, iter1
);
17936 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17938 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
17939 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
17940 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
17942 && TREE_ADDRESSABLE
17943 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
17945 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17946 tree def
= ssa_default_def (cfun
, orig_arg
);
17947 if (def
&& !has_zero_uses (def
))
17949 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
17950 gimple_add_tmp_var (iter1
);
17951 gsi
= gsi_after_labels (entry_bb
);
17952 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
17953 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17954 gsi
= gsi_after_labels (body_bb
);
17955 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
17956 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17959 else if (node
->simdclone
->args
[i
].alignment
17960 && node
->simdclone
->args
[i
].arg_type
17961 == SIMD_CLONE_ARG_TYPE_UNIFORM
17962 && (node
->simdclone
->args
[i
].alignment
17963 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
17964 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
17967 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
17968 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17969 tree def
= ssa_default_def (cfun
, orig_arg
);
17970 if (def
&& !has_zero_uses (def
))
17972 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
17973 gimple_seq seq
= NULL
;
17974 bool need_cvt
= false;
17976 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
17978 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
17981 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
17982 gimple_call_set_lhs (g
, t
);
17983 gimple_seq_add_stmt_without_update (&seq
, g
);
17986 t
= make_ssa_name (orig_arg
);
17987 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
17988 gimple_seq_add_stmt_without_update (&seq
, g
);
17990 gsi_insert_seq_on_edge_immediate
17991 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
17993 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17994 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
17996 node
->create_edge (cgraph_node::get_create (fn
),
17997 call
, entry_bb
->count
, freq
);
17999 imm_use_iterator iter
;
18000 use_operand_p use_p
;
18002 tree repl
= gimple_get_lhs (g
);
18003 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18004 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
18007 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18008 SET_USE (use_p
, repl
);
18011 else if ((node
->simdclone
->args
[i
].arg_type
18012 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
18013 || (node
->simdclone
->args
[i
].arg_type
18014 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
))
18016 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18017 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18018 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
18019 tree def
= NULL_TREE
;
18020 if (TREE_ADDRESSABLE (orig_arg
))
18022 def
= make_ssa_name (TREE_TYPE (orig_arg
));
18023 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
18024 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
18025 gsi
= gsi_after_labels (entry_bb
);
18026 g
= gimple_build_assign (def
, orig_arg
);
18027 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18031 def
= ssa_default_def (cfun
, orig_arg
);
18032 if (!def
|| has_zero_uses (def
))
18036 iter1
= make_ssa_name (orig_arg
);
18037 iter2
= make_ssa_name (orig_arg
);
18042 phi
= create_phi_node (iter1
, body_bb
);
18043 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
18044 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18045 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18046 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18047 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18048 ? TREE_TYPE (orig_arg
) : sizetype
;
18050 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
18051 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
18052 gsi
= gsi_last_bb (incr_bb
);
18053 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18055 imm_use_iterator iter
;
18056 use_operand_p use_p
;
18058 if (TREE_ADDRESSABLE (orig_arg
))
18060 gsi
= gsi_after_labels (body_bb
);
18061 g
= gimple_build_assign (orig_arg
, iter1
);
18062 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18065 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18066 if (use_stmt
== phi
)
18069 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18070 SET_USE (use_p
, iter1
);
18073 else if (node
->simdclone
->args
[i
].arg_type
18074 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
)
18076 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18077 tree def
= ssa_default_def (cfun
, orig_arg
);
18078 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
18079 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
18080 if (def
&& !has_zero_uses (def
))
18082 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
18083 iter1
= make_ssa_name (orig_arg
);
18084 iter2
= make_ssa_name (orig_arg
);
18085 tree iter3
= make_ssa_name (rtype
);
18086 tree iter4
= make_ssa_name (rtype
);
18087 tree iter5
= make_ssa_name (rtype
);
18088 gsi
= gsi_after_labels (entry_bb
);
18090 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
18091 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
18093 tree array
= node
->simdclone
->args
[i
].simd_array
;
18094 TREE_ADDRESSABLE (array
) = 1;
18095 tree ptr
= build_fold_addr_expr (array
);
18096 phi
= create_phi_node (iter1
, body_bb
);
18097 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
18098 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18099 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
18100 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
18101 gsi
= gsi_last_bb (incr_bb
);
18102 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18104 phi
= create_phi_node (iter4
, body_bb
);
18105 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
18106 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
18107 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18108 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18109 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18110 ? TREE_TYPE (iter3
) : sizetype
;
18112 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
18113 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
18114 gsi
= gsi_last_bb (incr_bb
);
18115 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18117 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
18118 gsi
= gsi_after_labels (body_bb
);
18119 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18121 imm_use_iterator iter
;
18122 use_operand_p use_p
;
18124 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18125 if (use_stmt
== load
)
18128 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18129 SET_USE (use_p
, iter1
);
18131 if (!TYPE_READONLY (rtype
))
18133 tree v
= make_ssa_name (rtype
);
18134 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
18135 size_zero_node
, NULL_TREE
,
18137 gsi
= gsi_after_labels (new_exit_bb
);
18138 g
= gimple_build_assign (v
, aref
);
18139 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18140 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
18141 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18146 calculate_dominance_info (CDI_DOMINATORS
);
18147 add_loop (loop
, loop
->header
->loop_father
);
18148 update_ssa (TODO_update_ssa
);
18153 /* If the function in NODE is tagged as an elemental SIMD function,
18154 create the appropriate SIMD clones. */
18157 expand_simd_clones (struct cgraph_node
*node
)
18159 tree attr
= lookup_attribute ("omp declare simd",
18160 DECL_ATTRIBUTES (node
->decl
));
18161 if (attr
== NULL_TREE
18162 || node
->global
.inlined_to
18163 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
18167 #pragma omp declare simd
18169 in C, there we don't know the argument types at all. */
18170 if (!node
->definition
18171 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
18176 /* Start with parsing the "omp declare simd" attribute(s). */
18177 bool inbranch_clause_specified
;
18178 struct cgraph_simd_clone
*clone_info
18179 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
18180 &inbranch_clause_specified
);
18181 if (clone_info
== NULL
)
18184 int orig_simdlen
= clone_info
->simdlen
;
18185 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
18186 /* The target can return 0 (no simd clones should be created),
18187 1 (just one ISA of simd clones should be created) or higher
18188 count of ISA variants. In that case, clone_info is initialized
18189 for the first ISA variant. */
18191 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
18196 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
18197 also create one inbranch and one !inbranch clone of it. */
18198 for (int i
= 0; i
< count
* 2; i
++)
18200 struct cgraph_simd_clone
*clone
= clone_info
;
18201 if (inbranch_clause_specified
&& (i
& 1) != 0)
18206 clone
= simd_clone_struct_alloc (clone_info
->nargs
18208 simd_clone_struct_copy (clone
, clone_info
);
18209 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
18210 and simd_clone_adjust_argument_types did to the first
18212 clone
->nargs
-= clone_info
->inbranch
;
18213 clone
->simdlen
= orig_simdlen
;
18214 /* And call the target hook again to get the right ISA. */
18215 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
18219 clone
->inbranch
= 1;
18222 /* simd_clone_mangle might fail if such a clone has been created
18224 tree id
= simd_clone_mangle (node
, clone
);
18225 if (id
== NULL_TREE
)
18228 /* Only when we are sure we want to create the clone actually
18229 clone the function (or definitions) or create another
18230 extern FUNCTION_DECL (for prototypes without definitions). */
18231 struct cgraph_node
*n
= simd_clone_create (node
);
18235 n
->simdclone
= clone
;
18236 clone
->origin
= node
;
18237 clone
->next_clone
= NULL
;
18238 if (node
->simd_clones
== NULL
)
18240 clone
->prev_clone
= n
;
18241 node
->simd_clones
= n
;
18245 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
18246 clone
->prev_clone
->simdclone
->next_clone
= n
;
18247 node
->simd_clones
->simdclone
->prev_clone
= n
;
18249 symtab
->change_decl_assembler_name (n
->decl
, id
);
18250 /* And finally adjust the return type, parameters and for
18251 definitions also function body. */
18252 if (node
->definition
)
18253 simd_clone_adjust (n
);
18256 simd_clone_adjust_return_type (n
);
18257 simd_clone_adjust_argument_types (n
);
18261 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
18264 /* Entry point for IPA simd clone creation pass. */
18266 static unsigned int
18267 ipa_omp_simd_clone (void)
18269 struct cgraph_node
*node
;
18270 FOR_EACH_FUNCTION (node
)
18271 expand_simd_clones (node
);
18277 const pass_data pass_data_omp_simd_clone
=
18279 SIMPLE_IPA_PASS
, /* type */
18280 "simdclone", /* name */
18281 OPTGROUP_NONE
, /* optinfo_flags */
18282 TV_NONE
, /* tv_id */
18283 ( PROP_ssa
| PROP_cfg
), /* properties_required */
18284 0, /* properties_provided */
18285 0, /* properties_destroyed */
18286 0, /* todo_flags_start */
18287 0, /* todo_flags_finish */
18290 class pass_omp_simd_clone
: public simple_ipa_opt_pass
18293 pass_omp_simd_clone(gcc::context
*ctxt
)
18294 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
18297 /* opt_pass methods: */
18298 virtual bool gate (function
*);
18299 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
18303 pass_omp_simd_clone::gate (function
*)
18305 return ((flag_openmp
|| flag_openmp_simd
18307 || (in_lto_p
&& !flag_wpa
))
18308 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
18311 } // anon namespace
18313 simple_ipa_opt_pass
*
18314 make_pass_omp_simd_clone (gcc::context
*ctxt
)
18316 return new pass_omp_simd_clone (ctxt
);
18319 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18320 adds their addresses and sizes to constructor-vector V_CTOR. */
18322 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18323 vec
<constructor_elt
, va_gc
> *v_ctor
)
18325 unsigned len
= vec_safe_length (v_decls
);
18326 for (unsigned i
= 0; i
< len
; i
++)
18328 tree it
= (*v_decls
)[i
];
18329 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
18331 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
18333 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
18334 fold_convert (const_ptr_type_node
,
18335 DECL_SIZE_UNIT (it
)));
18339 /* Create new symbols containing (address, size) pairs for global variables,
18340 marked with "omp declare target" attribute, as well as addresses for the
18341 functions, which are outlined offloading regions. */
18343 omp_finish_file (void)
18345 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18346 unsigned num_vars
= vec_safe_length (offload_vars
);
18348 if (num_funcs
== 0 && num_vars
== 0)
18351 if (targetm_common
.have_named_sections
)
18353 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18354 vec_alloc (v_f
, num_funcs
);
18355 vec_alloc (v_v
, num_vars
* 2);
18357 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18358 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18360 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18362 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18364 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18365 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18366 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18367 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18368 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18369 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18370 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18371 get_identifier (".offload_func_table"),
18373 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18374 get_identifier (".offload_var_table"),
18376 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18377 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18378 otherwise a joint table in a binary will contain padding between
18379 tables from multiple object files. */
18380 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18381 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
18382 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
18383 DECL_INITIAL (funcs_decl
) = ctor_f
;
18384 DECL_INITIAL (vars_decl
) = ctor_v
;
18385 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18386 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18388 varpool_node::finalize_decl (vars_decl
);
18389 varpool_node::finalize_decl (funcs_decl
);
18393 for (unsigned i
= 0; i
< num_funcs
; i
++)
18395 tree it
= (*offload_funcs
)[i
];
18396 targetm
.record_offload_symbol (it
);
18398 for (unsigned i
= 0; i
< num_vars
; i
++)
18400 tree it
= (*offload_vars
)[i
];
18401 targetm
.record_offload_symbol (it
);
18406 /* Find the number of threads (POS = false), or thread number (POS =
18407 true) for an OpenACC region partitioned as MASK. Setup code
18408 required for the calculation is added to SEQ. */
18411 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18413 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18416 /* Start at gang level, and examine relevant dimension indices. */
18417 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18418 if (GOMP_DIM_MASK (ix
) & mask
)
18420 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18424 /* We had an outer index, so scale that by the size of
18426 tree n
= create_tmp_var (integer_type_node
);
18428 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18430 gimple_call_set_lhs (call
, n
);
18431 gimple_seq_add_stmt (seq
, call
);
18432 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18436 /* Determine index in this dimension. */
18437 tree id
= create_tmp_var (integer_type_node
);
18438 gimple
*call
= gimple_build_call_internal
18439 (IFN_GOACC_DIM_POS
, 1, arg
);
18441 gimple_call_set_lhs (call
, id
);
18442 gimple_seq_add_stmt (seq
, call
);
18444 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18450 if (res
== NULL_TREE
)
18451 res
= integer_zero_node
;
18456 /* Transform IFN_GOACC_LOOP calls to actual code. See
18457 expand_oacc_for for where these are generated. At the vector
18458 level, we stride loops, such that each member of a warp will
18459 operate on adjacent iterations. At the worker and gang level,
18460 each gang/warp executes a set of contiguous iterations. Chunking
18461 can override this such that each iteration engine executes a
18462 contiguous chunk, and then moves on to stride to the next chunk. */
18465 oacc_xform_loop (gcall
*call
)
18467 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18468 enum ifn_goacc_loop_kind code
18469 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18470 tree dir
= gimple_call_arg (call
, 1);
18471 tree range
= gimple_call_arg (call
, 2);
18472 tree step
= gimple_call_arg (call
, 3);
18473 tree chunk_size
= NULL_TREE
;
18474 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18475 tree lhs
= gimple_call_lhs (call
);
18476 tree type
= TREE_TYPE (lhs
);
18477 tree diff_type
= TREE_TYPE (range
);
18478 tree r
= NULL_TREE
;
18479 gimple_seq seq
= NULL
;
18480 bool chunking
= false, striding
= true;
18481 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18482 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18484 #ifdef ACCEL_COMPILER
18485 chunk_size
= gimple_call_arg (call
, 4);
18486 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18487 || integer_zerop (chunk_size
)) /* Default (also static). */
18489 /* If we're at the gang level, we want each to execute a
18490 contiguous run of iterations. Otherwise we want each element
18492 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18497 /* Chunk of size 1 is striding. */
18498 striding
= integer_onep (chunk_size
);
18499 chunking
= !striding
;
18503 /* striding=true, chunking=true
18505 striding=true, chunking=false
18507 striding=false,chunking=true
18508 -> chunks=ceil (range/(chunksize*threads*step))
18509 striding=false,chunking=false
18510 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18511 push_gimplify_context (true);
18515 default: gcc_unreachable ();
18517 case IFN_GOACC_LOOP_CHUNKS
:
18519 r
= build_int_cst (type
, 1);
18523 = (range - dir) / (chunks * step * num_threads) + dir */
18524 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18525 per
= fold_convert (type
, per
);
18526 chunk_size
= fold_convert (type
, chunk_size
);
18527 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18528 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18529 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18530 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18531 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18535 case IFN_GOACC_LOOP_STEP
:
18537 /* If striding, step by the entire compute volume, otherwise
18538 step by the inner volume. */
18539 unsigned volume
= striding
? mask
: inner_mask
;
18541 r
= oacc_thread_numbers (false, volume
, &seq
);
18542 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18546 case IFN_GOACC_LOOP_OFFSET
:
18549 r
= oacc_thread_numbers (true, mask
, &seq
);
18550 r
= fold_convert (diff_type
, r
);
18554 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18555 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18556 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18557 inner_size
, outer_size
);
18559 volume
= fold_convert (diff_type
, volume
);
18561 chunk_size
= fold_convert (diff_type
, chunk_size
);
18564 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18566 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18567 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18568 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18571 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18572 fold_convert (diff_type
, inner_size
));
18573 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18574 r
= fold_convert (diff_type
, r
);
18575 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18577 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18578 inner
= fold_convert (diff_type
, inner
);
18579 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18583 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18585 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18586 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18588 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18591 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18592 if (type
!= diff_type
)
18593 r
= fold_convert (type
, r
);
18596 case IFN_GOACC_LOOP_BOUND
:
18601 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18602 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18603 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18604 inner_size
, outer_size
);
18606 volume
= fold_convert (diff_type
, volume
);
18608 chunk_size
= fold_convert (diff_type
, chunk_size
);
18611 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18613 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18614 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18615 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18618 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18619 fold_convert (diff_type
, inner_size
));
18621 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18623 tree offset
= gimple_call_arg (call
, 6);
18624 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18625 fold_convert (diff_type
, offset
));
18626 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18627 diff_type
, r
, range
);
18629 if (diff_type
!= type
)
18630 r
= fold_convert (type
, r
);
18634 gimplify_assign (lhs
, r
, &seq
);
18636 pop_gimplify_context (NULL
);
18638 gsi_replace_with_seq (&gsi
, seq
, true);
18641 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18642 raw attribute. DIMS is an array of dimensions, which is returned.
18643 Returns the function level dimensionality -- the level at which an
18644 offload routine wishes to partition a loop. */
18647 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
18649 tree purpose
[GOMP_DIM_MAX
];
18651 tree pos
= TREE_VALUE (attrs
);
18654 /* Make sure the attribute creator attached the dimension
18658 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18660 purpose
[ix
] = TREE_PURPOSE (pos
);
18664 if (integer_zerop (purpose
[ix
]))
18666 else if (fn_level
< 0)
18670 tree val
= TREE_VALUE (pos
);
18671 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18672 pos
= TREE_CHAIN (pos
);
18675 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
18677 /* Default anything left to 1. */
18678 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18687 /* Replace the attribute with new values. */
18689 for (ix
= GOMP_DIM_MAX
; ix
--;)
18690 pos
= tree_cons (purpose
[ix
],
18691 build_int_cst (integer_type_node
, dims
[ix
]),
18693 replace_oacc_fn_attrib (fn
, pos
);
18699 /* Create an empty OpenACC loop structure at LOC. */
18702 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18704 oacc_loop
*loop
= XCNEW (oacc_loop
);
18706 loop
->parent
= parent
;
18707 loop
->child
= loop
->sibling
= NULL
;
18711 loop
->sibling
= parent
->child
;
18712 parent
->child
= loop
;
18716 loop
->marker
= NULL
;
18717 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18718 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18719 loop
->routine
= NULL_TREE
;
18721 loop
->mask
= loop
->flags
= 0;
18722 loop
->chunk_size
= 0;
18723 loop
->head_end
= NULL
;
18728 /* Create an outermost, dummy OpenACC loop for offloaded function
18732 new_oacc_loop_outer (tree decl
)
18734 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18737 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18738 Link into PARENT loop. Return the new loop. */
18741 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18743 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18745 loop
->marker
= marker
;
18747 /* TODO: This is where device_type flattening would occur for the loop
18750 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18752 tree chunk_size
= integer_zero_node
;
18753 if (loop
->flags
& OLF_GANG_STATIC
)
18754 chunk_size
= gimple_call_arg (marker
, 4);
18755 loop
->chunk_size
= chunk_size
;
18760 /* Create a dummy loop encompassing a call to a openACC routine.
18761 Extract the routine's partitioning requirements. */
18764 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18766 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18767 int dims
[GOMP_DIM_MAX
];
18768 int level
= oacc_validate_dims (decl
, attrs
, dims
);
18770 gcc_assert (level
>= 0);
18772 loop
->marker
= call
;
18773 loop
->routine
= decl
;
18774 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18775 ^ (GOMP_DIM_MASK (level
) - 1));
18778 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18779 Return the parent loop. */
18782 finish_oacc_loop (oacc_loop
*loop
)
18784 return loop
->parent
;
18787 /* Free all OpenACC loop structures within LOOP (inclusive). */
18790 free_oacc_loop (oacc_loop
*loop
)
18793 free_oacc_loop (loop
->sibling
);
18795 free_oacc_loop (loop
->child
);
18800 /* Dump out the OpenACC loop head or tail beginning at FROM. */
18803 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
18804 const char *title
, int level
)
18806 enum ifn_unique_kind kind
18807 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
18809 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
18810 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
18812 gimple
*stmt
= gsi_stmt (gsi
);
18814 if (is_gimple_call (stmt
)
18815 && gimple_call_internal_p (stmt
)
18816 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
18818 enum ifn_unique_kind k
18819 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
18820 (gimple_call_arg (stmt
, 0)));
18822 if (k
== kind
&& stmt
!= from
)
18825 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
18828 while (gsi_end_p (gsi
))
18829 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
18833 /* Dump OpenACC loops LOOP, its siblings and its children. */
18836 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
18840 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
18841 loop
->flags
, loop
->mask
,
18842 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
18845 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
18848 fprintf (file
, "%*sRoutine %s:%u:%s\n",
18849 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
18850 DECL_SOURCE_LINE (loop
->routine
),
18851 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
18853 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18854 if (loop
->heads
[ix
])
18855 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
18856 for (ix
= GOMP_DIM_MAX
; ix
--;)
18857 if (loop
->tails
[ix
])
18858 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
18861 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
18863 dump_oacc_loop (file
, loop
->sibling
, depth
);
18866 void debug_oacc_loop (oacc_loop
*);
18868 /* Dump loops to stderr. */
18870 DEBUG_FUNCTION
void
18871 debug_oacc_loop (oacc_loop
*loop
)
18873 dump_oacc_loop (stderr
, loop
, 0);
18876 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
18877 structures as we go. By construction these loops are properly
18881 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
18886 if (bb
->flags
& BB_VISITED
)
18890 bb
->flags
|= BB_VISITED
;
18892 /* Scan for loop markers. */
18893 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
18896 gimple
*stmt
= gsi_stmt (gsi
);
18898 if (!is_gimple_call (stmt
))
18901 gcall
*call
= as_a
<gcall
*> (stmt
);
18903 /* If this is a routine, make a dummy loop for it. */
18904 if (tree decl
= gimple_call_fndecl (call
))
18905 if (tree attrs
= get_oacc_fn_attrib (decl
))
18907 gcc_assert (!marker
);
18908 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
18911 if (!gimple_call_internal_p (call
))
18914 if (gimple_call_internal_fn (call
) != IFN_UNIQUE
)
18917 enum ifn_unique_kind kind
18918 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18919 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
18920 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
18922 if (gimple_call_num_args (call
) == 2)
18924 gcc_assert (marker
&& !remaining
);
18926 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
18927 loop
= finish_oacc_loop (loop
);
18929 loop
->head_end
= call
;
18933 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
18937 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
18938 loop
= new_oacc_loop (loop
, call
);
18941 gcc_assert (count
== remaining
);
18945 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
18946 loop
->heads
[marker
] = call
;
18948 loop
->tails
[remaining
] = call
;
18954 if (remaining
|| marker
)
18956 bb
= single_succ (bb
);
18957 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
18961 /* Walk successor blocks. */
18965 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
18966 oacc_loop_discover_walk (loop
, e
->dest
);
18969 /* LOOP is the first sibling. Reverse the order in place and return
18970 the new first sibling. Recurse to child loops. */
18973 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
18975 oacc_loop
*last
= NULL
;
18979 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
18981 oacc_loop
*next
= loop
->sibling
;
18982 loop
->sibling
= last
;
18991 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
18992 the current function. */
18995 oacc_loop_discovery ()
18999 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19000 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19002 /* The siblings were constructed in reverse order, reverse them so
19003 that diagnostics come out in an unsurprising order. */
19004 top
= oacc_loop_sibling_nreverse (top
);
19006 /* Reset the visited flags. */
19007 FOR_ALL_BB_FN (bb
, cfun
)
19008 bb
->flags
&= ~BB_VISITED
;
19013 /* Transform the abstract internal function markers starting at FROM
19014 to be for partitioning level LEVEL. Stop when we meet another HEAD
19018 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19020 enum ifn_unique_kind kind
19021 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19022 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19024 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19026 gimple
*stmt
= gsi_stmt (gsi
);
19028 if (is_gimple_call (stmt
)
19029 && gimple_call_internal_p (stmt
)
19030 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19032 enum ifn_unique_kind k
19033 = ((enum ifn_unique_kind
)
19034 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19036 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19037 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19038 else if (k
== kind
&& stmt
!= from
)
19042 while (gsi_end_p (gsi
))
19043 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19047 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19048 determined partitioning mask and chunking argument. */
19051 oacc_loop_xform_loop (gcall
*end_marker
, tree mask_arg
, tree chunk_arg
)
19053 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19057 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19059 gimple
*stmt
= gsi_stmt (gsi
);
19061 if (!is_gimple_call (stmt
))
19064 gcall
*call
= as_a
<gcall
*> (stmt
);
19066 if (!gimple_call_internal_p (call
))
19069 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19072 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19073 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19074 if (TREE_INT_CST_LOW (gimple_call_arg (call
, 0))
19075 == IFN_GOACC_LOOP_BOUND
)
19079 /* If we didn't see LOOP_BOUND, it should be in the single
19080 successor block. */
19081 basic_block bb
= single_succ (gsi_bb (gsi
));
19082 gsi
= gsi_start_bb (bb
);
19086 /* Process the discovered OpenACC loops, setting the correct
19087 partitioning level etc. */
19090 oacc_loop_process (oacc_loop
*loop
)
19093 oacc_loop_process (loop
->child
);
19095 if (loop
->mask
&& !loop
->routine
)
19098 unsigned mask
= loop
->mask
;
19099 unsigned dim
= GOMP_DIM_GANG
;
19100 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19101 tree chunk_arg
= loop
->chunk_size
;
19103 oacc_loop_xform_loop (loop
->head_end
, mask_arg
, chunk_arg
);
19105 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
19109 while (!(GOMP_DIM_MASK (dim
) & mask
))
19112 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19113 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19115 mask
^= GOMP_DIM_MASK (dim
);
19120 oacc_loop_process (loop
->sibling
);
19123 /* Walk the OpenACC loop heirarchy checking and assigning the
19124 programmer-specified partitionings. OUTER_MASK is the partitioning
19125 this loop is contained within. Return partitiong mask used within
19129 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19131 unsigned this_mask
= loop
->mask
;
19132 bool has_auto
= false;
19135 #ifdef ACCEL_COMPILER
19136 /* When device_type is supported, we want the device compiler to be
19137 noisy, if the loop parameters are device_type-specific. */
19141 if (!loop
->routine
)
19143 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19144 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19146 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19147 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19149 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19152 error_at (loop
->loc
,
19154 ? "%<seq%> overrides other OpenACC loop specifiers"
19155 : "%<auto%> conflicts with other OpenACC loop specifiers");
19157 loop
->flags
&= ~OLF_AUTO
;
19161 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19165 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19169 if (this_mask
& outer_mask
)
19171 const oacc_loop
*outer
;
19172 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19173 if (outer
->mask
& this_mask
)
19180 error_at (loop
->loc
,
19181 "%s uses same OpenACC parallelism as containing loop",
19182 loop
->routine
? "routine call" : "inner loop");
19183 inform (outer
->loc
, "containing loop here");
19186 error_at (loop
->loc
,
19187 "%s uses OpenACC parallelism disallowed by containing routine",
19188 loop
->routine
? "routine call" : "loop");
19191 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19192 "routine %qD declared here", loop
->routine
);
19194 this_mask
&= ~outer_mask
;
19198 unsigned outermost
= this_mask
& -this_mask
;
19200 if (outermost
&& outermost
<= outer_mask
)
19204 error_at (loop
->loc
,
19205 "incorrectly nested OpenACC loop parallelism");
19207 const oacc_loop
*outer
;
19208 for (outer
= loop
->parent
;
19209 outer
->flags
&& outer
->flags
< outermost
;
19210 outer
= outer
->parent
)
19212 inform (outer
->loc
, "containing loop here");
19215 this_mask
&= ~outermost
;
19219 loop
->mask
= this_mask
;
19222 && oacc_loop_fixed_partitions (loop
->child
, outer_mask
| this_mask
))
19226 && oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
))
19232 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19236 oacc_loop_partition (oacc_loop
*loop
, int fn_level
)
19238 unsigned outer_mask
= 0;
19241 outer_mask
= GOMP_DIM_MASK (fn_level
) - 1;
19243 oacc_loop_fixed_partitions (loop
, outer_mask
);
19246 /* Default fork/join early expander. Delete the function calls if
19247 there is no RTL expander. */
19250 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19251 const int *ARG_UNUSED (dims
), bool is_fork
)
19254 return targetm
.have_oacc_fork ();
19256 return targetm
.have_oacc_join ();
19259 /* Main entry point for oacc transformations which run on the device
19260 compiler after LTO, so we know what the target device is at this
19261 point (including the host fallback). */
19263 static unsigned int
19264 execute_oacc_device_lower ()
19266 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19267 int dims
[GOMP_DIM_MAX
];
19270 /* Not an offloaded function. */
19273 int fn_level
= oacc_validate_dims (current_function_decl
, attrs
, dims
);
19275 /* Discover, partition and process the loops. */
19276 oacc_loop
*loops
= oacc_loop_discovery ();
19277 oacc_loop_partition (loops
, fn_level
);
19278 oacc_loop_process (loops
);
19281 fprintf (dump_file
, "OpenACC loops\n");
19282 dump_oacc_loop (dump_file
, loops
, 0);
19283 fprintf (dump_file
, "\n");
19286 /* Now lower internal loop functions to target-specific code
19289 FOR_ALL_BB_FN (bb
, cfun
)
19290 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19292 gimple
*stmt
= gsi_stmt (gsi
);
19293 if (!is_gimple_call (stmt
))
19299 gcall
*call
= as_a
<gcall
*> (stmt
);
19300 if (!gimple_call_internal_p (call
))
19306 /* Rewind to allow rescan. */
19308 bool rescan
= false, remove
= false;
19309 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19315 case IFN_GOACC_LOOP
:
19316 oacc_xform_loop (call
);
19322 enum ifn_unique_kind kind
19323 = ((enum ifn_unique_kind
)
19324 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19329 gcc_unreachable ();
19331 case IFN_UNIQUE_OACC_FORK
:
19332 case IFN_UNIQUE_OACC_JOIN
:
19333 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19335 else if (!targetm
.goacc
.fork_join
19336 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19340 case IFN_UNIQUE_OACC_HEAD_MARK
:
19341 case IFN_UNIQUE_OACC_TAIL_MARK
:
19349 if (gsi_end_p (gsi
))
19350 /* We rewound past the beginning of the BB. */
19351 gsi
= gsi_start_bb (bb
);
19353 /* Undo the rewind. */
19358 if (gimple_vdef (call
))
19359 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19360 if (gimple_call_lhs (call
))
19362 /* Propagate the data dependency var. */
19363 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19364 gimple_call_arg (call
, 1));
19365 gsi_replace (&gsi
, ass
, false);
19368 gsi_remove (&gsi
, true);
19371 /* If not rescanning, advance over the call. */
19375 free_oacc_loop (loops
);
19380 /* Default launch dimension validator. Force everything to 1. A
19381 backend that wants to provide larger dimensions must override this
19385 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19386 int ARG_UNUSED (fn_level
))
19388 bool changed
= false;
19390 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19404 const pass_data pass_data_oacc_device_lower
=
19406 GIMPLE_PASS
, /* type */
19407 "oaccdevlow", /* name */
19408 OPTGROUP_NONE
, /* optinfo_flags */
19409 TV_NONE
, /* tv_id */
19410 PROP_cfg
, /* properties_required */
19411 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19412 0, /* properties_destroyed */
19413 0, /* todo_flags_start */
19414 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19417 class pass_oacc_device_lower
: public gimple_opt_pass
19420 pass_oacc_device_lower (gcc::context
*ctxt
)
19421 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19424 /* opt_pass methods: */
19425 virtual unsigned int execute (function
*)
19427 bool gate
= flag_openacc
!= 0;
19432 return execute_oacc_device_lower ();
19435 }; // class pass_oacc_transform
19437 } // anon namespace
19440 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19442 return new pass_oacc_device_lower (ctxt
);
19445 #include "gt-omp-low.h"