1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
38 #include "internal-fn.h"
39 #include "gimple-fold.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
50 #include "tree-into-ssa.h"
52 #include "insn-config.h"
63 #include "tree-pass.h"
65 #include "splay-tree.h"
66 #include "insn-codes.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "pretty-print.h"
75 #include "alloc-pool.h"
76 #include "symbol-summary.h"
78 #include "tree-nested.h"
82 #include "lto-section-names.h"
83 #include "gomp-constants.h"
85 /* Lowering of OMP parallel and workshare constructs proceeds in two
86 phases. The first phase scans the function looking for OMP statements
87 and then for variables that must be replaced to satisfy data sharing
88 clauses. The second phase expands code for the constructs, as well as
89 re-gimplifying things when variables have been replaced with complex
92 Final code generation is done by pass_expand_omp. The flowgraph is
93 scanned for regions which are then moved to a new
94 function, to be invoked by the thread library, or offloaded. */
96 /* OMP region information. Every parallel and workshare
97 directive is enclosed between two markers, the OMP_* directive
98 and a corresponding GIMPLE_OMP_RETURN statement. */
102 /* The enclosing region. */
103 struct omp_region
*outer
;
105 /* First child region. */
106 struct omp_region
*inner
;
108 /* Next peer region. */
109 struct omp_region
*next
;
111 /* Block containing the omp directive as its last stmt. */
114 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
117 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
120 /* If this is a combined parallel+workshare region, this is a list
121 of additional arguments needed by the combined parallel+workshare
123 vec
<tree
, va_gc
> *ws_args
;
125 /* The code for the omp directive of this region. */
126 enum gimple_code type
;
128 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
129 enum omp_clause_schedule_kind sched_kind
;
131 /* True if this is a combined parallel+workshare region. */
132 bool is_combined_parallel
;
134 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
136 gomp_ordered
*ord_stmt
;
139 /* Levels of parallelism as defined by OpenACC. Increasing numbers
140 correspond to deeper loop nesting levels. */
142 #define MASK_WORKER 2
143 #define MASK_VECTOR 4
145 /* Context structure. Used to store information about each parallel
146 directive in the code. */
150 /* This field must be at the beginning, as we do "inheritance": Some
151 callback functions for tree-inline.c (e.g., omp_copy_decl)
152 receive a copy_body_data pointer that is up-casted to an
153 omp_context pointer. */
156 /* The tree of contexts corresponding to the encountered constructs. */
157 struct omp_context
*outer
;
160 /* Map variables to fields in a structure that allows communication
161 between sending and receiving threads. */
162 splay_tree field_map
;
167 /* These are used just by task contexts, if task firstprivate fn is
168 needed. srecord_type is used to communicate from the thread
169 that encountered the task construct to task firstprivate fn,
170 record_type is allocated by GOMP_task, initialized by task firstprivate
171 fn and passed to the task body fn. */
172 splay_tree sfield_map
;
175 /* A chain of variables to add to the top-level block surrounding the
176 construct. In the case of a parallel, this is in the child function. */
179 /* A map of reduction pointer variables. For accelerators, each
180 reduction variable is replaced with an array. Each thread, in turn,
181 is assigned to a slot on that array. */
182 splay_tree reduction_map
;
184 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
185 barriers should jump to during omplower pass. */
188 /* What to do with variables with implicitly determined sharing
190 enum omp_clause_default_kind default_kind
;
192 /* Nesting depth of this context. Used to beautify error messages re
193 invalid gotos. The outermost ctx is depth 1, with depth 0 being
194 reserved for the main body of the function. */
197 /* True if this parallel directive is nested within another. */
200 /* True if this construct can be cancelled. */
203 /* For OpenACC loops, a mask of gang, worker and vector used at
204 levels below this one. */
206 /* For OpenACC loops, a mask of gang, worker and vector used at
207 this level and above. For parallel and kernels clauses, a mask
208 indicating which of num_gangs/num_workers/num_vectors was used. */
212 /* A structure holding the elements of:
213 for (V = N1; V cond N2; V += STEP) [...] */
215 struct omp_for_data_loop
217 tree v
, n1
, n2
, step
;
218 enum tree_code cond_code
;
221 /* A structure describing the main elements of a parallel loop. */
225 struct omp_for_data_loop loop
;
231 bool have_nowait
, have_ordered
, simd_schedule
;
232 enum omp_clause_schedule_kind sched_kind
;
233 struct omp_for_data_loop
*loops
;
237 static splay_tree all_contexts
;
238 static int taskreg_nesting_level
;
239 static int target_nesting_level
;
240 static struct omp_region
*root_omp_region
;
241 static bitmap task_shared_vars
;
242 static vec
<omp_context
*> taskreg_contexts
;
244 static void scan_omp (gimple_seq
*, omp_context
*);
245 static tree
scan_omp_1_op (tree
*, int *, void *);
246 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
248 #define WALK_SUBSTMTS \
252 case GIMPLE_EH_FILTER: \
253 case GIMPLE_TRANSACTION: \
254 /* The sub-statements for these should be walked. */ \
255 *handled_ops_p = false; \
258 /* Helper function to get the name of the array containing the partial
259 reductions for OpenACC reductions. */
261 oacc_get_reduction_array_id (tree node
)
263 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
264 int len
= strlen ("OACC") + strlen (id
);
265 char *temp_name
= XALLOCAVEC (char, len
+ 1);
266 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
267 return IDENTIFIER_POINTER (get_identifier (temp_name
));
270 /* Determine the number of threads OpenACC threads used to determine the
271 size of the array of partial reductions. Currently, this is num_gangs
272 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
273 because it is independed of the device used. */
276 oacc_max_threads (omp_context
*ctx
)
278 tree nthreads
, vector_length
, gangs
, clauses
;
280 gangs
= fold_convert (sizetype
, integer_one_node
);
281 vector_length
= gangs
;
283 /* The reduction clause may be nested inside a loop directive.
284 Scan for the innermost vector_length clause. */
285 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
287 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
288 || (gimple_omp_target_kind (oc
->stmt
)
289 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
292 clauses
= gimple_omp_target_clauses (oc
->stmt
);
294 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
296 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
298 OMP_CLAUSE_VECTOR_LENGTH_EXPR
301 vector_length
= fold_convert (sizetype
, integer_one_node
);
303 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
305 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
306 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
308 gangs
= fold_convert (sizetype
, integer_one_node
);
313 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
318 /* If DECL is the artificial dummy VAR_DECL created for non-static
319 data member privatization, return the underlying "this" parameter,
320 otherwise return NULL. */
323 omp_member_access_dummy_var (tree decl
)
326 || !DECL_ARTIFICIAL (decl
)
327 || !DECL_IGNORED_P (decl
)
328 || !DECL_HAS_VALUE_EXPR_P (decl
)
329 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
332 tree v
= DECL_VALUE_EXPR (decl
);
333 if (TREE_CODE (v
) != COMPONENT_REF
)
337 switch (TREE_CODE (v
))
343 case POINTER_PLUS_EXPR
:
344 v
= TREE_OPERAND (v
, 0);
347 if (DECL_CONTEXT (v
) == current_function_decl
348 && DECL_ARTIFICIAL (v
)
349 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
357 /* Helper for unshare_and_remap, called through walk_tree. */
360 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
362 tree
*pair
= (tree
*) data
;
365 *tp
= unshare_expr (pair
[1]);
368 else if (IS_TYPE_OR_DECL_P (*tp
))
373 /* Return unshare_expr (X) with all occurrences of FROM
377 unshare_and_remap (tree x
, tree from
, tree to
)
379 tree pair
[2] = { from
, to
};
380 x
= unshare_expr (x
);
381 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
385 /* Holds offload tables with decls. */
386 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
388 /* Convenience function for calling scan_omp_1_op on tree operands. */
391 scan_omp_op (tree
*tp
, omp_context
*ctx
)
393 struct walk_stmt_info wi
;
395 memset (&wi
, 0, sizeof (wi
));
397 wi
.want_locations
= true;
399 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
402 static void lower_omp (gimple_seq
*, omp_context
*);
403 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
404 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
406 /* Find an OMP clause of type KIND within CLAUSES. */
409 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
411 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
412 if (OMP_CLAUSE_CODE (clauses
) == kind
)
418 /* Return true if CTX is for an omp parallel. */
421 is_parallel_ctx (omp_context
*ctx
)
423 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
427 /* Return true if CTX is for an omp task. */
430 is_task_ctx (omp_context
*ctx
)
432 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
436 /* Return true if CTX is for an omp taskloop. */
439 is_taskloop_ctx (omp_context
*ctx
)
441 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
442 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
446 /* Return true if CTX is for an omp parallel or omp task. */
449 is_taskreg_ctx (omp_context
*ctx
)
451 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
455 /* Return true if REGION is a combined parallel+workshare region. */
458 is_combined_parallel (struct omp_region
*region
)
460 return region
->is_combined_parallel
;
464 /* Extract the header elements of parallel loop FOR_STMT and store
468 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
469 struct omp_for_data_loop
*loops
)
471 tree t
, var
, *collapse_iter
, *collapse_count
;
472 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
473 struct omp_for_data_loop
*loop
;
475 struct omp_for_data_loop dummy_loop
;
476 location_t loc
= gimple_location (for_stmt
);
477 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
478 bool distribute
= gimple_omp_for_kind (for_stmt
)
479 == GF_OMP_FOR_KIND_DISTRIBUTE
;
480 bool taskloop
= gimple_omp_for_kind (for_stmt
)
481 == GF_OMP_FOR_KIND_TASKLOOP
;
484 fd
->for_stmt
= for_stmt
;
486 if (gimple_omp_for_collapse (for_stmt
) > 1)
489 fd
->loops
= &fd
->loop
;
491 fd
->have_nowait
= distribute
|| simd
;
492 fd
->have_ordered
= false;
495 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
496 fd
->chunk_size
= NULL_TREE
;
497 fd
->simd_schedule
= false;
498 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
499 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
500 collapse_iter
= NULL
;
501 collapse_count
= NULL
;
503 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
504 switch (OMP_CLAUSE_CODE (t
))
506 case OMP_CLAUSE_NOWAIT
:
507 fd
->have_nowait
= true;
509 case OMP_CLAUSE_ORDERED
:
510 fd
->have_ordered
= true;
511 if (OMP_CLAUSE_ORDERED_EXPR (t
))
512 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
514 case OMP_CLAUSE_SCHEDULE
:
515 gcc_assert (!distribute
&& !taskloop
);
516 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
517 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
518 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
520 case OMP_CLAUSE_DIST_SCHEDULE
:
521 gcc_assert (distribute
);
522 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
524 case OMP_CLAUSE_COLLAPSE
:
525 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
526 if (fd
->collapse
> 1)
528 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
529 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
535 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
540 collapse_iter
= &iterv
;
541 collapse_count
= &countv
;
544 /* FIXME: for now map schedule(auto) to schedule(static).
545 There should be analysis to determine whether all iterations
546 are approximately the same amount of work (then schedule(static)
547 is best) or if it varies (then schedule(dynamic,N) is better). */
548 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
550 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
551 gcc_assert (fd
->chunk_size
== NULL
);
553 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
555 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
556 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
557 gcc_assert (fd
->chunk_size
== NULL
);
558 else if (fd
->chunk_size
== NULL
)
560 /* We only need to compute a default chunk size for ordered
561 static loops and dynamic loops. */
562 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
564 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
565 ? integer_zero_node
: integer_one_node
;
568 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
569 for (i
= 0; i
< cnt
; i
++)
571 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
573 else if (loops
!= NULL
)
578 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
579 gcc_assert (SSA_VAR_P (loop
->v
));
580 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
581 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
582 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
583 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
585 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
586 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
587 switch (loop
->cond_code
)
593 gcc_assert (gimple_omp_for_kind (for_stmt
)
594 == GF_OMP_FOR_KIND_CILKSIMD
595 || (gimple_omp_for_kind (for_stmt
)
596 == GF_OMP_FOR_KIND_CILKFOR
));
599 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
600 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
602 loop
->n2
= fold_build2_loc (loc
,
603 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
604 build_int_cst (TREE_TYPE (loop
->n2
), 1));
605 loop
->cond_code
= LT_EXPR
;
608 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
609 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
611 loop
->n2
= fold_build2_loc (loc
,
612 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
613 build_int_cst (TREE_TYPE (loop
->n2
), 1));
614 loop
->cond_code
= GT_EXPR
;
620 t
= gimple_omp_for_incr (for_stmt
, i
);
621 gcc_assert (TREE_OPERAND (t
, 0) == var
);
622 switch (TREE_CODE (t
))
625 loop
->step
= TREE_OPERAND (t
, 1);
627 case POINTER_PLUS_EXPR
:
628 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
631 loop
->step
= TREE_OPERAND (t
, 1);
632 loop
->step
= fold_build1_loc (loc
,
633 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
641 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
642 && !fd
->have_ordered
))
644 if (fd
->collapse
== 1)
645 iter_type
= TREE_TYPE (loop
->v
);
647 || TYPE_PRECISION (iter_type
)
648 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
650 = build_nonstandard_integer_type
651 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
653 else if (iter_type
!= long_long_unsigned_type_node
)
655 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
656 iter_type
= long_long_unsigned_type_node
;
657 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
658 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
659 >= TYPE_PRECISION (iter_type
))
663 if (loop
->cond_code
== LT_EXPR
)
664 n
= fold_build2_loc (loc
,
665 PLUS_EXPR
, TREE_TYPE (loop
->v
),
666 loop
->n2
, loop
->step
);
669 if (TREE_CODE (n
) != INTEGER_CST
670 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
671 iter_type
= long_long_unsigned_type_node
;
673 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
674 > TYPE_PRECISION (iter_type
))
678 if (loop
->cond_code
== LT_EXPR
)
681 n2
= fold_build2_loc (loc
,
682 PLUS_EXPR
, TREE_TYPE (loop
->v
),
683 loop
->n2
, loop
->step
);
687 n1
= fold_build2_loc (loc
,
688 MINUS_EXPR
, TREE_TYPE (loop
->v
),
689 loop
->n2
, loop
->step
);
692 if (TREE_CODE (n1
) != INTEGER_CST
693 || TREE_CODE (n2
) != INTEGER_CST
694 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
695 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
696 iter_type
= long_long_unsigned_type_node
;
700 if (i
>= fd
->collapse
)
703 if (collapse_count
&& *collapse_count
== NULL
)
705 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
706 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
707 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
708 if (t
&& integer_zerop (t
))
709 count
= build_zero_cst (long_long_unsigned_type_node
);
710 else if ((i
== 0 || count
!= NULL_TREE
)
711 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
712 && TREE_CONSTANT (loop
->n1
)
713 && TREE_CONSTANT (loop
->n2
)
714 && TREE_CODE (loop
->step
) == INTEGER_CST
)
716 tree itype
= TREE_TYPE (loop
->v
);
718 if (POINTER_TYPE_P (itype
))
719 itype
= signed_type_for (itype
);
720 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
721 t
= fold_build2_loc (loc
,
723 fold_convert_loc (loc
, itype
, loop
->step
), t
);
724 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
725 fold_convert_loc (loc
, itype
, loop
->n2
));
726 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
727 fold_convert_loc (loc
, itype
, loop
->n1
));
728 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
729 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
730 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
731 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
732 fold_convert_loc (loc
, itype
,
735 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
736 fold_convert_loc (loc
, itype
, loop
->step
));
737 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
738 if (count
!= NULL_TREE
)
739 count
= fold_build2_loc (loc
,
740 MULT_EXPR
, long_long_unsigned_type_node
,
744 if (TREE_CODE (count
) != INTEGER_CST
)
747 else if (count
&& !integer_zerop (count
))
754 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
755 || fd
->have_ordered
))
757 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
758 iter_type
= long_long_unsigned_type_node
;
760 iter_type
= long_integer_type_node
;
762 else if (collapse_iter
&& *collapse_iter
!= NULL
)
763 iter_type
= TREE_TYPE (*collapse_iter
);
764 fd
->iter_type
= iter_type
;
765 if (collapse_iter
&& *collapse_iter
== NULL
)
766 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
767 if (collapse_count
&& *collapse_count
== NULL
)
770 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
772 *collapse_count
= create_tmp_var (iter_type
, ".count");
775 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
777 fd
->loop
.v
= *collapse_iter
;
778 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
779 fd
->loop
.n2
= *collapse_count
;
780 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
781 fd
->loop
.cond_code
= LT_EXPR
;
786 /* For OpenACC loops, force a chunk size of one, as this avoids the default
787 scheduling where several subsequent iterations are being executed by the
789 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
791 gcc_assert (fd
->chunk_size
== NULL_TREE
);
792 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
797 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
798 is the immediate dominator of PAR_ENTRY_BB, return true if there
799 are no data dependencies that would prevent expanding the parallel
800 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
802 When expanding a combined parallel+workshare region, the call to
803 the child function may need additional arguments in the case of
804 GIMPLE_OMP_FOR regions. In some cases, these arguments are
805 computed out of variables passed in from the parent to the child
806 via 'struct .omp_data_s'. For instance:
808 #pragma omp parallel for schedule (guided, i * 4)
813 # BLOCK 2 (PAR_ENTRY_BB)
815 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
817 # BLOCK 3 (WS_ENTRY_BB)
818 .omp_data_i = &.omp_data_o;
819 D.1667 = .omp_data_i->i;
821 #pragma omp for schedule (guided, D.1598)
823 When we outline the parallel region, the call to the child function
824 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
825 that value is computed *after* the call site. So, in principle we
826 cannot do the transformation.
828 To see whether the code in WS_ENTRY_BB blocks the combined
829 parallel+workshare call, we collect all the variables used in the
830 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
831 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
834 FIXME. If we had the SSA form built at this point, we could merely
835 hoist the code in block 3 into block 2 and be done with it. But at
836 this point we don't have dataflow information and though we could
837 hack something up here, it is really not worth the aggravation. */
840 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
842 struct omp_for_data fd
;
843 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
845 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
848 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
850 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
852 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
854 if (fd
.iter_type
!= long_integer_type_node
)
857 /* FIXME. We give up too easily here. If any of these arguments
858 are not constants, they will likely involve variables that have
859 been mapped into fields of .omp_data_s for sharing with the child
860 function. With appropriate data flow, it would be possible to
862 if (!is_gimple_min_invariant (fd
.loop
.n1
)
863 || !is_gimple_min_invariant (fd
.loop
.n2
)
864 || !is_gimple_min_invariant (fd
.loop
.step
)
865 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
872 static int omp_max_vf (void);
874 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
875 presence (SIMD_SCHEDULE). */
878 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
883 int vf
= omp_max_vf ();
887 tree type
= TREE_TYPE (chunk_size
);
888 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
889 build_int_cst (type
, vf
- 1));
890 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
891 build_int_cst (type
, -vf
));
895 /* Collect additional arguments needed to emit a combined
896 parallel+workshare call. WS_STMT is the workshare directive being
899 static vec
<tree
, va_gc
> *
900 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
903 location_t loc
= gimple_location (ws_stmt
);
904 vec
<tree
, va_gc
> *ws_args
;
906 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
908 struct omp_for_data fd
;
911 extract_omp_for_data (for_stmt
, &fd
, NULL
);
915 if (gimple_omp_for_combined_into_p (for_stmt
))
918 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
919 OMP_CLAUSE__LOOPTEMP_
);
921 n1
= OMP_CLAUSE_DECL (innerc
);
922 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
923 OMP_CLAUSE__LOOPTEMP_
);
925 n2
= OMP_CLAUSE_DECL (innerc
);
928 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
930 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
931 ws_args
->quick_push (t
);
933 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
934 ws_args
->quick_push (t
);
936 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
937 ws_args
->quick_push (t
);
941 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
942 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
943 ws_args
->quick_push (t
);
948 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
950 /* Number of sections is equal to the number of edges from the
951 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
952 the exit of the sections region. */
953 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
954 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
955 vec_alloc (ws_args
, 1);
956 ws_args
->quick_push (t
);
964 /* Discover whether REGION is a combined parallel+workshare region. */
967 determine_parallel_type (struct omp_region
*region
)
969 basic_block par_entry_bb
, par_exit_bb
;
970 basic_block ws_entry_bb
, ws_exit_bb
;
972 if (region
== NULL
|| region
->inner
== NULL
973 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
974 || region
->inner
->cont
== NULL
)
977 /* We only support parallel+for and parallel+sections. */
978 if (region
->type
!= GIMPLE_OMP_PARALLEL
979 || (region
->inner
->type
!= GIMPLE_OMP_FOR
980 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
983 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
984 WS_EXIT_BB -> PAR_EXIT_BB. */
985 par_entry_bb
= region
->entry
;
986 par_exit_bb
= region
->exit
;
987 ws_entry_bb
= region
->inner
->entry
;
988 ws_exit_bb
= region
->inner
->exit
;
990 if (single_succ (par_entry_bb
) == ws_entry_bb
991 && single_succ (ws_exit_bb
) == par_exit_bb
992 && workshare_safe_to_combine_p (ws_entry_bb
)
993 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
994 || (last_and_only_stmt (ws_entry_bb
)
995 && last_and_only_stmt (par_exit_bb
))))
997 gimple
*par_stmt
= last_stmt (par_entry_bb
);
998 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1000 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1002 /* If this is a combined parallel loop, we need to determine
1003 whether or not to use the combined library calls. There
1004 are two cases where we do not apply the transformation:
1005 static loops and any kind of ordered loop. In the first
1006 case, we already open code the loop so there is no need
1007 to do anything else. In the latter case, the combined
1008 parallel loop call would still need extra synchronization
1009 to implement ordered semantics, so there would not be any
1010 gain in using the combined call. */
1011 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1012 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1014 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
1015 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1017 region
->is_combined_parallel
= false;
1018 region
->inner
->is_combined_parallel
= false;
1023 region
->is_combined_parallel
= true;
1024 region
->inner
->is_combined_parallel
= true;
1025 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1030 /* Return true if EXPR is variable sized. */
1033 is_variable_sized (const_tree expr
)
1035 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1038 /* Return true if DECL is a reference type. */
1041 is_reference (tree decl
)
1043 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1046 /* Return the type of a decl. If the decl is reference type,
1047 return its base type. */
1049 get_base_type (tree decl
)
1051 tree type
= TREE_TYPE (decl
);
1052 if (is_reference (decl
))
1053 type
= TREE_TYPE (type
);
1057 /* Lookup variables. The "maybe" form
1058 allows for the variable form to not have been entered, otherwise we
1059 assert that the variable must have been entered. */
1062 lookup_decl (tree var
, omp_context
*ctx
)
1064 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1069 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1071 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1072 return n
? *n
: NULL_TREE
;
1076 lookup_field (tree var
, omp_context
*ctx
)
1079 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1080 return (tree
) n
->value
;
1084 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1087 n
= splay_tree_lookup (ctx
->sfield_map
1088 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1089 return (tree
) n
->value
;
1093 lookup_sfield (tree var
, omp_context
*ctx
)
1095 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1099 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1102 n
= splay_tree_lookup (ctx
->field_map
, key
);
1103 return n
? (tree
) n
->value
: NULL_TREE
;
1107 maybe_lookup_field (tree var
, omp_context
*ctx
)
1109 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1113 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
1116 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
1117 return (tree
) n
->value
;
1121 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1123 splay_tree_node n
= NULL
;
1124 if (ctx
->reduction_map
)
1125 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1126 return n
? (tree
) n
->value
: NULL_TREE
;
1129 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1130 the parallel context if DECL is to be shared. */
1133 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1135 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1138 /* We can only use copy-in/copy-out semantics for shared variables
1139 when we know the value is not accessible from an outer scope. */
1142 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1144 /* ??? Trivially accessible from anywhere. But why would we even
1145 be passing an address in this case? Should we simply assert
1146 this to be false, or should we have a cleanup pass that removes
1147 these from the list of mappings? */
1148 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1151 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1152 without analyzing the expression whether or not its location
1153 is accessible to anyone else. In the case of nested parallel
1154 regions it certainly may be. */
1155 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1158 /* Do not use copy-in/copy-out for variables that have their
1160 if (TREE_ADDRESSABLE (decl
))
1163 /* lower_send_shared_vars only uses copy-in, but not copy-out
1165 if (TREE_READONLY (decl
)
1166 || ((TREE_CODE (decl
) == RESULT_DECL
1167 || TREE_CODE (decl
) == PARM_DECL
)
1168 && DECL_BY_REFERENCE (decl
)))
1171 /* Disallow copy-in/out in nested parallel if
1172 decl is shared in outer parallel, otherwise
1173 each thread could store the shared variable
1174 in its own copy-in location, making the
1175 variable no longer really shared. */
1176 if (shared_ctx
->is_nested
)
1180 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1181 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1188 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1189 c
; c
= OMP_CLAUSE_CHAIN (c
))
1190 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1191 && OMP_CLAUSE_DECL (c
) == decl
)
1195 goto maybe_mark_addressable_and_ret
;
1199 /* For tasks avoid using copy-in/out. As tasks can be
1200 deferred or executed in different thread, when GOMP_task
1201 returns, the task hasn't necessarily terminated. */
1202 if (is_task_ctx (shared_ctx
))
1205 maybe_mark_addressable_and_ret
:
1206 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1207 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1209 /* Taking address of OUTER in lower_send_shared_vars
1210 might need regimplification of everything that uses the
1212 if (!task_shared_vars
)
1213 task_shared_vars
= BITMAP_ALLOC (NULL
);
1214 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1215 TREE_ADDRESSABLE (outer
) = 1;
1224 /* Construct a new automatic decl similar to VAR. */
1227 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1229 tree copy
= copy_var_decl (var
, name
, type
);
1231 DECL_CONTEXT (copy
) = current_function_decl
;
1232 DECL_CHAIN (copy
) = ctx
->block_vars
;
1233 /* If VAR is listed in task_shared_vars, it means it wasn't
1234 originally addressable and is just because task needs to take
1235 it's address. But we don't need to take address of privatizations
1237 if (TREE_ADDRESSABLE (var
)
1239 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1240 TREE_ADDRESSABLE (copy
) = 0;
1241 ctx
->block_vars
= copy
;
1247 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1249 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1252 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1255 omp_build_component_ref (tree obj
, tree field
)
1257 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1258 if (TREE_THIS_VOLATILE (field
))
1259 TREE_THIS_VOLATILE (ret
) |= 1;
1260 if (TREE_READONLY (field
))
1261 TREE_READONLY (ret
) |= 1;
1265 /* Build tree nodes to access the field for VAR on the receiver side. */
1268 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1270 tree x
, field
= lookup_field (var
, ctx
);
1272 /* If the receiver record type was remapped in the child function,
1273 remap the field into the new record type. */
1274 x
= maybe_lookup_field (field
, ctx
);
1278 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1279 TREE_THIS_NOTRAP (x
) = 1;
1280 x
= omp_build_component_ref (x
, field
);
1282 x
= build_simple_mem_ref (x
);
1287 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1288 of a parallel, this is a component reference; for workshare constructs
1289 this is some variable. */
1292 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1296 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1298 else if (is_variable_sized (var
))
1300 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1301 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1302 x
= build_simple_mem_ref (x
);
1304 else if (is_taskreg_ctx (ctx
))
1306 bool by_ref
= use_pointer_for_field (var
, NULL
);
1307 x
= build_receiver_ref (var
, by_ref
, ctx
);
1309 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1310 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1312 /* #pragma omp simd isn't a worksharing construct, and can reference even
1313 private vars in its linear etc. clauses. */
1315 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1316 x
= lookup_decl (var
, ctx
->outer
);
1317 else if (ctx
->outer
)
1318 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1322 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1324 gcc_assert (ctx
->outer
);
1326 = splay_tree_lookup (ctx
->outer
->field_map
,
1327 (splay_tree_key
) &DECL_UID (var
));
1330 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1333 x
= lookup_decl (var
, ctx
->outer
);
1337 tree field
= (tree
) n
->value
;
1338 /* If the receiver record type was remapped in the child function,
1339 remap the field into the new record type. */
1340 x
= maybe_lookup_field (field
, ctx
->outer
);
1344 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1345 x
= omp_build_component_ref (x
, field
);
1346 if (use_pointer_for_field (var
, ctx
->outer
))
1347 x
= build_simple_mem_ref (x
);
1350 else if (ctx
->outer
)
1351 x
= lookup_decl (var
, ctx
->outer
);
1352 else if (is_reference (var
))
1353 /* This can happen with orphaned constructs. If var is reference, it is
1354 possible it is shared and as such valid. */
1356 else if (omp_member_access_dummy_var (var
))
1363 tree t
= omp_member_access_dummy_var (var
);
1366 x
= DECL_VALUE_EXPR (var
);
1367 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1369 x
= unshare_and_remap (x
, t
, o
);
1371 x
= unshare_expr (x
);
1375 if (is_reference (var
))
1376 x
= build_simple_mem_ref (x
);
1381 /* Build tree nodes to access the field for VAR on the sender side. */
1384 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1386 tree field
= lookup_sfield (key
, ctx
);
1387 return omp_build_component_ref (ctx
->sender_decl
, field
);
1391 build_sender_ref (tree var
, omp_context
*ctx
)
1393 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1396 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1399 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1401 tree field
, type
, sfield
= NULL_TREE
;
1402 splay_tree_key key
= (splay_tree_key
) var
;
1404 if ((mask
& 8) != 0)
1406 key
= (splay_tree_key
) &DECL_UID (var
);
1407 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1409 gcc_assert ((mask
& 1) == 0
1410 || !splay_tree_lookup (ctx
->field_map
, key
));
1411 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1412 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1413 gcc_assert ((mask
& 3) == 3
1414 || !is_gimple_omp_oacc (ctx
->stmt
));
1416 type
= TREE_TYPE (var
);
1419 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1420 type
= build_pointer_type (build_pointer_type (type
));
1423 type
= build_pointer_type (type
);
1424 else if ((mask
& 3) == 1 && is_reference (var
))
1425 type
= TREE_TYPE (type
);
1427 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1428 FIELD_DECL
, DECL_NAME (var
), type
);
1430 /* Remember what variable this field was created for. This does have a
1431 side effect of making dwarf2out ignore this member, so for helpful
1432 debugging we clear it later in delete_omp_context. */
1433 DECL_ABSTRACT_ORIGIN (field
) = var
;
1434 if (type
== TREE_TYPE (var
))
1436 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1437 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1438 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1441 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1443 if ((mask
& 3) == 3)
1445 insert_field_into_struct (ctx
->record_type
, field
);
1446 if (ctx
->srecord_type
)
1448 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1449 FIELD_DECL
, DECL_NAME (var
), type
);
1450 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1451 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1452 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1453 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1454 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1459 if (ctx
->srecord_type
== NULL_TREE
)
1463 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1464 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1465 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1467 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1468 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1469 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1470 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1471 splay_tree_insert (ctx
->sfield_map
,
1472 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1473 (splay_tree_value
) sfield
);
1477 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1478 : ctx
->srecord_type
, field
);
1482 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1483 if ((mask
& 2) && ctx
->sfield_map
)
1484 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1488 install_var_local (tree var
, omp_context
*ctx
)
1490 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1491 insert_decl_map (&ctx
->cb
, var
, new_var
);
1495 /* Adjust the replacement for DECL in CTX for the new context. This means
1496 copying the DECL_VALUE_EXPR, and fixing up the type. */
1499 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1501 tree new_decl
, size
;
1503 new_decl
= lookup_decl (decl
, ctx
);
1505 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1507 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1508 && DECL_HAS_VALUE_EXPR_P (decl
))
1510 tree ve
= DECL_VALUE_EXPR (decl
);
1511 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1512 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1513 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1516 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1518 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1519 if (size
== error_mark_node
)
1520 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1521 DECL_SIZE (new_decl
) = size
;
1523 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1524 if (size
== error_mark_node
)
1525 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1526 DECL_SIZE_UNIT (new_decl
) = size
;
1530 /* The callback for remap_decl. Search all containing contexts for a
1531 mapping of the variable; this avoids having to duplicate the splay
1532 tree ahead of time. We know a mapping doesn't already exist in the
1533 given context. Create new mappings to implement default semantics. */
1536 omp_copy_decl (tree var
, copy_body_data
*cb
)
1538 omp_context
*ctx
= (omp_context
*) cb
;
1541 if (TREE_CODE (var
) == LABEL_DECL
)
1543 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1544 DECL_CONTEXT (new_var
) = current_function_decl
;
1545 insert_decl_map (&ctx
->cb
, var
, new_var
);
1549 while (!is_taskreg_ctx (ctx
))
1554 new_var
= maybe_lookup_decl (var
, ctx
);
1559 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1562 return error_mark_node
;
1566 /* Debugging dumps for parallel regions. */
1567 void dump_omp_region (FILE *, struct omp_region
*, int);
1568 void debug_omp_region (struct omp_region
*);
1569 void debug_all_omp_regions (void);
1571 /* Dump the parallel region tree rooted at REGION. */
1574 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1576 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1577 gimple_code_name
[region
->type
]);
1580 dump_omp_region (file
, region
->inner
, indent
+ 4);
1584 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1585 region
->cont
->index
);
1589 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1590 region
->exit
->index
);
1592 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1595 dump_omp_region (file
, region
->next
, indent
);
1599 debug_omp_region (struct omp_region
*region
)
1601 dump_omp_region (stderr
, region
, 0);
1605 debug_all_omp_regions (void)
1607 dump_omp_region (stderr
, root_omp_region
, 0);
1611 /* Create a new parallel region starting at STMT inside region PARENT. */
1613 static struct omp_region
*
1614 new_omp_region (basic_block bb
, enum gimple_code type
,
1615 struct omp_region
*parent
)
1617 struct omp_region
*region
= XCNEW (struct omp_region
);
1619 region
->outer
= parent
;
1621 region
->type
= type
;
1625 /* This is a nested region. Add it to the list of inner
1626 regions in PARENT. */
1627 region
->next
= parent
->inner
;
1628 parent
->inner
= region
;
1632 /* This is a toplevel region. Add it to the list of toplevel
1633 regions in ROOT_OMP_REGION. */
1634 region
->next
= root_omp_region
;
1635 root_omp_region
= region
;
1641 /* Release the memory associated with the region tree rooted at REGION. */
1644 free_omp_region_1 (struct omp_region
*region
)
1646 struct omp_region
*i
, *n
;
1648 for (i
= region
->inner
; i
; i
= n
)
1651 free_omp_region_1 (i
);
1657 /* Release the memory for the entire omp region tree. */
1660 free_omp_regions (void)
1662 struct omp_region
*r
, *n
;
1663 for (r
= root_omp_region
; r
; r
= n
)
1666 free_omp_region_1 (r
);
1668 root_omp_region
= NULL
;
1672 /* Create a new context, with OUTER_CTX being the surrounding context. */
1674 static omp_context
*
1675 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1677 omp_context
*ctx
= XCNEW (omp_context
);
1679 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1680 (splay_tree_value
) ctx
);
1685 ctx
->outer
= outer_ctx
;
1686 ctx
->cb
= outer_ctx
->cb
;
1687 ctx
->cb
.block
= NULL
;
1688 ctx
->depth
= outer_ctx
->depth
+ 1;
1689 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1693 ctx
->cb
.src_fn
= current_function_decl
;
1694 ctx
->cb
.dst_fn
= current_function_decl
;
1695 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1696 gcc_checking_assert (ctx
->cb
.src_node
);
1697 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1698 ctx
->cb
.src_cfun
= cfun
;
1699 ctx
->cb
.copy_decl
= omp_copy_decl
;
1700 ctx
->cb
.eh_lp_nr
= 0;
1701 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1705 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1710 static gimple_seq
maybe_catch_exception (gimple_seq
);
1712 /* Finalize task copyfn. */
1715 finalize_task_copyfn (gomp_task
*task_stmt
)
1717 struct function
*child_cfun
;
1719 gimple_seq seq
= NULL
, new_seq
;
1722 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1723 if (child_fn
== NULL_TREE
)
1726 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1727 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1729 push_cfun (child_cfun
);
1730 bind
= gimplify_body (child_fn
, false);
1731 gimple_seq_add_stmt (&seq
, bind
);
1732 new_seq
= maybe_catch_exception (seq
);
1735 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1737 gimple_seq_add_stmt (&seq
, bind
);
1739 gimple_set_body (child_fn
, seq
);
1742 /* Inform the callgraph about the new function. */
1743 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1744 node
->parallelized_function
= 1;
1745 cgraph_node::add_new_function (child_fn
, false);
1748 /* Destroy a omp_context data structures. Called through the splay tree
1749 value delete callback. */
1752 delete_omp_context (splay_tree_value value
)
1754 omp_context
*ctx
= (omp_context
*) value
;
1756 delete ctx
->cb
.decl_map
;
1759 splay_tree_delete (ctx
->field_map
);
1760 if (ctx
->sfield_map
)
1761 splay_tree_delete (ctx
->sfield_map
);
1762 /* Reduction map is copied to nested contexts, so only delete it in the
1764 if (ctx
->reduction_map
1765 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1766 && is_gimple_omp_offloaded (ctx
->stmt
)
1767 && is_gimple_omp_oacc (ctx
->stmt
))
1768 splay_tree_delete (ctx
->reduction_map
);
1770 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1771 it produces corrupt debug information. */
1772 if (ctx
->record_type
)
1775 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1776 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1778 if (ctx
->srecord_type
)
1781 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1782 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1785 if (is_task_ctx (ctx
))
1786 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1791 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1795 fixup_child_record_type (omp_context
*ctx
)
1797 tree f
, type
= ctx
->record_type
;
1799 /* ??? It isn't sufficient to just call remap_type here, because
1800 variably_modified_type_p doesn't work the way we expect for
1801 record types. Testing each field for whether it needs remapping
1802 and creating a new record by hand works, however. */
1803 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1804 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1808 tree name
, new_fields
= NULL
;
1810 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1811 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1812 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1813 TYPE_DECL
, name
, type
);
1814 TYPE_NAME (type
) = name
;
1816 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1818 tree new_f
= copy_node (f
);
1819 DECL_CONTEXT (new_f
) = type
;
1820 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1821 DECL_CHAIN (new_f
) = new_fields
;
1822 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1823 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1825 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1829 /* Arrange to be able to look up the receiver field
1830 given the sender field. */
1831 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1832 (splay_tree_value
) new_f
);
1834 TYPE_FIELDS (type
) = nreverse (new_fields
);
1838 /* In a target region we never modify any of the pointers in *.omp_data_i,
1839 so attempt to help the optimizers. */
1840 if (is_gimple_omp_offloaded (ctx
->stmt
))
1841 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1843 TREE_TYPE (ctx
->receiver_decl
)
1844 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1847 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1848 specified by CLAUSES. */
1851 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1854 bool scan_array_reductions
= false;
1856 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1860 switch (OMP_CLAUSE_CODE (c
))
1862 case OMP_CLAUSE_PRIVATE
:
1863 decl
= OMP_CLAUSE_DECL (c
);
1864 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1866 else if (!is_variable_sized (decl
))
1867 install_var_local (decl
, ctx
);
1870 case OMP_CLAUSE_SHARED
:
1871 decl
= OMP_CLAUSE_DECL (c
);
1872 /* Ignore shared directives in teams construct. */
1873 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1875 /* Global variables don't need to be copied,
1876 the receiver side will use them directly. */
1877 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1878 if (is_global_var (odecl
))
1880 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1883 gcc_assert (is_taskreg_ctx (ctx
));
1884 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1885 || !is_variable_sized (decl
));
1886 /* Global variables don't need to be copied,
1887 the receiver side will use them directly. */
1888 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1890 by_ref
= use_pointer_for_field (decl
, ctx
);
1891 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1893 if (! TREE_READONLY (decl
)
1894 || TREE_ADDRESSABLE (decl
)
1896 || is_reference (decl
))
1898 install_var_field (decl
, by_ref
, 3, ctx
);
1899 install_var_local (decl
, ctx
);
1902 /* We don't need to copy const scalar vars back. */
1903 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1906 case OMP_CLAUSE_REDUCTION
:
1907 decl
= OMP_CLAUSE_DECL (c
);
1908 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1909 && TREE_CODE (decl
) == MEM_REF
)
1911 tree t
= TREE_OPERAND (decl
, 0);
1912 if (TREE_CODE (t
) == INDIRECT_REF
1913 || TREE_CODE (t
) == ADDR_EXPR
)
1914 t
= TREE_OPERAND (t
, 0);
1915 install_var_local (t
, ctx
);
1916 if (is_taskreg_ctx (ctx
)
1917 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1918 && !is_variable_sized (t
))
1920 by_ref
= use_pointer_for_field (t
, ctx
);
1921 install_var_field (t
, by_ref
, 3, ctx
);
1927 case OMP_CLAUSE_LASTPRIVATE
:
1928 /* Let the corresponding firstprivate clause create
1930 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1934 case OMP_CLAUSE_FIRSTPRIVATE
:
1935 if (is_gimple_omp_oacc (ctx
->stmt
))
1937 sorry ("clause not supported yet");
1941 case OMP_CLAUSE_LINEAR
:
1942 decl
= OMP_CLAUSE_DECL (c
);
1944 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1945 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1946 && is_gimple_omp_offloaded (ctx
->stmt
))
1948 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1949 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1950 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1951 install_var_field (decl
, true, 3, ctx
);
1953 install_var_field (decl
, false, 3, ctx
);
1955 if (is_variable_sized (decl
))
1957 if (is_task_ctx (ctx
))
1958 install_var_field (decl
, false, 1, ctx
);
1961 else if (is_taskreg_ctx (ctx
))
1964 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1965 by_ref
= use_pointer_for_field (decl
, NULL
);
1967 if (is_task_ctx (ctx
)
1968 && (global
|| by_ref
|| is_reference (decl
)))
1970 install_var_field (decl
, false, 1, ctx
);
1972 install_var_field (decl
, by_ref
, 2, ctx
);
1975 install_var_field (decl
, by_ref
, 3, ctx
);
1977 install_var_local (decl
, ctx
);
1978 if (is_gimple_omp_oacc (ctx
->stmt
)
1979 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1981 /* Create a decl for the reduction array. */
1982 tree var
= OMP_CLAUSE_DECL (c
);
1983 tree type
= get_base_type (var
);
1984 tree ptype
= build_pointer_type (type
);
1985 tree array
= create_tmp_var (ptype
,
1986 oacc_get_reduction_array_id (var
));
1987 omp_context
*octx
= (ctx
->field_map
? ctx
: ctx
->outer
);
1988 install_var_field (array
, true, 3, octx
);
1989 install_var_local (array
, octx
);
1991 /* Insert it into the current context. */
1992 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1993 oacc_get_reduction_array_id (var
),
1994 (splay_tree_value
) array
);
1995 splay_tree_insert (ctx
->reduction_map
,
1996 (splay_tree_key
) array
,
1997 (splay_tree_value
) array
);
2001 case OMP_CLAUSE_USE_DEVICE_PTR
:
2002 decl
= OMP_CLAUSE_DECL (c
);
2003 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2004 install_var_field (decl
, true, 3, ctx
);
2006 install_var_field (decl
, false, 3, ctx
);
2007 if (DECL_SIZE (decl
)
2008 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2010 tree decl2
= DECL_VALUE_EXPR (decl
);
2011 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2012 decl2
= TREE_OPERAND (decl2
, 0);
2013 gcc_assert (DECL_P (decl2
));
2014 install_var_local (decl2
, ctx
);
2016 install_var_local (decl
, ctx
);
2019 case OMP_CLAUSE_IS_DEVICE_PTR
:
2020 decl
= OMP_CLAUSE_DECL (c
);
2023 case OMP_CLAUSE__LOOPTEMP_
:
2024 gcc_assert (is_taskreg_ctx (ctx
));
2025 decl
= OMP_CLAUSE_DECL (c
);
2026 install_var_field (decl
, false, 3, ctx
);
2027 install_var_local (decl
, ctx
);
2030 case OMP_CLAUSE_COPYPRIVATE
:
2031 case OMP_CLAUSE_COPYIN
:
2032 decl
= OMP_CLAUSE_DECL (c
);
2033 by_ref
= use_pointer_for_field (decl
, NULL
);
2034 install_var_field (decl
, by_ref
, 3, ctx
);
2037 case OMP_CLAUSE_DEFAULT
:
2038 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2041 case OMP_CLAUSE_FINAL
:
2043 case OMP_CLAUSE_NUM_THREADS
:
2044 case OMP_CLAUSE_NUM_TEAMS
:
2045 case OMP_CLAUSE_THREAD_LIMIT
:
2046 case OMP_CLAUSE_DEVICE
:
2047 case OMP_CLAUSE_SCHEDULE
:
2048 case OMP_CLAUSE_DIST_SCHEDULE
:
2049 case OMP_CLAUSE_DEPEND
:
2050 case OMP_CLAUSE_PRIORITY
:
2051 case OMP_CLAUSE_GRAINSIZE
:
2052 case OMP_CLAUSE_NUM_TASKS
:
2053 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2054 case OMP_CLAUSE_NUM_GANGS
:
2055 case OMP_CLAUSE_NUM_WORKERS
:
2056 case OMP_CLAUSE_VECTOR_LENGTH
:
2058 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2062 case OMP_CLAUSE_FROM
:
2063 case OMP_CLAUSE_MAP
:
2065 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2066 decl
= OMP_CLAUSE_DECL (c
);
2067 /* Global variables with "omp declare target" attribute
2068 don't need to be copied, the receiver side will use them
2070 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2072 && (OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2073 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2074 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2075 && varpool_node::get_create (decl
)->offloadable
)
2077 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2078 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2080 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2081 not offloaded; there is nothing to map for those. */
2082 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2083 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2084 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2087 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2088 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2090 if (TREE_CODE (decl
) == COMPONENT_REF
2091 || (TREE_CODE (decl
) == INDIRECT_REF
2092 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2093 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2094 == REFERENCE_TYPE
)))
2096 if (DECL_SIZE (decl
)
2097 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2099 tree decl2
= DECL_VALUE_EXPR (decl
);
2100 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2101 decl2
= TREE_OPERAND (decl2
, 0);
2102 gcc_assert (DECL_P (decl2
));
2103 install_var_local (decl2
, ctx
);
2105 install_var_local (decl
, ctx
);
2110 if (DECL_SIZE (decl
)
2111 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2113 tree decl2
= DECL_VALUE_EXPR (decl
);
2114 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2115 decl2
= TREE_OPERAND (decl2
, 0);
2116 gcc_assert (DECL_P (decl2
));
2117 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2118 && OMP_CLAUSE_MAP_PRIVATE (c
))
2119 install_var_field (decl2
, true, 11, ctx
);
2121 install_var_field (decl2
, true, 3, ctx
);
2122 install_var_local (decl2
, ctx
);
2123 install_var_local (decl
, ctx
);
2127 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2128 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2129 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2130 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2131 install_var_field (decl
, true, 7, ctx
);
2132 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2133 && OMP_CLAUSE_MAP_PRIVATE (c
))
2134 install_var_field (decl
, true, 11, ctx
);
2136 install_var_field (decl
, true, 3, ctx
);
2137 if (is_gimple_omp_offloaded (ctx
->stmt
))
2138 install_var_local (decl
, ctx
);
2143 tree base
= get_base_address (decl
);
2144 tree nc
= OMP_CLAUSE_CHAIN (c
);
2147 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2148 && OMP_CLAUSE_DECL (nc
) == base
2149 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2150 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2152 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2153 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2159 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2160 decl
= OMP_CLAUSE_DECL (c
);
2162 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2163 (splay_tree_key
) decl
));
2165 = build_decl (OMP_CLAUSE_LOCATION (c
),
2166 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2167 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2168 insert_field_into_struct (ctx
->record_type
, field
);
2169 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2170 (splay_tree_value
) field
);
2175 case OMP_CLAUSE_NOWAIT
:
2176 case OMP_CLAUSE_ORDERED
:
2177 case OMP_CLAUSE_COLLAPSE
:
2178 case OMP_CLAUSE_UNTIED
:
2179 case OMP_CLAUSE_MERGEABLE
:
2180 case OMP_CLAUSE_PROC_BIND
:
2181 case OMP_CLAUSE_SAFELEN
:
2182 case OMP_CLAUSE_SIMDLEN
:
2183 case OMP_CLAUSE_THREADS
:
2184 case OMP_CLAUSE_SIMD
:
2185 case OMP_CLAUSE_NOGROUP
:
2186 case OMP_CLAUSE_DEFAULTMAP
:
2187 case OMP_CLAUSE_ASYNC
:
2188 case OMP_CLAUSE_WAIT
:
2189 case OMP_CLAUSE_GANG
:
2190 case OMP_CLAUSE_WORKER
:
2191 case OMP_CLAUSE_VECTOR
:
2194 case OMP_CLAUSE_ALIGNED
:
2195 decl
= OMP_CLAUSE_DECL (c
);
2196 if (is_global_var (decl
)
2197 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2198 install_var_local (decl
, ctx
);
2201 case OMP_CLAUSE_DEVICE_RESIDENT
:
2202 case OMP_CLAUSE_USE_DEVICE
:
2203 case OMP_CLAUSE__CACHE_
:
2204 case OMP_CLAUSE_INDEPENDENT
:
2205 case OMP_CLAUSE_AUTO
:
2206 case OMP_CLAUSE_SEQ
:
2207 sorry ("Clause not supported yet");
2215 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2217 switch (OMP_CLAUSE_CODE (c
))
2219 case OMP_CLAUSE_LASTPRIVATE
:
2220 /* Let the corresponding firstprivate clause create
2222 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2223 scan_array_reductions
= true;
2224 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2228 case OMP_CLAUSE_FIRSTPRIVATE
:
2229 if (is_gimple_omp_oacc (ctx
->stmt
))
2231 sorry ("clause not supported yet");
2235 case OMP_CLAUSE_PRIVATE
:
2236 case OMP_CLAUSE_LINEAR
:
2237 case OMP_CLAUSE_IS_DEVICE_PTR
:
2238 decl
= OMP_CLAUSE_DECL (c
);
2239 if (is_variable_sized (decl
))
2241 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2242 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2243 && is_gimple_omp_offloaded (ctx
->stmt
))
2245 tree decl2
= DECL_VALUE_EXPR (decl
);
2246 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2247 decl2
= TREE_OPERAND (decl2
, 0);
2248 gcc_assert (DECL_P (decl2
));
2249 install_var_local (decl2
, ctx
);
2250 fixup_remapped_decl (decl2
, ctx
, false);
2252 install_var_local (decl
, ctx
);
2254 fixup_remapped_decl (decl
, ctx
,
2255 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2256 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2257 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2258 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2259 scan_array_reductions
= true;
2262 case OMP_CLAUSE_REDUCTION
:
2263 decl
= OMP_CLAUSE_DECL (c
);
2264 if (TREE_CODE (decl
) != MEM_REF
)
2266 if (is_variable_sized (decl
))
2267 install_var_local (decl
, ctx
);
2268 fixup_remapped_decl (decl
, ctx
, false);
2270 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2271 scan_array_reductions
= true;
2274 case OMP_CLAUSE_SHARED
:
2275 /* Ignore shared directives in teams construct. */
2276 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2278 decl
= OMP_CLAUSE_DECL (c
);
2279 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2281 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2283 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2286 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2287 install_var_field (decl
, by_ref
, 11, ctx
);
2290 fixup_remapped_decl (decl
, ctx
, false);
2293 case OMP_CLAUSE_MAP
:
2294 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2296 decl
= OMP_CLAUSE_DECL (c
);
2298 && (OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2299 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2300 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2301 && varpool_node::get_create (decl
)->offloadable
)
2305 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2306 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2307 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2308 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2310 tree new_decl
= lookup_decl (decl
, ctx
);
2311 TREE_TYPE (new_decl
)
2312 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2314 else if (DECL_SIZE (decl
)
2315 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2317 tree decl2
= DECL_VALUE_EXPR (decl
);
2318 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2319 decl2
= TREE_OPERAND (decl2
, 0);
2320 gcc_assert (DECL_P (decl2
));
2321 fixup_remapped_decl (decl2
, ctx
, false);
2322 fixup_remapped_decl (decl
, ctx
, true);
2325 fixup_remapped_decl (decl
, ctx
, false);
2329 case OMP_CLAUSE_COPYPRIVATE
:
2330 case OMP_CLAUSE_COPYIN
:
2331 case OMP_CLAUSE_DEFAULT
:
2333 case OMP_CLAUSE_NUM_THREADS
:
2334 case OMP_CLAUSE_NUM_TEAMS
:
2335 case OMP_CLAUSE_THREAD_LIMIT
:
2336 case OMP_CLAUSE_DEVICE
:
2337 case OMP_CLAUSE_SCHEDULE
:
2338 case OMP_CLAUSE_DIST_SCHEDULE
:
2339 case OMP_CLAUSE_NOWAIT
:
2340 case OMP_CLAUSE_ORDERED
:
2341 case OMP_CLAUSE_COLLAPSE
:
2342 case OMP_CLAUSE_UNTIED
:
2343 case OMP_CLAUSE_FINAL
:
2344 case OMP_CLAUSE_MERGEABLE
:
2345 case OMP_CLAUSE_PROC_BIND
:
2346 case OMP_CLAUSE_SAFELEN
:
2347 case OMP_CLAUSE_SIMDLEN
:
2348 case OMP_CLAUSE_ALIGNED
:
2349 case OMP_CLAUSE_DEPEND
:
2350 case OMP_CLAUSE__LOOPTEMP_
:
2352 case OMP_CLAUSE_FROM
:
2353 case OMP_CLAUSE_PRIORITY
:
2354 case OMP_CLAUSE_GRAINSIZE
:
2355 case OMP_CLAUSE_NUM_TASKS
:
2356 case OMP_CLAUSE_THREADS
:
2357 case OMP_CLAUSE_SIMD
:
2358 case OMP_CLAUSE_NOGROUP
:
2359 case OMP_CLAUSE_DEFAULTMAP
:
2360 case OMP_CLAUSE_USE_DEVICE_PTR
:
2361 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2362 case OMP_CLAUSE_ASYNC
:
2363 case OMP_CLAUSE_WAIT
:
2364 case OMP_CLAUSE_NUM_GANGS
:
2365 case OMP_CLAUSE_NUM_WORKERS
:
2366 case OMP_CLAUSE_VECTOR_LENGTH
:
2367 case OMP_CLAUSE_GANG
:
2368 case OMP_CLAUSE_WORKER
:
2369 case OMP_CLAUSE_VECTOR
:
2372 case OMP_CLAUSE_DEVICE_RESIDENT
:
2373 case OMP_CLAUSE_USE_DEVICE
:
2374 case OMP_CLAUSE__CACHE_
:
2375 case OMP_CLAUSE_INDEPENDENT
:
2376 case OMP_CLAUSE_AUTO
:
2377 case OMP_CLAUSE_SEQ
:
2378 sorry ("Clause not supported yet");
2386 gcc_checking_assert (!scan_array_reductions
2387 || !is_gimple_omp_oacc (ctx
->stmt
));
2388 if (scan_array_reductions
)
2389 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2390 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2391 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2393 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2394 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2396 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2397 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2398 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2399 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2400 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2401 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2404 /* Create a new name for omp child function. Returns an identifier. If
2405 IS_CILK_FOR is true then the suffix for the child function is
2409 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2412 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2413 return clone_function_name (current_function_decl
,
2414 task_copy
? "_omp_cpyfn" : "_omp_fn");
2417 /* Returns the type of the induction variable for the child function for
2418 _Cilk_for and the types for _high and _low variables based on TYPE. */
2421 cilk_for_check_loop_diff_type (tree type
)
2423 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2425 if (TYPE_UNSIGNED (type
))
2426 return uint32_type_node
;
2428 return integer_type_node
;
2432 if (TYPE_UNSIGNED (type
))
2433 return uint64_type_node
;
2435 return long_long_integer_type_node
;
2439 /* Build a decl for the omp child function. It'll not contain a body
2440 yet, just the bare decl. */
2443 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2445 tree decl
, type
, name
, t
;
2448 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2449 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2450 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2451 tree cilk_var_type
= NULL_TREE
;
2453 name
= create_omp_child_function_name (task_copy
,
2454 cilk_for_count
!= NULL_TREE
);
2456 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2457 ptr_type_node
, NULL_TREE
);
2458 else if (cilk_for_count
)
2460 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2461 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2462 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2463 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2466 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2468 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2470 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2473 ctx
->cb
.dst_fn
= decl
;
2475 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2477 TREE_STATIC (decl
) = 1;
2478 TREE_USED (decl
) = 1;
2479 DECL_ARTIFICIAL (decl
) = 1;
2480 DECL_IGNORED_P (decl
) = 0;
2481 TREE_PUBLIC (decl
) = 0;
2482 DECL_UNINLINABLE (decl
) = 1;
2483 DECL_EXTERNAL (decl
) = 0;
2484 DECL_CONTEXT (decl
) = NULL_TREE
;
2485 DECL_INITIAL (decl
) = make_node (BLOCK
);
2486 if (cgraph_node::get (current_function_decl
)->offloadable
)
2487 cgraph_node::get_create (decl
)->offloadable
= 1;
2491 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2492 if (is_gimple_omp_offloaded (octx
->stmt
))
2494 cgraph_node::get_create (decl
)->offloadable
= 1;
2495 #ifdef ENABLE_OFFLOADING
2496 g
->have_offload
= true;
2502 if (cgraph_node::get_create (decl
)->offloadable
2503 && !lookup_attribute ("omp declare target",
2504 DECL_ATTRIBUTES (current_function_decl
)))
2505 DECL_ATTRIBUTES (decl
)
2506 = tree_cons (get_identifier ("omp target entrypoint"),
2507 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2509 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2510 RESULT_DECL
, NULL_TREE
, void_type_node
);
2511 DECL_ARTIFICIAL (t
) = 1;
2512 DECL_IGNORED_P (t
) = 1;
2513 DECL_CONTEXT (t
) = decl
;
2514 DECL_RESULT (decl
) = t
;
2516 /* _Cilk_for's child function requires two extra parameters called
2517 __low and __high that are set the by Cilk runtime when it calls this
2521 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2522 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2523 DECL_ARTIFICIAL (t
) = 1;
2524 DECL_NAMELESS (t
) = 1;
2525 DECL_ARG_TYPE (t
) = ptr_type_node
;
2526 DECL_CONTEXT (t
) = current_function_decl
;
2528 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2529 DECL_ARGUMENTS (decl
) = t
;
2531 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2532 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2533 DECL_ARTIFICIAL (t
) = 1;
2534 DECL_NAMELESS (t
) = 1;
2535 DECL_ARG_TYPE (t
) = ptr_type_node
;
2536 DECL_CONTEXT (t
) = current_function_decl
;
2538 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2539 DECL_ARGUMENTS (decl
) = t
;
2542 tree data_name
= get_identifier (".omp_data_i");
2543 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2545 DECL_ARTIFICIAL (t
) = 1;
2546 DECL_NAMELESS (t
) = 1;
2547 DECL_ARG_TYPE (t
) = ptr_type_node
;
2548 DECL_CONTEXT (t
) = current_function_decl
;
2550 TREE_READONLY (t
) = 1;
2552 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2553 DECL_ARGUMENTS (decl
) = t
;
2555 ctx
->receiver_decl
= t
;
2558 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2559 PARM_DECL
, get_identifier (".omp_data_o"),
2561 DECL_ARTIFICIAL (t
) = 1;
2562 DECL_NAMELESS (t
) = 1;
2563 DECL_ARG_TYPE (t
) = ptr_type_node
;
2564 DECL_CONTEXT (t
) = current_function_decl
;
2566 TREE_ADDRESSABLE (t
) = 1;
2567 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2568 DECL_ARGUMENTS (decl
) = t
;
2571 /* Allocate memory for the function structure. The call to
2572 allocate_struct_function clobbers CFUN, so we need to restore
2574 push_struct_function (decl
);
2575 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2579 /* Callback for walk_gimple_seq. Check if combined parallel
2580 contains gimple_omp_for_combined_into_p OMP_FOR. */
2583 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2584 bool *handled_ops_p
,
2585 struct walk_stmt_info
*wi
)
2587 gimple
*stmt
= gsi_stmt (*gsi_p
);
2589 *handled_ops_p
= true;
2590 switch (gimple_code (stmt
))
2594 case GIMPLE_OMP_FOR
:
2595 if (gimple_omp_for_combined_into_p (stmt
)
2596 && gimple_omp_for_kind (stmt
)
2597 == *(const enum gf_mask
*) (wi
->info
))
2600 return integer_zero_node
;
2609 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2612 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2613 omp_context
*outer_ctx
)
2615 struct walk_stmt_info wi
;
2617 memset (&wi
, 0, sizeof (wi
));
2619 wi
.info
= (void *) &msk
;
2620 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2621 if (wi
.info
!= (void *) &msk
)
2623 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2624 struct omp_for_data fd
;
2625 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2626 /* We need two temporaries with fd.loop.v type (istart/iend)
2627 and then (fd.collapse - 1) temporaries with the same
2628 type for count2 ... countN-1 vars if not constant. */
2629 size_t count
= 2, i
;
2630 tree type
= fd
.iter_type
;
2632 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2634 count
+= fd
.collapse
- 1;
2635 /* For taskloop, if there are lastprivate clauses on the inner
2636 GIMPLE_OMP_FOR, add one more temporaries for the total number
2637 of iterations (product of count1 ... countN-1). */
2638 if (msk
== GF_OMP_FOR_KIND_TASKLOOP
2639 && find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2640 OMP_CLAUSE_LASTPRIVATE
))
2643 for (i
= 0; i
< count
; i
++)
2645 tree temp
= create_tmp_var (type
);
2646 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2647 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2648 OMP_CLAUSE_DECL (c
) = temp
;
2649 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2650 gimple_omp_taskreg_set_clauses (stmt
, c
);
2655 /* Scan an OpenMP parallel directive. */
2658 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2662 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2664 /* Ignore parallel directives with empty bodies, unless there
2665 are copyin clauses. */
2667 && empty_body_p (gimple_omp_body (stmt
))
2668 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2669 OMP_CLAUSE_COPYIN
) == NULL
)
2671 gsi_replace (gsi
, gimple_build_nop (), false);
2675 if (gimple_omp_parallel_combined_p (stmt
))
2676 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2678 ctx
= new_omp_context (stmt
, outer_ctx
);
2679 taskreg_contexts
.safe_push (ctx
);
2680 if (taskreg_nesting_level
> 1)
2681 ctx
->is_nested
= true;
2682 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2683 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2684 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2685 name
= create_tmp_var_name (".omp_data_s");
2686 name
= build_decl (gimple_location (stmt
),
2687 TYPE_DECL
, name
, ctx
->record_type
);
2688 DECL_ARTIFICIAL (name
) = 1;
2689 DECL_NAMELESS (name
) = 1;
2690 TYPE_NAME (ctx
->record_type
) = name
;
2691 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2692 create_omp_child_function (ctx
, false);
2693 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2695 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2696 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2698 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2699 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2702 /* Scan an OpenMP task directive. */
2705 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2709 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2711 /* Ignore task directives with empty bodies. */
2713 && empty_body_p (gimple_omp_body (stmt
)))
2715 gsi_replace (gsi
, gimple_build_nop (), false);
2719 if (gimple_omp_task_taskloop_p (stmt
))
2720 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2722 ctx
= new_omp_context (stmt
, outer_ctx
);
2723 taskreg_contexts
.safe_push (ctx
);
2724 if (taskreg_nesting_level
> 1)
2725 ctx
->is_nested
= true;
2726 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2727 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2728 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2729 name
= create_tmp_var_name (".omp_data_s");
2730 name
= build_decl (gimple_location (stmt
),
2731 TYPE_DECL
, name
, ctx
->record_type
);
2732 DECL_ARTIFICIAL (name
) = 1;
2733 DECL_NAMELESS (name
) = 1;
2734 TYPE_NAME (ctx
->record_type
) = name
;
2735 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2736 create_omp_child_function (ctx
, false);
2737 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2739 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2741 if (ctx
->srecord_type
)
2743 name
= create_tmp_var_name (".omp_data_a");
2744 name
= build_decl (gimple_location (stmt
),
2745 TYPE_DECL
, name
, ctx
->srecord_type
);
2746 DECL_ARTIFICIAL (name
) = 1;
2747 DECL_NAMELESS (name
) = 1;
2748 TYPE_NAME (ctx
->srecord_type
) = name
;
2749 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2750 create_omp_child_function (ctx
, true);
2753 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2755 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2757 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2758 t
= build_int_cst (long_integer_type_node
, 0);
2759 gimple_omp_task_set_arg_size (stmt
, t
);
2760 t
= build_int_cst (long_integer_type_node
, 1);
2761 gimple_omp_task_set_arg_align (stmt
, t
);
2766 /* If any decls have been made addressable during scan_omp,
2767 adjust their fields if needed, and layout record types
2768 of parallel/task constructs. */
2771 finish_taskreg_scan (omp_context
*ctx
)
2773 if (ctx
->record_type
== NULL_TREE
)
2776 /* If any task_shared_vars were needed, verify all
2777 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2778 statements if use_pointer_for_field hasn't changed
2779 because of that. If it did, update field types now. */
2780 if (task_shared_vars
)
2784 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2785 c
; c
= OMP_CLAUSE_CHAIN (c
))
2786 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2787 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2789 tree decl
= OMP_CLAUSE_DECL (c
);
2791 /* Global variables don't need to be copied,
2792 the receiver side will use them directly. */
2793 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2795 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2796 || !use_pointer_for_field (decl
, ctx
))
2798 tree field
= lookup_field (decl
, ctx
);
2799 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2800 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2802 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2803 TREE_THIS_VOLATILE (field
) = 0;
2804 DECL_USER_ALIGN (field
) = 0;
2805 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2806 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2807 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2808 if (ctx
->srecord_type
)
2810 tree sfield
= lookup_sfield (decl
, ctx
);
2811 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2812 TREE_THIS_VOLATILE (sfield
) = 0;
2813 DECL_USER_ALIGN (sfield
) = 0;
2814 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2815 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2816 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2821 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2823 layout_type (ctx
->record_type
);
2824 fixup_child_record_type (ctx
);
2828 location_t loc
= gimple_location (ctx
->stmt
);
2829 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2830 /* Move VLA fields to the end. */
2831 p
= &TYPE_FIELDS (ctx
->record_type
);
2833 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2834 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2837 *p
= TREE_CHAIN (*p
);
2838 TREE_CHAIN (*q
) = NULL_TREE
;
2839 q
= &TREE_CHAIN (*q
);
2842 p
= &DECL_CHAIN (*p
);
2844 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2846 /* Move fields corresponding to first and second _looptemp_
2847 clause first. There are filled by GOMP_taskloop
2848 and thus need to be in specific positions. */
2849 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2850 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2851 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2852 OMP_CLAUSE__LOOPTEMP_
);
2853 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2854 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2855 p
= &TYPE_FIELDS (ctx
->record_type
);
2857 if (*p
== f1
|| *p
== f2
)
2858 *p
= DECL_CHAIN (*p
);
2860 p
= &DECL_CHAIN (*p
);
2861 DECL_CHAIN (f1
) = f2
;
2862 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2863 TYPE_FIELDS (ctx
->record_type
) = f1
;
2864 if (ctx
->srecord_type
)
2866 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2867 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2868 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2870 if (*p
== f1
|| *p
== f2
)
2871 *p
= DECL_CHAIN (*p
);
2873 p
= &DECL_CHAIN (*p
);
2874 DECL_CHAIN (f1
) = f2
;
2875 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2876 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2879 layout_type (ctx
->record_type
);
2880 fixup_child_record_type (ctx
);
2881 if (ctx
->srecord_type
)
2882 layout_type (ctx
->srecord_type
);
2883 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2884 TYPE_SIZE_UNIT (ctx
->record_type
));
2885 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2886 t
= build_int_cst (long_integer_type_node
,
2887 TYPE_ALIGN_UNIT (ctx
->record_type
));
2888 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2893 static omp_context
*
2894 enclosing_target_ctx (omp_context
*ctx
)
2897 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2899 gcc_assert (ctx
!= NULL
);
2904 oacc_loop_or_target_p (gimple
*stmt
)
2906 enum gimple_code outer_type
= gimple_code (stmt
);
2907 return ((outer_type
== GIMPLE_OMP_TARGET
2908 && ((gimple_omp_target_kind (stmt
)
2909 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2910 || (gimple_omp_target_kind (stmt
)
2911 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2912 || (outer_type
== GIMPLE_OMP_FOR
2913 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2916 /* Scan a GIMPLE_OMP_FOR. */
2919 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2921 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2924 tree clauses
= gimple_omp_for_clauses (stmt
);
2927 outer_type
= gimple_code (outer_ctx
->stmt
);
2929 ctx
= new_omp_context (stmt
, outer_ctx
);
2931 if (is_gimple_omp_oacc (stmt
))
2933 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2934 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2935 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2938 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2940 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2942 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2946 ctx
->gwv_this
|= val
;
2949 /* Skip; not nested inside a region. */
2952 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2954 /* Skip; not nested inside an OpenACC region. */
2957 if (outer_type
== GIMPLE_OMP_FOR
)
2958 outer_ctx
->gwv_below
|= val
;
2959 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2961 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2962 if (gimple_omp_target_kind (enclosing
->stmt
)
2963 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2964 error_at (gimple_location (stmt
),
2965 "no arguments allowed to gang, worker and vector clauses inside parallel");
2970 scan_sharing_clauses (clauses
, ctx
);
2972 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2973 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2975 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2976 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2977 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2978 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2980 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2982 if (is_gimple_omp_oacc (stmt
))
2984 if (ctx
->gwv_this
& ctx
->gwv_below
)
2985 error_at (gimple_location (stmt
),
2986 "gang, worker and vector may occur only once in a loop nest");
2987 else if (ctx
->gwv_below
!= 0
2988 && ctx
->gwv_this
> ctx
->gwv_below
)
2989 error_at (gimple_location (stmt
),
2990 "gang, worker and vector must occur in this order in a loop nest");
2991 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2992 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2996 /* Scan an OpenMP sections directive. */
2999 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3003 ctx
= new_omp_context (stmt
, outer_ctx
);
3004 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3005 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3008 /* Scan an OpenMP single directive. */
3011 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3016 ctx
= new_omp_context (stmt
, outer_ctx
);
3017 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3018 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3019 name
= create_tmp_var_name (".omp_copy_s");
3020 name
= build_decl (gimple_location (stmt
),
3021 TYPE_DECL
, name
, ctx
->record_type
);
3022 TYPE_NAME (ctx
->record_type
) = name
;
3024 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3025 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3027 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3028 ctx
->record_type
= NULL
;
3030 layout_type (ctx
->record_type
);
3033 /* Scan a GIMPLE_OMP_TARGET. */
3036 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3040 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3041 tree clauses
= gimple_omp_target_clauses (stmt
);
3043 ctx
= new_omp_context (stmt
, outer_ctx
);
3044 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3045 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3046 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3047 name
= create_tmp_var_name (".omp_data_t");
3048 name
= build_decl (gimple_location (stmt
),
3049 TYPE_DECL
, name
, ctx
->record_type
);
3050 DECL_ARTIFICIAL (name
) = 1;
3051 DECL_NAMELESS (name
) = 1;
3052 TYPE_NAME (ctx
->record_type
) = name
;
3053 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3056 if (is_gimple_omp_oacc (stmt
))
3057 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
3060 create_omp_child_function (ctx
, false);
3061 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3064 if (is_gimple_omp_oacc (stmt
))
3066 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3068 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
3069 ctx
->gwv_this
|= MASK_GANG
;
3070 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
3071 ctx
->gwv_this
|= MASK_WORKER
;
3072 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
3073 ctx
->gwv_this
|= MASK_VECTOR
;
3077 scan_sharing_clauses (clauses
, ctx
);
3078 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3080 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3081 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3084 TYPE_FIELDS (ctx
->record_type
)
3085 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3086 #ifdef ENABLE_CHECKING
3088 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3089 for (field
= TYPE_FIELDS (ctx
->record_type
);
3091 field
= DECL_CHAIN (field
))
3092 gcc_assert (DECL_ALIGN (field
) == align
);
3094 layout_type (ctx
->record_type
);
3096 fixup_child_record_type (ctx
);
3100 /* Scan an OpenMP teams directive. */
3103 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3105 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3106 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3107 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3110 /* Check nesting restrictions. */
3112 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3116 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3117 inside an OpenACC CTX. */
3118 if (!(is_gimple_omp (stmt
)
3119 && is_gimple_omp_oacc (stmt
)))
3121 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
3122 if (is_gimple_omp (ctx_
->stmt
)
3123 && is_gimple_omp_oacc (ctx_
->stmt
))
3125 error_at (gimple_location (stmt
),
3126 "non-OpenACC construct inside of OpenACC region");
3133 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3134 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3137 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3139 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3140 if (c
&& OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
)
3143 error_at (gimple_location (stmt
),
3144 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3145 " may not be nested inside simd region");
3148 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3150 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3151 || (gimple_omp_for_kind (stmt
)
3152 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3153 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3155 error_at (gimple_location (stmt
),
3156 "only distribute or parallel constructs are allowed to "
3157 "be closely nested inside teams construct");
3162 switch (gimple_code (stmt
))
3164 case GIMPLE_OMP_FOR
:
3165 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3167 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3169 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3171 error_at (gimple_location (stmt
),
3172 "distribute construct must be closely nested inside "
3178 /* We split taskloop into task and nested taskloop in it. */
3179 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3181 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3186 switch (gimple_code (ctx
->stmt
))
3188 case GIMPLE_OMP_FOR
:
3189 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3190 == GF_OMP_FOR_KIND_OACC_LOOP
);
3193 case GIMPLE_OMP_TARGET
:
3194 switch (gimple_omp_target_kind (ctx
->stmt
))
3196 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3197 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3208 else if (get_oacc_fn_attrib (current_function_decl
))
3212 error_at (gimple_location (stmt
),
3213 "OpenACC loop directive must be associated with"
3214 " an OpenACC compute region");
3220 if (is_gimple_call (stmt
)
3221 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3222 == BUILT_IN_GOMP_CANCEL
3223 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3224 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3226 const char *bad
= NULL
;
3227 const char *kind
= NULL
;
3230 error_at (gimple_location (stmt
), "orphaned %qs construct",
3231 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3232 == BUILT_IN_GOMP_CANCEL
3233 ? "#pragma omp cancel"
3234 : "#pragma omp cancellation point");
3237 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3238 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3242 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3243 bad
= "#pragma omp parallel";
3244 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3245 == BUILT_IN_GOMP_CANCEL
3246 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3247 ctx
->cancellable
= true;
3251 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3252 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3253 bad
= "#pragma omp for";
3254 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3255 == BUILT_IN_GOMP_CANCEL
3256 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3258 ctx
->cancellable
= true;
3259 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3261 warning_at (gimple_location (stmt
), 0,
3262 "%<#pragma omp cancel for%> inside "
3263 "%<nowait%> for construct");
3264 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3265 OMP_CLAUSE_ORDERED
))
3266 warning_at (gimple_location (stmt
), 0,
3267 "%<#pragma omp cancel for%> inside "
3268 "%<ordered%> for construct");
3273 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3274 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3275 bad
= "#pragma omp sections";
3276 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3277 == BUILT_IN_GOMP_CANCEL
3278 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3280 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3282 ctx
->cancellable
= true;
3283 if (find_omp_clause (gimple_omp_sections_clauses
3286 warning_at (gimple_location (stmt
), 0,
3287 "%<#pragma omp cancel sections%> inside "
3288 "%<nowait%> sections construct");
3292 gcc_assert (ctx
->outer
3293 && gimple_code (ctx
->outer
->stmt
)
3294 == GIMPLE_OMP_SECTIONS
);
3295 ctx
->outer
->cancellable
= true;
3296 if (find_omp_clause (gimple_omp_sections_clauses
3299 warning_at (gimple_location (stmt
), 0,
3300 "%<#pragma omp cancel sections%> inside "
3301 "%<nowait%> sections construct");
3307 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3308 bad
= "#pragma omp task";
3310 ctx
->cancellable
= true;
3314 error_at (gimple_location (stmt
), "invalid arguments");
3319 error_at (gimple_location (stmt
),
3320 "%<%s %s%> construct not closely nested inside of %qs",
3321 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3322 == BUILT_IN_GOMP_CANCEL
3323 ? "#pragma omp cancel"
3324 : "#pragma omp cancellation point", kind
, bad
);
3329 case GIMPLE_OMP_SECTIONS
:
3330 case GIMPLE_OMP_SINGLE
:
3331 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3332 switch (gimple_code (ctx
->stmt
))
3334 case GIMPLE_OMP_FOR
:
3335 case GIMPLE_OMP_SECTIONS
:
3336 case GIMPLE_OMP_SINGLE
:
3337 case GIMPLE_OMP_ORDERED
:
3338 case GIMPLE_OMP_MASTER
:
3339 case GIMPLE_OMP_TASK
:
3340 case GIMPLE_OMP_CRITICAL
:
3341 if (is_gimple_call (stmt
))
3343 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3344 != BUILT_IN_GOMP_BARRIER
)
3346 error_at (gimple_location (stmt
),
3347 "barrier region may not be closely nested inside "
3348 "of work-sharing, critical, ordered, master or "
3349 "explicit task region");
3352 error_at (gimple_location (stmt
),
3353 "work-sharing region may not be closely nested inside "
3354 "of work-sharing, critical, ordered, master or explicit "
3357 case GIMPLE_OMP_PARALLEL
:
3363 case GIMPLE_OMP_MASTER
:
3364 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3365 switch (gimple_code (ctx
->stmt
))
3367 case GIMPLE_OMP_FOR
:
3368 case GIMPLE_OMP_SECTIONS
:
3369 case GIMPLE_OMP_SINGLE
:
3370 case GIMPLE_OMP_TASK
:
3371 error_at (gimple_location (stmt
),
3372 "master region may not be closely nested inside "
3373 "of work-sharing or explicit task region");
3375 case GIMPLE_OMP_PARALLEL
:
3381 case GIMPLE_OMP_TASK
:
3382 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3383 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3384 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3385 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3387 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3388 error_at (OMP_CLAUSE_LOCATION (c
),
3389 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3390 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3394 case GIMPLE_OMP_ORDERED
:
3395 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3396 c
; c
= OMP_CLAUSE_CHAIN (c
))
3398 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3400 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3402 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
));
3405 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3406 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3407 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3410 /* Look for containing ordered(N) loop. */
3412 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3414 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3415 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3417 error_at (OMP_CLAUSE_LOCATION (c
),
3418 "%<depend%> clause must be closely nested "
3419 "inside an ordered loop");
3422 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3424 error_at (OMP_CLAUSE_LOCATION (c
),
3425 "%<depend%> clause must be closely nested "
3426 "inside a loop with %<ordered%> clause with "
3433 error_at (OMP_CLAUSE_LOCATION (c
),
3434 "invalid depend kind in omp ordered depend");
3438 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3439 switch (gimple_code (ctx
->stmt
))
3441 case GIMPLE_OMP_CRITICAL
:
3442 case GIMPLE_OMP_TASK
:
3443 error_at (gimple_location (stmt
),
3444 "ordered region may not be closely nested inside "
3445 "of critical or explicit task region");
3447 case GIMPLE_OMP_FOR
:
3448 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3449 OMP_CLAUSE_ORDERED
) == NULL
)
3451 error_at (gimple_location (stmt
),
3452 "ordered region must be closely nested inside "
3453 "a loop region with an ordered clause");
3457 case GIMPLE_OMP_PARALLEL
:
3458 error_at (gimple_location (stmt
),
3459 "ordered region must be closely nested inside "
3460 "a loop region with an ordered clause");
3466 case GIMPLE_OMP_CRITICAL
:
3469 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3470 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3471 if (gomp_critical
*other_crit
3472 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3473 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3475 error_at (gimple_location (stmt
),
3476 "critical region may not be nested inside a critical "
3477 "region with the same name");
3482 case GIMPLE_OMP_TEAMS
:
3484 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3485 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3487 error_at (gimple_location (stmt
),
3488 "teams construct not closely nested inside of target "
3493 case GIMPLE_OMP_TARGET
:
3494 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3495 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3496 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3497 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3499 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3500 error_at (OMP_CLAUSE_LOCATION (c
),
3501 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3502 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3505 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3507 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3509 if (is_gimple_omp (stmt
)
3510 && is_gimple_omp_oacc (stmt
)
3511 && is_gimple_omp (ctx
->stmt
))
3513 error_at (gimple_location (stmt
),
3514 "OpenACC construct inside of non-OpenACC region");
3520 const char *stmt_name
, *ctx_stmt_name
;
3521 switch (gimple_omp_target_kind (stmt
))
3523 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3524 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3525 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3526 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3527 stmt_name
= "target enter data"; break;
3528 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3529 stmt_name
= "target exit data"; break;
3530 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3531 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3532 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3533 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3534 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3535 stmt_name
= "enter/exit data"; break;
3536 default: gcc_unreachable ();
3538 switch (gimple_omp_target_kind (ctx
->stmt
))
3540 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3541 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3542 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3543 ctx_stmt_name
= "parallel"; break;
3544 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3545 ctx_stmt_name
= "kernels"; break;
3546 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3547 default: gcc_unreachable ();
3550 /* OpenACC/OpenMP mismatch? */
3551 if (is_gimple_omp_oacc (stmt
)
3552 != is_gimple_omp_oacc (ctx
->stmt
))
3554 error_at (gimple_location (stmt
),
3555 "%s %s construct inside of %s %s region",
3556 (is_gimple_omp_oacc (stmt
)
3557 ? "OpenACC" : "OpenMP"), stmt_name
,
3558 (is_gimple_omp_oacc (ctx
->stmt
)
3559 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3562 if (is_gimple_omp_offloaded (ctx
->stmt
))
3564 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3565 if (is_gimple_omp_oacc (ctx
->stmt
))
3567 error_at (gimple_location (stmt
),
3568 "%s construct inside of %s region",
3569 stmt_name
, ctx_stmt_name
);
3574 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3575 warning_at (gimple_location (stmt
), 0,
3576 "%s construct inside of %s region",
3577 stmt_name
, ctx_stmt_name
);
3589 /* Helper function scan_omp.
3591 Callback for walk_tree or operators in walk_gimple_stmt used to
3592 scan for OMP directives in TP. */
3595 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3597 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3598 omp_context
*ctx
= (omp_context
*) wi
->info
;
3601 switch (TREE_CODE (t
))
3608 *tp
= remap_decl (t
, &ctx
->cb
);
3612 if (ctx
&& TYPE_P (t
))
3613 *tp
= remap_type (t
, &ctx
->cb
);
3614 else if (!DECL_P (t
))
3619 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3620 if (tem
!= TREE_TYPE (t
))
3622 if (TREE_CODE (t
) == INTEGER_CST
)
3623 *tp
= wide_int_to_tree (tem
, t
);
3625 TREE_TYPE (t
) = tem
;
3635 /* Return true if FNDECL is a setjmp or a longjmp. */
3638 setjmp_or_longjmp_p (const_tree fndecl
)
3640 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3641 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3642 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3645 tree declname
= DECL_NAME (fndecl
);
3648 const char *name
= IDENTIFIER_POINTER (declname
);
3649 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3653 /* Helper function for scan_omp.
3655 Callback for walk_gimple_stmt used to scan for OMP directives in
3656 the current statement in GSI. */
3659 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3660 struct walk_stmt_info
*wi
)
3662 gimple
*stmt
= gsi_stmt (*gsi
);
3663 omp_context
*ctx
= (omp_context
*) wi
->info
;
3665 if (gimple_has_location (stmt
))
3666 input_location
= gimple_location (stmt
);
3668 /* Check the nesting restrictions. */
3669 bool remove
= false;
3670 if (is_gimple_omp (stmt
))
3671 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3672 else if (is_gimple_call (stmt
))
3674 tree fndecl
= gimple_call_fndecl (stmt
);
3677 if (setjmp_or_longjmp_p (fndecl
)
3679 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3680 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3683 error_at (gimple_location (stmt
),
3684 "setjmp/longjmp inside simd construct");
3686 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3687 switch (DECL_FUNCTION_CODE (fndecl
))
3689 case BUILT_IN_GOMP_BARRIER
:
3690 case BUILT_IN_GOMP_CANCEL
:
3691 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3692 case BUILT_IN_GOMP_TASKYIELD
:
3693 case BUILT_IN_GOMP_TASKWAIT
:
3694 case BUILT_IN_GOMP_TASKGROUP_START
:
3695 case BUILT_IN_GOMP_TASKGROUP_END
:
3696 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3705 stmt
= gimple_build_nop ();
3706 gsi_replace (gsi
, stmt
, false);
3709 *handled_ops_p
= true;
3711 switch (gimple_code (stmt
))
3713 case GIMPLE_OMP_PARALLEL
:
3714 taskreg_nesting_level
++;
3715 scan_omp_parallel (gsi
, ctx
);
3716 taskreg_nesting_level
--;
3719 case GIMPLE_OMP_TASK
:
3720 taskreg_nesting_level
++;
3721 scan_omp_task (gsi
, ctx
);
3722 taskreg_nesting_level
--;
3725 case GIMPLE_OMP_FOR
:
3726 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3729 case GIMPLE_OMP_SECTIONS
:
3730 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3733 case GIMPLE_OMP_SINGLE
:
3734 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3737 case GIMPLE_OMP_SECTION
:
3738 case GIMPLE_OMP_MASTER
:
3739 case GIMPLE_OMP_TASKGROUP
:
3740 case GIMPLE_OMP_ORDERED
:
3741 case GIMPLE_OMP_CRITICAL
:
3742 ctx
= new_omp_context (stmt
, ctx
);
3743 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3746 case GIMPLE_OMP_TARGET
:
3747 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3750 case GIMPLE_OMP_TEAMS
:
3751 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3758 *handled_ops_p
= false;
3760 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3762 var
= DECL_CHAIN (var
))
3763 insert_decl_map (&ctx
->cb
, var
, var
);
3767 *handled_ops_p
= false;
3775 /* Scan all the statements starting at the current statement. CTX
3776 contains context information about the OMP directives and
3777 clauses found during the scan. */
3780 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3782 location_t saved_location
;
3783 struct walk_stmt_info wi
;
3785 memset (&wi
, 0, sizeof (wi
));
3787 wi
.want_locations
= true;
3789 saved_location
= input_location
;
3790 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3791 input_location
= saved_location
;
3794 /* Re-gimplification and code generation routines. */
3796 /* Build a call to GOMP_barrier. */
3799 build_omp_barrier (tree lhs
)
3801 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3802 : BUILT_IN_GOMP_BARRIER
);
3803 gcall
*g
= gimple_build_call (fndecl
, 0);
3805 gimple_call_set_lhs (g
, lhs
);
3809 /* If a context was created for STMT when it was scanned, return it. */
3811 static omp_context
*
3812 maybe_lookup_ctx (gimple
*stmt
)
3815 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3816 return n
? (omp_context
*) n
->value
: NULL
;
3820 /* Find the mapping for DECL in CTX or the immediately enclosing
3821 context that has a mapping for DECL.
3823 If CTX is a nested parallel directive, we may have to use the decl
3824 mappings created in CTX's parent context. Suppose that we have the
3825 following parallel nesting (variable UIDs showed for clarity):
3828 #omp parallel shared(iD.1562) -> outer parallel
3829 iD.1562 = iD.1562 + 1;
3831 #omp parallel shared (iD.1562) -> inner parallel
3832 iD.1562 = iD.1562 - 1;
3834 Each parallel structure will create a distinct .omp_data_s structure
3835 for copying iD.1562 in/out of the directive:
3837 outer parallel .omp_data_s.1.i -> iD.1562
3838 inner parallel .omp_data_s.2.i -> iD.1562
3840 A shared variable mapping will produce a copy-out operation before
3841 the parallel directive and a copy-in operation after it. So, in
3842 this case we would have:
3845 .omp_data_o.1.i = iD.1562;
3846 #omp parallel shared(iD.1562) -> outer parallel
3847 .omp_data_i.1 = &.omp_data_o.1
3848 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3850 .omp_data_o.2.i = iD.1562; -> **
3851 #omp parallel shared(iD.1562) -> inner parallel
3852 .omp_data_i.2 = &.omp_data_o.2
3853 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3856 ** This is a problem. The symbol iD.1562 cannot be referenced
3857 inside the body of the outer parallel region. But since we are
3858 emitting this copy operation while expanding the inner parallel
3859 directive, we need to access the CTX structure of the outer
3860 parallel directive to get the correct mapping:
3862 .omp_data_o.2.i = .omp_data_i.1->i
3864 Since there may be other workshare or parallel directives enclosing
3865 the parallel directive, it may be necessary to walk up the context
3866 parent chain. This is not a problem in general because nested
3867 parallelism happens only rarely. */
3870 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3875 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3876 t
= maybe_lookup_decl (decl
, up
);
3878 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3880 return t
? t
: decl
;
3884 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3885 in outer contexts. */
3888 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3893 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3894 t
= maybe_lookup_decl (decl
, up
);
3896 return t
? t
: decl
;
3900 /* Construct the initialization value for reduction operation OP. */
3903 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
3912 case TRUTH_ORIF_EXPR
:
3913 case TRUTH_XOR_EXPR
:
3915 return build_zero_cst (type
);
3918 case TRUTH_AND_EXPR
:
3919 case TRUTH_ANDIF_EXPR
:
3921 return fold_convert_loc (loc
, type
, integer_one_node
);
3924 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3927 if (SCALAR_FLOAT_TYPE_P (type
))
3929 REAL_VALUE_TYPE max
, min
;
3930 if (HONOR_INFINITIES (type
))
3933 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3936 real_maxval (&min
, 1, TYPE_MODE (type
));
3937 return build_real (type
, min
);
3939 else if (POINTER_TYPE_P (type
))
3942 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
3943 return wide_int_to_tree (type
, min
);
3947 gcc_assert (INTEGRAL_TYPE_P (type
));
3948 return TYPE_MIN_VALUE (type
);
3952 if (SCALAR_FLOAT_TYPE_P (type
))
3954 REAL_VALUE_TYPE max
;
3955 if (HONOR_INFINITIES (type
))
3958 real_maxval (&max
, 0, TYPE_MODE (type
));
3959 return build_real (type
, max
);
3961 else if (POINTER_TYPE_P (type
))
3964 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
3965 return wide_int_to_tree (type
, max
);
3969 gcc_assert (INTEGRAL_TYPE_P (type
));
3970 return TYPE_MAX_VALUE (type
);
3978 /* Construct the initialization value for reduction CLAUSE. */
3981 omp_reduction_init (tree clause
, tree type
)
3983 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
3984 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
3987 /* Return alignment to be assumed for var in CLAUSE, which should be
3988 OMP_CLAUSE_ALIGNED. */
3991 omp_clause_aligned_alignment (tree clause
)
3993 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3994 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3996 /* Otherwise return implementation defined alignment. */
3997 unsigned int al
= 1;
3998 machine_mode mode
, vmode
;
3999 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4001 vs
= 1 << floor_log2 (vs
);
4002 static enum mode_class classes
[]
4003 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4004 for (int i
= 0; i
< 4; i
+= 2)
4005 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4007 mode
= GET_MODE_WIDER_MODE (mode
))
4009 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4010 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4013 && GET_MODE_SIZE (vmode
) < vs
4014 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4015 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4017 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4018 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4020 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4021 / GET_MODE_SIZE (mode
));
4022 if (TYPE_MODE (type
) != vmode
)
4024 if (TYPE_ALIGN_UNIT (type
) > al
)
4025 al
= TYPE_ALIGN_UNIT (type
);
4027 return build_int_cst (integer_type_node
, al
);
4030 /* Return maximum possible vectorization factor for the target. */
4037 || !flag_tree_loop_optimize
4038 || (!flag_tree_loop_vectorize
4039 && (global_options_set
.x_flag_tree_loop_vectorize
4040 || global_options_set
.x_flag_tree_vectorize
)))
4043 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4046 vs
= 1 << floor_log2 (vs
);
4049 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4050 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4051 return GET_MODE_NUNITS (vqimode
);
4055 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4059 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4060 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4064 max_vf
= omp_max_vf ();
4067 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4068 OMP_CLAUSE_SAFELEN
);
4069 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4071 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4073 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4077 idx
= create_tmp_var (unsigned_type_node
);
4078 lane
= create_tmp_var (unsigned_type_node
);
4084 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4085 tree avar
= create_tmp_var_raw (atype
);
4086 if (TREE_ADDRESSABLE (new_var
))
4087 TREE_ADDRESSABLE (avar
) = 1;
4088 DECL_ATTRIBUTES (avar
)
4089 = tree_cons (get_identifier ("omp simd array"), NULL
,
4090 DECL_ATTRIBUTES (avar
));
4091 gimple_add_tmp_var (avar
);
4092 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4093 NULL_TREE
, NULL_TREE
);
4094 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4095 NULL_TREE
, NULL_TREE
);
4096 if (DECL_P (new_var
))
4098 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4099 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4104 /* Helper function of lower_rec_input_clauses. For a reference
4105 in simd reduction, add an underlying variable it will reference. */
4108 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4110 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4111 if (TREE_CONSTANT (z
))
4113 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4114 get_name (new_vard
));
4115 gimple_add_tmp_var (z
);
4116 TREE_ADDRESSABLE (z
) = 1;
4117 z
= build_fold_addr_expr_loc (loc
, z
);
4118 gimplify_assign (new_vard
, z
, ilist
);
4122 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4123 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4124 private variables. Initialization statements go in ILIST, while calls
4125 to destructors go in DLIST. */
4128 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4129 omp_context
*ctx
, struct omp_for_data
*fd
)
4131 tree c
, dtor
, copyin_seq
, x
, ptr
;
4132 bool copyin_by_ref
= false;
4133 bool lastprivate_firstprivate
= false;
4134 bool reduction_omp_orig_ref
= false;
4136 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4137 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4139 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4140 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4141 gimple_seq llist
[2] = { NULL
, NULL
};
4145 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4146 with data sharing clauses referencing variable sized vars. That
4147 is unnecessarily hard to support and very unlikely to result in
4148 vectorized code anyway. */
4150 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4151 switch (OMP_CLAUSE_CODE (c
))
4153 case OMP_CLAUSE_LINEAR
:
4154 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4157 case OMP_CLAUSE_PRIVATE
:
4158 case OMP_CLAUSE_FIRSTPRIVATE
:
4159 case OMP_CLAUSE_LASTPRIVATE
:
4160 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4163 case OMP_CLAUSE_REDUCTION
:
4164 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4165 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4172 /* Do all the fixed sized types in the first pass, and the variable sized
4173 types in the second pass. This makes sure that the scalar arguments to
4174 the variable sized types are processed before we use them in the
4175 variable sized operations. */
4176 for (pass
= 0; pass
< 2; ++pass
)
4178 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4180 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4183 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4187 case OMP_CLAUSE_PRIVATE
:
4188 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4191 case OMP_CLAUSE_SHARED
:
4192 /* Ignore shared directives in teams construct. */
4193 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4195 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4197 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4198 || is_global_var (OMP_CLAUSE_DECL (c
)));
4201 case OMP_CLAUSE_FIRSTPRIVATE
:
4202 case OMP_CLAUSE_COPYIN
:
4204 case OMP_CLAUSE_LINEAR
:
4205 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4206 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4207 lastprivate_firstprivate
= true;
4209 case OMP_CLAUSE_REDUCTION
:
4210 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4211 reduction_omp_orig_ref
= true;
4213 case OMP_CLAUSE__LOOPTEMP_
:
4214 /* Handle _looptemp_ clauses only on parallel/task. */
4218 case OMP_CLAUSE_LASTPRIVATE
:
4219 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4221 lastprivate_firstprivate
= true;
4222 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4225 /* Even without corresponding firstprivate, if
4226 decl is Fortran allocatable, it needs outer var
4229 && lang_hooks
.decls
.omp_private_outer_ref
4230 (OMP_CLAUSE_DECL (c
)))
4231 lastprivate_firstprivate
= true;
4233 case OMP_CLAUSE_ALIGNED
:
4236 var
= OMP_CLAUSE_DECL (c
);
4237 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4238 && !is_global_var (var
))
4240 new_var
= maybe_lookup_decl (var
, ctx
);
4241 if (new_var
== NULL_TREE
)
4242 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4243 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4244 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4245 omp_clause_aligned_alignment (c
));
4246 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4247 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4248 gimplify_and_add (x
, ilist
);
4250 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4251 && is_global_var (var
))
4253 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4254 new_var
= lookup_decl (var
, ctx
);
4255 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4256 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4257 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4258 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4259 omp_clause_aligned_alignment (c
));
4260 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4261 x
= create_tmp_var (ptype
);
4262 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4263 gimplify_and_add (t
, ilist
);
4264 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4265 SET_DECL_VALUE_EXPR (new_var
, t
);
4266 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4273 new_var
= var
= OMP_CLAUSE_DECL (c
);
4274 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4276 var
= TREE_OPERAND (var
, 0);
4277 if (TREE_CODE (var
) == INDIRECT_REF
4278 || TREE_CODE (var
) == ADDR_EXPR
)
4279 var
= TREE_OPERAND (var
, 0);
4280 if (is_variable_sized (var
))
4282 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4283 var
= DECL_VALUE_EXPR (var
);
4284 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4285 var
= TREE_OPERAND (var
, 0);
4286 gcc_assert (DECL_P (var
));
4290 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4291 new_var
= lookup_decl (var
, ctx
);
4293 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4298 /* C/C++ array section reductions. */
4299 else if (c_kind
== OMP_CLAUSE_REDUCTION
4300 && var
!= OMP_CLAUSE_DECL (c
))
4305 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4306 if (TREE_CODE (orig_var
) == INDIRECT_REF
4307 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4308 orig_var
= TREE_OPERAND (orig_var
, 0);
4309 tree d
= OMP_CLAUSE_DECL (c
);
4310 tree type
= TREE_TYPE (d
);
4311 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4312 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4313 const char *name
= get_name (orig_var
);
4314 if (TREE_CONSTANT (v
))
4316 x
= create_tmp_var_raw (type
, name
);
4317 gimple_add_tmp_var (x
);
4318 TREE_ADDRESSABLE (x
) = 1;
4319 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4324 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4325 tree t
= maybe_lookup_decl (v
, ctx
);
4329 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4330 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4331 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4333 build_int_cst (TREE_TYPE (v
), 1));
4334 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4336 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4337 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4338 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4341 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4342 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4343 tree y
= create_tmp_var (ptype
, name
);
4344 gimplify_assign (y
, x
, ilist
);
4346 if (TREE_CODE (TREE_OPERAND (d
, 0)) == ADDR_EXPR
)
4348 if (orig_var
!= var
)
4350 gcc_assert (is_variable_sized (orig_var
));
4351 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4353 gimplify_assign (new_var
, x
, ilist
);
4354 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4355 tree t
= build_fold_indirect_ref (new_var
);
4356 DECL_IGNORED_P (new_var
) = 0;
4357 TREE_THIS_NOTRAP (t
);
4358 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4359 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4363 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4364 build_int_cst (ptype
, 0));
4365 SET_DECL_VALUE_EXPR (new_var
, x
);
4366 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4371 gcc_assert (orig_var
== var
);
4372 if (TREE_CODE (TREE_OPERAND (d
, 0)) == INDIRECT_REF
)
4374 x
= create_tmp_var (ptype
, name
);
4375 TREE_ADDRESSABLE (x
) = 1;
4376 gimplify_assign (x
, y
, ilist
);
4377 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4379 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4380 gimplify_assign (new_var
, x
, ilist
);
4382 tree y1
= create_tmp_var (ptype
, NULL
);
4383 gimplify_assign (y1
, y
, ilist
);
4384 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4385 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4386 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4387 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4389 y2
= create_tmp_var (ptype
, NULL
);
4390 gimplify_assign (y2
, y
, ilist
);
4391 tree ref
= build_outer_var_ref (var
, ctx
);
4392 /* For ref build_outer_var_ref already performs this. */
4393 if (TREE_CODE (TREE_OPERAND (d
, 0)) == INDIRECT_REF
)
4394 gcc_assert (is_reference (var
));
4395 else if (TREE_CODE (TREE_OPERAND (d
, 0)) == ADDR_EXPR
)
4396 ref
= build_fold_addr_expr (ref
);
4397 else if (is_reference (var
))
4398 ref
= build_fold_addr_expr (ref
);
4399 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4400 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4401 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4403 y3
= create_tmp_var (ptype
, NULL
);
4404 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4408 y4
= create_tmp_var (ptype
, NULL
);
4409 gimplify_assign (y4
, ref
, dlist
);
4412 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4413 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4414 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4415 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4416 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4419 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4420 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4421 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4422 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4423 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4425 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4427 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4428 tree decl_placeholder
4429 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4430 SET_DECL_VALUE_EXPR (decl_placeholder
,
4431 build_simple_mem_ref (y1
));
4432 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4433 SET_DECL_VALUE_EXPR (placeholder
,
4434 y3
? build_simple_mem_ref (y3
)
4436 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4437 x
= lang_hooks
.decls
.omp_clause_default_ctor
4438 (c
, build_simple_mem_ref (y1
),
4439 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4441 gimplify_and_add (x
, ilist
);
4442 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4444 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4445 lower_omp (&tseq
, ctx
);
4446 gimple_seq_add_seq (ilist
, tseq
);
4448 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4451 SET_DECL_VALUE_EXPR (decl_placeholder
,
4452 build_simple_mem_ref (y2
));
4453 SET_DECL_VALUE_EXPR (placeholder
,
4454 build_simple_mem_ref (y4
));
4455 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4456 lower_omp (&tseq
, ctx
);
4457 gimple_seq_add_seq (dlist
, tseq
);
4458 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4460 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4461 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4462 x
= lang_hooks
.decls
.omp_clause_dtor
4463 (c
, build_simple_mem_ref (y2
));
4466 gimple_seq tseq
= NULL
;
4468 gimplify_stmt (&dtor
, &tseq
);
4469 gimple_seq_add_seq (dlist
, tseq
);
4474 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4475 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4477 /* reduction(-:var) sums up the partial results, so it
4478 acts identically to reduction(+:var). */
4479 if (code
== MINUS_EXPR
)
4482 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4485 x
= build2 (code
, TREE_TYPE (type
),
4486 build_simple_mem_ref (y4
),
4487 build_simple_mem_ref (y2
));
4488 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4492 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4493 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4494 gimple_seq_add_stmt (ilist
, g
);
4497 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4498 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4499 gimple_seq_add_stmt (ilist
, g
);
4501 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4502 build_int_cst (TREE_TYPE (i
), 1));
4503 gimple_seq_add_stmt (ilist
, g
);
4504 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4505 gimple_seq_add_stmt (ilist
, g
);
4506 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4509 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4510 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4511 gimple_seq_add_stmt (dlist
, g
);
4514 g
= gimple_build_assign
4515 (y4
, POINTER_PLUS_EXPR
, y4
,
4516 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4517 gimple_seq_add_stmt (dlist
, g
);
4519 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4520 build_int_cst (TREE_TYPE (i2
), 1));
4521 gimple_seq_add_stmt (dlist
, g
);
4522 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4523 gimple_seq_add_stmt (dlist
, g
);
4524 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4528 else if (is_variable_sized (var
))
4530 /* For variable sized types, we need to allocate the
4531 actual storage here. Call alloca and store the
4532 result in the pointer decl that we created elsewhere. */
4536 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4541 ptr
= DECL_VALUE_EXPR (new_var
);
4542 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4543 ptr
= TREE_OPERAND (ptr
, 0);
4544 gcc_assert (DECL_P (ptr
));
4545 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4547 /* void *tmp = __builtin_alloca */
4548 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4549 stmt
= gimple_build_call (atmp
, 2, x
,
4550 size_int (DECL_ALIGN (var
)));
4551 tmp
= create_tmp_var_raw (ptr_type_node
);
4552 gimple_add_tmp_var (tmp
);
4553 gimple_call_set_lhs (stmt
, tmp
);
4555 gimple_seq_add_stmt (ilist
, stmt
);
4557 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4558 gimplify_assign (ptr
, x
, ilist
);
4561 else if (is_reference (var
))
4563 /* For references that are being privatized for Fortran,
4564 allocate new backing storage for the new pointer
4565 variable. This allows us to avoid changing all the
4566 code that expects a pointer to something that expects
4567 a direct variable. */
4571 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4572 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4574 x
= build_receiver_ref (var
, false, ctx
);
4575 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4577 else if (TREE_CONSTANT (x
))
4579 /* For reduction in SIMD loop, defer adding the
4580 initialization of the reference, because if we decide
4581 to use SIMD array for it, the initilization could cause
4583 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4587 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4589 gimple_add_tmp_var (x
);
4590 TREE_ADDRESSABLE (x
) = 1;
4591 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4597 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4598 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4599 tree al
= size_int (TYPE_ALIGN (rtype
));
4600 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4605 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4606 gimplify_assign (new_var
, x
, ilist
);
4609 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4611 else if (c_kind
== OMP_CLAUSE_REDUCTION
4612 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4620 switch (OMP_CLAUSE_CODE (c
))
4622 case OMP_CLAUSE_SHARED
:
4623 /* Ignore shared directives in teams construct. */
4624 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4626 /* Shared global vars are just accessed directly. */
4627 if (is_global_var (new_var
))
4629 /* For taskloop firstprivate/lastprivate, represented
4630 as firstprivate and shared clause on the task, new_var
4631 is the firstprivate var. */
4632 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4634 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4635 needs to be delayed until after fixup_child_record_type so
4636 that we get the correct type during the dereference. */
4637 by_ref
= use_pointer_for_field (var
, ctx
);
4638 x
= build_receiver_ref (var
, by_ref
, ctx
);
4639 SET_DECL_VALUE_EXPR (new_var
, x
);
4640 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4642 /* ??? If VAR is not passed by reference, and the variable
4643 hasn't been initialized yet, then we'll get a warning for
4644 the store into the omp_data_s structure. Ideally, we'd be
4645 able to notice this and not store anything at all, but
4646 we're generating code too early. Suppress the warning. */
4648 TREE_NO_WARNING (var
) = 1;
4651 case OMP_CLAUSE_LASTPRIVATE
:
4652 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4656 case OMP_CLAUSE_PRIVATE
:
4657 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4658 x
= build_outer_var_ref (var
, ctx
);
4659 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4661 if (is_task_ctx (ctx
))
4662 x
= build_receiver_ref (var
, false, ctx
);
4664 x
= build_outer_var_ref (var
, ctx
);
4670 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4671 (c
, unshare_expr (new_var
), x
);
4674 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4675 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4676 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4677 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4678 idx
, lane
, ivar
, lvar
))
4681 x
= lang_hooks
.decls
.omp_clause_default_ctor
4682 (c
, unshare_expr (ivar
), x
);
4684 gimplify_and_add (x
, &llist
[0]);
4687 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4690 gimple_seq tseq
= NULL
;
4693 gimplify_stmt (&dtor
, &tseq
);
4694 gimple_seq_add_seq (&llist
[1], tseq
);
4701 gimplify_and_add (nx
, ilist
);
4705 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4708 gimple_seq tseq
= NULL
;
4711 gimplify_stmt (&dtor
, &tseq
);
4712 gimple_seq_add_seq (dlist
, tseq
);
4716 case OMP_CLAUSE_LINEAR
:
4717 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4718 goto do_firstprivate
;
4719 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4722 x
= build_outer_var_ref (var
, ctx
);
4725 case OMP_CLAUSE_FIRSTPRIVATE
:
4726 if (is_task_ctx (ctx
))
4728 if (is_reference (var
) || is_variable_sized (var
))
4730 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4732 || use_pointer_for_field (var
, NULL
))
4734 x
= build_receiver_ref (var
, false, ctx
);
4735 SET_DECL_VALUE_EXPR (new_var
, x
);
4736 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4741 x
= build_outer_var_ref (var
, ctx
);
4744 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4745 && gimple_omp_for_combined_into_p (ctx
->stmt
))
4747 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4748 tree stept
= TREE_TYPE (t
);
4749 tree ct
= find_omp_clause (clauses
,
4750 OMP_CLAUSE__LOOPTEMP_
);
4752 tree l
= OMP_CLAUSE_DECL (ct
);
4753 tree n1
= fd
->loop
.n1
;
4754 tree step
= fd
->loop
.step
;
4755 tree itype
= TREE_TYPE (l
);
4756 if (POINTER_TYPE_P (itype
))
4757 itype
= signed_type_for (itype
);
4758 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
4759 if (TYPE_UNSIGNED (itype
)
4760 && fd
->loop
.cond_code
== GT_EXPR
)
4761 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4762 fold_build1 (NEGATE_EXPR
, itype
, l
),
4763 fold_build1 (NEGATE_EXPR
,
4766 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
4767 t
= fold_build2 (MULT_EXPR
, stept
,
4768 fold_convert (stept
, l
), t
);
4770 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4772 x
= lang_hooks
.decls
.omp_clause_linear_ctor
4774 gimplify_and_add (x
, ilist
);
4778 if (POINTER_TYPE_P (TREE_TYPE (x
)))
4779 x
= fold_build2 (POINTER_PLUS_EXPR
,
4780 TREE_TYPE (x
), x
, t
);
4782 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4785 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4786 || TREE_ADDRESSABLE (new_var
))
4787 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4788 idx
, lane
, ivar
, lvar
))
4790 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4792 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4793 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4794 gimplify_and_add (x
, ilist
);
4795 gimple_stmt_iterator gsi
4796 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4798 = gimple_build_assign (unshare_expr (lvar
), iv
);
4799 gsi_insert_before_without_update (&gsi
, g
,
4801 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4802 enum tree_code code
= PLUS_EXPR
;
4803 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4804 code
= POINTER_PLUS_EXPR
;
4805 g
= gimple_build_assign (iv
, code
, iv
, t
);
4806 gsi_insert_before_without_update (&gsi
, g
,
4810 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4811 (c
, unshare_expr (ivar
), x
);
4812 gimplify_and_add (x
, &llist
[0]);
4813 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4816 gimple_seq tseq
= NULL
;
4819 gimplify_stmt (&dtor
, &tseq
);
4820 gimple_seq_add_seq (&llist
[1], tseq
);
4825 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4826 (c
, unshare_expr (new_var
), x
);
4827 gimplify_and_add (x
, ilist
);
4830 case OMP_CLAUSE__LOOPTEMP_
:
4831 gcc_assert (is_taskreg_ctx (ctx
));
4832 x
= build_outer_var_ref (var
, ctx
);
4833 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4834 gimplify_and_add (x
, ilist
);
4837 case OMP_CLAUSE_COPYIN
:
4838 by_ref
= use_pointer_for_field (var
, NULL
);
4839 x
= build_receiver_ref (var
, by_ref
, ctx
);
4840 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4841 append_to_statement_list (x
, ©in_seq
);
4842 copyin_by_ref
|= by_ref
;
4845 case OMP_CLAUSE_REDUCTION
:
4846 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4848 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4850 x
= build_outer_var_ref (var
, ctx
);
4852 if (is_reference (var
)
4853 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4855 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4856 SET_DECL_VALUE_EXPR (placeholder
, x
);
4857 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4858 tree new_vard
= new_var
;
4859 if (is_reference (var
))
4861 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4862 new_vard
= TREE_OPERAND (new_var
, 0);
4863 gcc_assert (DECL_P (new_vard
));
4866 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4867 idx
, lane
, ivar
, lvar
))
4869 if (new_vard
== new_var
)
4871 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4872 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4876 SET_DECL_VALUE_EXPR (new_vard
,
4877 build_fold_addr_expr (ivar
));
4878 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4880 x
= lang_hooks
.decls
.omp_clause_default_ctor
4881 (c
, unshare_expr (ivar
),
4882 build_outer_var_ref (var
, ctx
));
4884 gimplify_and_add (x
, &llist
[0]);
4885 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4887 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4888 lower_omp (&tseq
, ctx
);
4889 gimple_seq_add_seq (&llist
[0], tseq
);
4891 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4892 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4893 lower_omp (&tseq
, ctx
);
4894 gimple_seq_add_seq (&llist
[1], tseq
);
4895 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4896 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4897 if (new_vard
== new_var
)
4898 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4900 SET_DECL_VALUE_EXPR (new_vard
,
4901 build_fold_addr_expr (lvar
));
4902 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4907 gimplify_stmt (&dtor
, &tseq
);
4908 gimple_seq_add_seq (&llist
[1], tseq
);
4912 /* If this is a reference to constant size reduction var
4913 with placeholder, we haven't emitted the initializer
4914 for it because it is undesirable if SIMD arrays are used.
4915 But if they aren't used, we need to emit the deferred
4916 initialization now. */
4917 else if (is_reference (var
) && is_simd
)
4918 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4919 x
= lang_hooks
.decls
.omp_clause_default_ctor
4920 (c
, unshare_expr (new_var
),
4921 build_outer_var_ref (var
, ctx
));
4923 gimplify_and_add (x
, ilist
);
4924 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4926 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4927 lower_omp (&tseq
, ctx
);
4928 gimple_seq_add_seq (ilist
, tseq
);
4930 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4933 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4934 lower_omp (&tseq
, ctx
);
4935 gimple_seq_add_seq (dlist
, tseq
);
4936 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4938 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4943 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4944 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4945 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4947 /* reduction(-:var) sums up the partial results, so it
4948 acts identically to reduction(+:var). */
4949 if (code
== MINUS_EXPR
)
4952 tree new_vard
= new_var
;
4953 if (is_simd
&& is_reference (var
))
4955 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4956 new_vard
= TREE_OPERAND (new_var
, 0);
4957 gcc_assert (DECL_P (new_vard
));
4960 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4961 idx
, lane
, ivar
, lvar
))
4963 tree ref
= build_outer_var_ref (var
, ctx
);
4965 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4967 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4968 ref
= build_outer_var_ref (var
, ctx
);
4969 gimplify_assign (ref
, x
, &llist
[1]);
4971 if (new_vard
!= new_var
)
4973 SET_DECL_VALUE_EXPR (new_vard
,
4974 build_fold_addr_expr (lvar
));
4975 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4980 if (is_reference (var
) && is_simd
)
4981 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4982 gimplify_assign (new_var
, x
, ilist
);
4985 tree ref
= build_outer_var_ref (var
, ctx
);
4987 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4988 ref
= build_outer_var_ref (var
, ctx
);
4989 gimplify_assign (ref
, x
, dlist
);
5003 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5004 /* Don't want uninit warnings on simduid, it is always uninitialized,
5005 but we use it not for the value, but for the DECL_UID only. */
5006 TREE_NO_WARNING (uid
) = 1;
5008 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5009 gimple_call_set_lhs (g
, lane
);
5010 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5011 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5012 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5013 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5014 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5015 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5016 g
= gimple_build_assign (lane
, INTEGER_CST
,
5017 build_int_cst (unsigned_type_node
, 0));
5018 gimple_seq_add_stmt (ilist
, g
);
5019 for (int i
= 0; i
< 2; i
++)
5022 tree vf
= create_tmp_var (unsigned_type_node
);
5023 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5024 gimple_call_set_lhs (g
, vf
);
5025 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5026 gimple_seq_add_stmt (seq
, g
);
5027 tree t
= build_int_cst (unsigned_type_node
, 0);
5028 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5029 gimple_seq_add_stmt (seq
, g
);
5030 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5031 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5032 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5033 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5034 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5035 gimple_seq_add_seq (seq
, llist
[i
]);
5036 t
= build_int_cst (unsigned_type_node
, 1);
5037 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5038 gimple_seq_add_stmt (seq
, g
);
5039 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5040 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5041 gimple_seq_add_stmt (seq
, g
);
5042 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5046 /* The copyin sequence is not to be executed by the main thread, since
5047 that would result in self-copies. Perhaps not visible to scalars,
5048 but it certainly is to C++ operator=. */
5051 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5053 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5054 build_int_cst (TREE_TYPE (x
), 0));
5055 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5056 gimplify_and_add (x
, ilist
);
5059 /* If any copyin variable is passed by reference, we must ensure the
5060 master thread doesn't modify it before it is copied over in all
5061 threads. Similarly for variables in both firstprivate and
5062 lastprivate clauses we need to ensure the lastprivate copying
5063 happens after firstprivate copying in all threads. And similarly
5064 for UDRs if initializer expression refers to omp_orig. */
5065 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5067 /* Don't add any barrier for #pragma omp simd or
5068 #pragma omp distribute. */
5069 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5070 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5071 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5074 /* If max_vf is non-zero, then we can use only a vectorization factor
5075 up to the max_vf we chose. So stick it into the safelen clause. */
5078 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5079 OMP_CLAUSE_SAFELEN
);
5081 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5082 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5085 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5086 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5088 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5089 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5095 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5096 both parallel and workshare constructs. PREDICATE may be NULL if it's
5100 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5103 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5104 bool par_clauses
= false;
5105 tree simduid
= NULL
, lastlane
= NULL
;
5107 /* Early exit if there are no lastprivate or linear clauses. */
5108 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5109 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5110 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5111 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5113 if (clauses
== NULL
)
5115 /* If this was a workshare clause, see if it had been combined
5116 with its parallel. In that case, look for the clauses on the
5117 parallel statement itself. */
5118 if (is_parallel_ctx (ctx
))
5122 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5125 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5126 OMP_CLAUSE_LASTPRIVATE
);
5127 if (clauses
== NULL
)
5135 tree label_true
, arm1
, arm2
;
5137 label
= create_artificial_label (UNKNOWN_LOCATION
);
5138 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5139 arm1
= TREE_OPERAND (predicate
, 0);
5140 arm2
= TREE_OPERAND (predicate
, 1);
5141 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5142 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5143 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5145 gimple_seq_add_stmt (stmt_list
, stmt
);
5146 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5149 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5150 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5152 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5154 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5157 for (c
= clauses
; c
;)
5160 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5162 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5163 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5164 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5166 var
= OMP_CLAUSE_DECL (c
);
5167 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5168 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5169 && is_taskloop_ctx (ctx
))
5171 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5172 new_var
= lookup_decl (var
, ctx
->outer
);
5175 new_var
= lookup_decl (var
, ctx
);
5177 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5179 tree val
= DECL_VALUE_EXPR (new_var
);
5180 if (TREE_CODE (val
) == ARRAY_REF
5181 && VAR_P (TREE_OPERAND (val
, 0))
5182 && lookup_attribute ("omp simd array",
5183 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5186 if (lastlane
== NULL
)
5188 lastlane
= create_tmp_var (unsigned_type_node
);
5190 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5192 TREE_OPERAND (val
, 1));
5193 gimple_call_set_lhs (g
, lastlane
);
5194 gimple_seq_add_stmt (stmt_list
, g
);
5196 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5197 TREE_OPERAND (val
, 0), lastlane
,
5198 NULL_TREE
, NULL_TREE
);
5202 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5203 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5205 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5206 gimple_seq_add_seq (stmt_list
,
5207 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5208 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5210 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5211 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5213 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5214 gimple_seq_add_seq (stmt_list
,
5215 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5216 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5220 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5221 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5223 gcc_checking_assert (is_taskloop_ctx (ctx
));
5224 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5226 if (is_global_var (ovar
))
5230 x
= build_outer_var_ref (var
, ctx
, true);
5231 if (is_reference (var
))
5232 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5233 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5234 gimplify_and_add (x
, stmt_list
);
5236 c
= OMP_CLAUSE_CHAIN (c
);
5237 if (c
== NULL
&& !par_clauses
)
5239 /* If this was a workshare clause, see if it had been combined
5240 with its parallel. In that case, continue looking for the
5241 clauses also on the parallel statement itself. */
5242 if (is_parallel_ctx (ctx
))
5246 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5249 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5250 OMP_CLAUSE_LASTPRIVATE
);
5256 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5260 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
5261 tree tid
, tree var
, tree new_var
)
5263 /* The atomic add at the end of the sum creates unnecessary
5264 write contention on accelerators. To work around this,
5265 create an array to store the partial reductions. Later, in
5266 lower_omp_for (for openacc), the values of array will be
5269 tree t
= NULL_TREE
, array
, x
;
5270 tree type
= get_base_type (var
);
5273 /* Now insert the partial reductions into the array. */
5275 /* Find the reduction array. */
5277 tree ptype
= build_pointer_type (type
);
5279 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
5280 t
= build_receiver_ref (t
, false, ctx
->outer
);
5282 array
= create_tmp_var (ptype
);
5283 gimplify_assign (array
, t
, stmt_seqp
);
5285 tree ptr
= create_tmp_var (TREE_TYPE (array
));
5287 /* Find the reduction array. */
5289 /* testing a unary conversion. */
5290 tree offset
= create_tmp_var (sizetype
);
5291 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
5293 t
= create_tmp_var (sizetype
);
5294 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
5296 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
5297 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5299 /* Offset expression. Does the POINTER_PLUS_EXPR take care
5300 of adding sizeof(var) to the array? */
5301 ptr
= create_tmp_var (ptype
);
5302 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
5304 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5306 /* Move the local sum to gfc$sum[i]. */
5307 x
= unshare_expr (build_simple_mem_ref (ptr
));
5308 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
5311 /* Generate code to implement the REDUCTION clauses. */
5314 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5316 gimple_seq sub_seq
= NULL
;
5318 tree x
, c
, tid
= NULL_TREE
;
5321 /* SIMD reductions are handled in lower_rec_input_clauses. */
5322 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5323 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5326 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5327 update in that case, otherwise use a lock. */
5328 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5329 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5331 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5332 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5334 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5344 /* Initialize thread info for OpenACC. */
5345 if (is_gimple_omp_oacc (ctx
->stmt
))
5347 /* Get the current thread id. */
5348 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
5349 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
5350 gimple
*stmt
= gimple_build_call (call
, 0);
5351 gimple_call_set_lhs (stmt
, tid
);
5352 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5355 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5357 tree var
, ref
, new_var
, orig_var
;
5358 enum tree_code code
;
5359 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5361 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5364 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5365 if (TREE_CODE (var
) == MEM_REF
)
5367 var
= TREE_OPERAND (var
, 0);
5368 if (TREE_CODE (var
) == INDIRECT_REF
5369 || TREE_CODE (var
) == ADDR_EXPR
)
5370 var
= TREE_OPERAND (var
, 0);
5372 if (is_variable_sized (var
))
5374 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5375 var
= DECL_VALUE_EXPR (var
);
5376 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5377 var
= TREE_OPERAND (var
, 0);
5378 gcc_assert (DECL_P (var
));
5381 new_var
= lookup_decl (var
, ctx
);
5382 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5383 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5384 ref
= build_outer_var_ref (var
, ctx
);
5385 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5387 /* reduction(-:var) sums up the partial results, so it acts
5388 identically to reduction(+:var). */
5389 if (code
== MINUS_EXPR
)
5392 if (is_gimple_omp_oacc (ctx
->stmt
))
5394 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
5396 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
5398 else if (count
== 1)
5400 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5402 addr
= save_expr (addr
);
5403 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5404 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5405 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5406 gimplify_and_add (x
, stmt_seqp
);
5409 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5411 tree d
= OMP_CLAUSE_DECL (c
);
5412 tree type
= TREE_TYPE (d
);
5413 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5414 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5415 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5416 /* For ref build_outer_var_ref already performs this, so
5417 only new_var needs a dereference. */
5418 if (TREE_CODE (TREE_OPERAND (d
, 0)) == INDIRECT_REF
)
5420 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5421 gcc_assert (is_reference (var
) && var
== orig_var
);
5423 else if (TREE_CODE (TREE_OPERAND (d
, 0)) == ADDR_EXPR
)
5425 if (orig_var
== var
)
5427 new_var
= build_fold_addr_expr (new_var
);
5428 ref
= build_fold_addr_expr (ref
);
5433 gcc_assert (orig_var
== var
);
5434 if (is_reference (var
))
5435 ref
= build_fold_addr_expr (ref
);
5439 tree t
= maybe_lookup_decl (v
, ctx
);
5443 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5444 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5446 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5447 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5448 tree m
= create_tmp_var (ptype
, NULL
);
5449 gimplify_assign (m
, new_var
, stmt_seqp
);
5451 m
= create_tmp_var (ptype
, NULL
);
5452 gimplify_assign (m
, ref
, stmt_seqp
);
5454 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5455 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5456 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5457 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5458 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5459 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5460 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5462 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5463 tree decl_placeholder
5464 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5465 SET_DECL_VALUE_EXPR (placeholder
, out
);
5466 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5467 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5468 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5469 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5470 gimple_seq_add_seq (&sub_seq
,
5471 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5472 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5473 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5474 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5478 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5479 out
= unshare_expr (out
);
5480 gimplify_assign (out
, x
, &sub_seq
);
5482 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5483 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5484 gimple_seq_add_stmt (&sub_seq
, g
);
5485 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5486 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5487 gimple_seq_add_stmt (&sub_seq
, g
);
5488 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5489 build_int_cst (TREE_TYPE (i
), 1));
5490 gimple_seq_add_stmt (&sub_seq
, g
);
5491 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5492 gimple_seq_add_stmt (&sub_seq
, g
);
5493 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5495 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5497 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5499 if (is_reference (var
)
5500 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5502 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5503 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5504 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5505 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5506 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5507 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5508 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5512 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5513 ref
= build_outer_var_ref (var
, ctx
);
5514 gimplify_assign (ref
, x
, &sub_seq
);
5518 if (is_gimple_omp_oacc (ctx
->stmt
))
5521 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5523 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5525 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5527 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5529 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5533 /* Generate code to implement the COPYPRIVATE clauses. */
5536 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5541 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5543 tree var
, new_var
, ref
, x
;
5545 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5547 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5550 var
= OMP_CLAUSE_DECL (c
);
5551 by_ref
= use_pointer_for_field (var
, NULL
);
5553 ref
= build_sender_ref (var
, ctx
);
5554 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5557 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5558 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5560 gimplify_assign (ref
, x
, slist
);
5562 ref
= build_receiver_ref (var
, false, ctx
);
5565 ref
= fold_convert_loc (clause_loc
,
5566 build_pointer_type (TREE_TYPE (new_var
)),
5568 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5570 if (is_reference (var
))
5572 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5573 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5574 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5576 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5577 gimplify_and_add (x
, rlist
);
5582 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5583 and REDUCTION from the sender (aka parent) side. */
5586 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5590 int ignored_looptemp
= 0;
5591 bool is_taskloop
= false;
5593 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5594 by GOMP_taskloop. */
5595 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5597 ignored_looptemp
= 2;
5601 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5603 tree val
, ref
, x
, var
;
5604 bool by_ref
, do_in
= false, do_out
= false;
5605 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5607 switch (OMP_CLAUSE_CODE (c
))
5609 case OMP_CLAUSE_PRIVATE
:
5610 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
5613 case OMP_CLAUSE_FIRSTPRIVATE
:
5614 case OMP_CLAUSE_COPYIN
:
5615 case OMP_CLAUSE_LASTPRIVATE
:
5616 case OMP_CLAUSE_REDUCTION
:
5618 case OMP_CLAUSE_SHARED
:
5619 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5622 case OMP_CLAUSE__LOOPTEMP_
:
5623 if (ignored_looptemp
)
5633 val
= OMP_CLAUSE_DECL (c
);
5634 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5635 && TREE_CODE (val
) == MEM_REF
)
5637 val
= TREE_OPERAND (val
, 0);
5638 if (TREE_CODE (val
) == INDIRECT_REF
5639 || TREE_CODE (val
) == ADDR_EXPR
)
5640 val
= TREE_OPERAND (val
, 0);
5641 if (is_variable_sized (val
))
5645 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
5646 outer taskloop region. */
5647 omp_context
*ctx_for_o
= ctx
;
5649 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
5650 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5651 ctx_for_o
= ctx
->outer
;
5653 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
5655 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
5656 && is_global_var (var
))
5659 t
= omp_member_access_dummy_var (var
);
5662 var
= DECL_VALUE_EXPR (var
);
5663 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
5665 var
= unshare_and_remap (var
, t
, o
);
5667 var
= unshare_expr (var
);
5670 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
5672 /* Handle taskloop firstprivate/lastprivate, where the
5673 lastprivate on GIMPLE_OMP_TASK is represented as
5674 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
5675 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
5676 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
5677 if (use_pointer_for_field (val
, ctx
))
5678 var
= build_fold_addr_expr (var
);
5679 gimplify_assign (x
, var
, ilist
);
5680 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
5684 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
5685 || val
== OMP_CLAUSE_DECL (c
))
5686 && is_variable_sized (val
))
5688 by_ref
= use_pointer_for_field (val
, NULL
);
5690 switch (OMP_CLAUSE_CODE (c
))
5692 case OMP_CLAUSE_PRIVATE
:
5693 case OMP_CLAUSE_FIRSTPRIVATE
:
5694 case OMP_CLAUSE_COPYIN
:
5695 case OMP_CLAUSE__LOOPTEMP_
:
5699 case OMP_CLAUSE_LASTPRIVATE
:
5700 if (by_ref
|| is_reference (val
))
5702 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
5709 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
5714 case OMP_CLAUSE_REDUCTION
:
5716 if (val
== OMP_CLAUSE_DECL (c
))
5717 do_out
= !(by_ref
|| is_reference (val
));
5719 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
5728 ref
= build_sender_ref (val
, ctx
);
5729 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
5730 gimplify_assign (ref
, x
, ilist
);
5731 if (is_task_ctx (ctx
))
5732 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
5737 ref
= build_sender_ref (val
, ctx
);
5738 gimplify_assign (var
, ref
, olist
);
5743 /* Generate code to implement SHARED from the sender (aka parent)
5744 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
5745 list things that got automatically shared. */
5748 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
5750 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
5752 if (ctx
->record_type
== NULL
)
5755 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
5756 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
5758 ovar
= DECL_ABSTRACT_ORIGIN (f
);
5759 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
5762 nvar
= maybe_lookup_decl (ovar
, ctx
);
5763 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
5766 /* If CTX is a nested parallel directive. Find the immediately
5767 enclosing parallel or workshare construct that contains a
5768 mapping for OVAR. */
5769 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
5771 t
= omp_member_access_dummy_var (var
);
5774 var
= DECL_VALUE_EXPR (var
);
5775 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
5777 var
= unshare_and_remap (var
, t
, o
);
5779 var
= unshare_expr (var
);
5782 if (use_pointer_for_field (ovar
, ctx
))
5784 x
= build_sender_ref (ovar
, ctx
);
5785 var
= build_fold_addr_expr (var
);
5786 gimplify_assign (x
, var
, ilist
);
5790 x
= build_sender_ref (ovar
, ctx
);
5791 gimplify_assign (x
, var
, ilist
);
5793 if (!TREE_READONLY (var
)
5794 /* We don't need to receive a new reference to a result
5795 or parm decl. In fact we may not store to it as we will
5796 invalidate any pending RSO and generate wrong gimple
5798 && !((TREE_CODE (var
) == RESULT_DECL
5799 || TREE_CODE (var
) == PARM_DECL
)
5800 && DECL_BY_REFERENCE (var
)))
5802 x
= build_sender_ref (ovar
, ctx
);
5803 gimplify_assign (var
, x
, olist
);
5810 /* A convenience function to build an empty GIMPLE_COND with just the
5814 gimple_build_cond_empty (tree cond
)
5816 enum tree_code pred_code
;
5819 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
5820 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
5823 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
5826 /* Build the function calls to GOMP_parallel_start etc to actually
5827 generate the parallel operation. REGION is the parallel region
5828 being expanded. BB is the block where to insert the code. WS_ARGS
5829 will be set if this is a call to a combined parallel+workshare
5830 construct, it contains the list of additional arguments needed by
5831 the workshare construct. */
5834 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
5835 gomp_parallel
*entry_stmt
,
5836 vec
<tree
, va_gc
> *ws_args
)
5838 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
5839 gimple_stmt_iterator gsi
;
5841 enum built_in_function start_ix
;
5843 location_t clause_loc
;
5844 vec
<tree
, va_gc
> *args
;
5846 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5848 /* Determine what flavor of GOMP_parallel we will be
5850 start_ix
= BUILT_IN_GOMP_PARALLEL
;
5851 if (is_combined_parallel (region
))
5853 switch (region
->inner
->type
)
5855 case GIMPLE_OMP_FOR
:
5856 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
5857 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
5858 + (region
->inner
->sched_kind
5859 == OMP_CLAUSE_SCHEDULE_RUNTIME
5860 ? 3 : region
->inner
->sched_kind
));
5861 start_ix
= (enum built_in_function
)start_ix2
;
5863 case GIMPLE_OMP_SECTIONS
:
5864 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
5871 /* By default, the value of NUM_THREADS is zero (selected at run time)
5872 and there is no conditional. */
5874 val
= build_int_cst (unsigned_type_node
, 0);
5875 flags
= build_int_cst (unsigned_type_node
, 0);
5877 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5879 cond
= OMP_CLAUSE_IF_EXPR (c
);
5881 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
5884 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
5885 clause_loc
= OMP_CLAUSE_LOCATION (c
);
5888 clause_loc
= gimple_location (entry_stmt
);
5890 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
5892 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
5894 /* Ensure 'val' is of the correct type. */
5895 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
5897 /* If we found the clause 'if (cond)', build either
5898 (cond != 0) or (cond ? val : 1u). */
5901 cond
= gimple_boolify (cond
);
5903 if (integer_zerop (val
))
5904 val
= fold_build2_loc (clause_loc
,
5905 EQ_EXPR
, unsigned_type_node
, cond
,
5906 build_int_cst (TREE_TYPE (cond
), 0));
5909 basic_block cond_bb
, then_bb
, else_bb
;
5910 edge e
, e_then
, e_else
;
5911 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
5913 tmp_var
= create_tmp_var (TREE_TYPE (val
));
5914 if (gimple_in_ssa_p (cfun
))
5916 tmp_then
= make_ssa_name (tmp_var
);
5917 tmp_else
= make_ssa_name (tmp_var
);
5918 tmp_join
= make_ssa_name (tmp_var
);
5927 e
= split_block_after_labels (bb
);
5932 then_bb
= create_empty_bb (cond_bb
);
5933 else_bb
= create_empty_bb (then_bb
);
5934 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
5935 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
5937 stmt
= gimple_build_cond_empty (cond
);
5938 gsi
= gsi_start_bb (cond_bb
);
5939 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5941 gsi
= gsi_start_bb (then_bb
);
5942 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
5944 gsi
= gsi_start_bb (else_bb
);
5945 expand_omp_build_assign (&gsi
, tmp_else
,
5946 build_int_cst (unsigned_type_node
, 1),
5949 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
5950 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
5951 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
5952 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
5953 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
5954 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
5956 if (gimple_in_ssa_p (cfun
))
5958 gphi
*phi
= create_phi_node (tmp_join
, bb
);
5959 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
5960 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
5966 gsi
= gsi_start_bb (bb
);
5967 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
5968 false, GSI_CONTINUE_LINKING
);
5971 gsi
= gsi_last_bb (bb
);
5972 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5974 t1
= null_pointer_node
;
5976 t1
= build_fold_addr_expr (t
);
5977 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5979 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
5980 args
->quick_push (t2
);
5981 args
->quick_push (t1
);
5982 args
->quick_push (val
);
5984 args
->splice (*ws_args
);
5985 args
->quick_push (flags
);
5987 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
5988 builtin_decl_explicit (start_ix
), args
);
5990 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5991 false, GSI_CONTINUE_LINKING
);
5994 /* Insert a function call whose name is FUNC_NAME with the information from
5995 ENTRY_STMT into the basic_block BB. */
5998 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
5999 vec
<tree
, va_gc
> *ws_args
)
6002 gimple_stmt_iterator gsi
;
6003 vec
<tree
, va_gc
> *args
;
6005 gcc_assert (vec_safe_length (ws_args
) == 2);
6006 tree func_name
= (*ws_args
)[0];
6007 tree grain
= (*ws_args
)[1];
6009 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6010 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6011 gcc_assert (count
!= NULL_TREE
);
6012 count
= OMP_CLAUSE_OPERAND (count
, 0);
6014 gsi
= gsi_last_bb (bb
);
6015 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6017 t1
= null_pointer_node
;
6019 t1
= build_fold_addr_expr (t
);
6020 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6022 vec_alloc (args
, 4);
6023 args
->quick_push (t2
);
6024 args
->quick_push (t1
);
6025 args
->quick_push (count
);
6026 args
->quick_push (grain
);
6027 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6029 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6030 GSI_CONTINUE_LINKING
);
6033 /* Build the function call to GOMP_task to actually
6034 generate the task operation. BB is the block where to insert the code. */
6037 expand_task_call (struct omp_region
*region
, basic_block bb
,
6038 gomp_task
*entry_stmt
)
6041 gimple_stmt_iterator gsi
;
6042 location_t loc
= gimple_location (entry_stmt
);
6044 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6046 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6047 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6048 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6049 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6050 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6051 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6054 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6055 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6056 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6058 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6059 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6060 tree num_tasks
= NULL_TREE
;
6064 gimple
*g
= last_stmt (region
->outer
->entry
);
6065 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6066 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6067 struct omp_for_data fd
;
6068 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6069 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6070 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6071 OMP_CLAUSE__LOOPTEMP_
);
6072 startvar
= OMP_CLAUSE_DECL (startvar
);
6073 endvar
= OMP_CLAUSE_DECL (endvar
);
6074 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6075 if (fd
.loop
.cond_code
== LT_EXPR
)
6076 iflags
|= GOMP_TASK_FLAG_UP
;
6077 tree tclauses
= gimple_omp_for_clauses (g
);
6078 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6080 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6083 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6086 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6087 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6090 num_tasks
= integer_zero_node
;
6092 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6093 if (ifc
== NULL_TREE
)
6094 iflags
|= GOMP_TASK_FLAG_IF
;
6095 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6096 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6097 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6100 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6102 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6104 tree cond
= boolean_true_node
;
6109 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6110 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6111 build_int_cst (unsigned_type_node
,
6113 build_int_cst (unsigned_type_node
, 0));
6114 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6118 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6123 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6124 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6125 build_int_cst (unsigned_type_node
,
6126 GOMP_TASK_FLAG_FINAL
),
6127 build_int_cst (unsigned_type_node
, 0));
6128 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6131 depend
= OMP_CLAUSE_DECL (depend
);
6133 depend
= build_int_cst (ptr_type_node
, 0);
6135 priority
= fold_convert (integer_type_node
,
6136 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6138 priority
= integer_zero_node
;
6140 gsi
= gsi_last_bb (bb
);
6141 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6143 t2
= null_pointer_node
;
6145 t2
= build_fold_addr_expr_loc (loc
, t
);
6146 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6147 t
= gimple_omp_task_copy_fn (entry_stmt
);
6149 t3
= null_pointer_node
;
6151 t3
= build_fold_addr_expr_loc (loc
, t
);
6154 t
= build_call_expr (ull
6155 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6156 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6158 gimple_omp_task_arg_size (entry_stmt
),
6159 gimple_omp_task_arg_align (entry_stmt
), flags
,
6160 num_tasks
, priority
, startvar
, endvar
, step
);
6162 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6164 gimple_omp_task_arg_size (entry_stmt
),
6165 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6168 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6169 false, GSI_CONTINUE_LINKING
);
6173 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6174 catch handler and return it. This prevents programs from violating the
6175 structured block semantics with throws. */
6178 maybe_catch_exception (gimple_seq body
)
6183 if (!flag_exceptions
)
6186 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6187 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6189 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6191 g
= gimple_build_eh_must_not_throw (decl
);
6192 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6195 return gimple_seq_alloc_with_stmt (g
);
6198 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6201 vec2chain (vec
<tree
, va_gc
> *v
)
6203 tree chain
= NULL_TREE
, t
;
6206 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6208 DECL_CHAIN (t
) = chain
;
6216 /* Remove barriers in REGION->EXIT's block. Note that this is only
6217 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6218 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6219 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6223 remove_exit_barrier (struct omp_region
*region
)
6225 gimple_stmt_iterator gsi
;
6226 basic_block exit_bb
;
6230 int any_addressable_vars
= -1;
6232 exit_bb
= region
->exit
;
6234 /* If the parallel region doesn't return, we don't have REGION->EXIT
6239 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6240 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6241 statements that can appear in between are extremely limited -- no
6242 memory operations at all. Here, we allow nothing at all, so the
6243 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6244 gsi
= gsi_last_bb (exit_bb
);
6245 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6247 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6250 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6252 gsi
= gsi_last_bb (e
->src
);
6253 if (gsi_end_p (gsi
))
6255 stmt
= gsi_stmt (gsi
);
6256 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6257 && !gimple_omp_return_nowait_p (stmt
))
6259 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6260 in many cases. If there could be tasks queued, the barrier
6261 might be needed to let the tasks run before some local
6262 variable of the parallel that the task uses as shared
6263 runs out of scope. The task can be spawned either
6264 from within current function (this would be easy to check)
6265 or from some function it calls and gets passed an address
6266 of such a variable. */
6267 if (any_addressable_vars
< 0)
6269 gomp_parallel
*parallel_stmt
6270 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6271 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6272 tree local_decls
, block
, decl
;
6275 any_addressable_vars
= 0;
6276 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6277 if (TREE_ADDRESSABLE (decl
))
6279 any_addressable_vars
= 1;
6282 for (block
= gimple_block (stmt
);
6283 !any_addressable_vars
6285 && TREE_CODE (block
) == BLOCK
;
6286 block
= BLOCK_SUPERCONTEXT (block
))
6288 for (local_decls
= BLOCK_VARS (block
);
6290 local_decls
= DECL_CHAIN (local_decls
))
6291 if (TREE_ADDRESSABLE (local_decls
))
6293 any_addressable_vars
= 1;
6296 if (block
== gimple_block (parallel_stmt
))
6300 if (!any_addressable_vars
)
6301 gimple_omp_return_set_nowait (stmt
);
6307 remove_exit_barriers (struct omp_region
*region
)
6309 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6310 remove_exit_barrier (region
);
6314 region
= region
->inner
;
6315 remove_exit_barriers (region
);
6316 while (region
->next
)
6318 region
= region
->next
;
6319 remove_exit_barriers (region
);
6324 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6325 calls. These can't be declared as const functions, but
6326 within one parallel body they are constant, so they can be
6327 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6328 which are declared const. Similarly for task body, except
6329 that in untied task omp_get_thread_num () can change at any task
6330 scheduling point. */
6333 optimize_omp_library_calls (gimple
*entry_stmt
)
6336 gimple_stmt_iterator gsi
;
6337 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6338 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6339 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6340 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6341 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6342 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6343 OMP_CLAUSE_UNTIED
) != NULL
);
6345 FOR_EACH_BB_FN (bb
, cfun
)
6346 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6348 gimple
*call
= gsi_stmt (gsi
);
6351 if (is_gimple_call (call
)
6352 && (decl
= gimple_call_fndecl (call
))
6353 && DECL_EXTERNAL (decl
)
6354 && TREE_PUBLIC (decl
)
6355 && DECL_INITIAL (decl
) == NULL
)
6359 if (DECL_NAME (decl
) == thr_num_id
)
6361 /* In #pragma omp task untied omp_get_thread_num () can change
6362 during the execution of the task region. */
6365 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6367 else if (DECL_NAME (decl
) == num_thr_id
)
6368 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6372 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6373 || gimple_call_num_args (call
) != 0)
6376 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
6379 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
6380 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
6381 TREE_TYPE (TREE_TYPE (built_in
))))
6384 gimple_call_set_fndecl (call
, built_in
);
6389 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
6393 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
6397 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
6398 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
6401 if (TREE_CODE (t
) == ADDR_EXPR
)
6402 recompute_tree_invariant_for_addr_expr (t
);
6404 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
6408 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
6411 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
6414 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
6415 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
6416 !after
, after
? GSI_CONTINUE_LINKING
6418 gimple
*stmt
= gimple_build_assign (to
, from
);
6420 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
6422 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
6423 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
6424 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
6426 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
6427 gimple_regimplify_operands (stmt
, &gsi
);
6431 /* Expand the OpenMP parallel or task directive starting at REGION. */
6434 expand_omp_taskreg (struct omp_region
*region
)
6436 basic_block entry_bb
, exit_bb
, new_bb
;
6437 struct function
*child_cfun
;
6438 tree child_fn
, block
, t
;
6439 gimple_stmt_iterator gsi
;
6440 gimple
*entry_stmt
, *stmt
;
6442 vec
<tree
, va_gc
> *ws_args
;
6444 entry_stmt
= last_stmt (region
->entry
);
6445 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
6446 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
6448 entry_bb
= region
->entry
;
6449 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
6450 exit_bb
= region
->cont
;
6452 exit_bb
= region
->exit
;
6456 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
6457 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
6458 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
6461 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
6462 and the inner statement contains the name of the built-in function
6464 ws_args
= region
->inner
->ws_args
;
6465 else if (is_combined_parallel (region
))
6466 ws_args
= region
->ws_args
;
6470 if (child_cfun
->cfg
)
6472 /* Due to inlining, it may happen that we have already outlined
6473 the region, in which case all we need to do is make the
6474 sub-graph unreachable and emit the parallel call. */
6475 edge entry_succ_e
, exit_succ_e
;
6477 entry_succ_e
= single_succ_edge (entry_bb
);
6479 gsi
= gsi_last_bb (entry_bb
);
6480 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
6481 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
6482 gsi_remove (&gsi
, true);
6487 exit_succ_e
= single_succ_edge (exit_bb
);
6488 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
6490 remove_edge_and_dominated_blocks (entry_succ_e
);
6494 unsigned srcidx
, dstidx
, num
;
6496 /* If the parallel region needs data sent from the parent
6497 function, then the very first statement (except possible
6498 tree profile counter updates) of the parallel body
6499 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
6500 &.OMP_DATA_O is passed as an argument to the child function,
6501 we need to replace it with the argument as seen by the child
6504 In most cases, this will end up being the identity assignment
6505 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
6506 a function call that has been inlined, the original PARM_DECL
6507 .OMP_DATA_I may have been converted into a different local
6508 variable. In which case, we need to keep the assignment. */
6509 if (gimple_omp_taskreg_data_arg (entry_stmt
))
6511 basic_block entry_succ_bb
6512 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
6513 : FALLTHRU_EDGE (entry_bb
)->dest
;
6515 gimple
*parcopy_stmt
= NULL
;
6517 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
6521 gcc_assert (!gsi_end_p (gsi
));
6522 stmt
= gsi_stmt (gsi
);
6523 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
6526 if (gimple_num_ops (stmt
) == 2)
6528 tree arg
= gimple_assign_rhs1 (stmt
);
6530 /* We're ignore the subcode because we're
6531 effectively doing a STRIP_NOPS. */
6533 if (TREE_CODE (arg
) == ADDR_EXPR
6534 && TREE_OPERAND (arg
, 0)
6535 == gimple_omp_taskreg_data_arg (entry_stmt
))
6537 parcopy_stmt
= stmt
;
6543 gcc_assert (parcopy_stmt
!= NULL
);
6544 arg
= DECL_ARGUMENTS (child_fn
);
6546 if (!gimple_in_ssa_p (cfun
))
6548 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
6549 gsi_remove (&gsi
, true);
6552 /* ?? Is setting the subcode really necessary ?? */
6553 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
6554 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6559 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
6560 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
6561 /* We'd like to set the rhs to the default def in the child_fn,
6562 but it's too early to create ssa names in the child_fn.
6563 Instead, we set the rhs to the parm. In
6564 move_sese_region_to_fn, we introduce a default def for the
6565 parm, map the parm to it's default def, and once we encounter
6566 this stmt, replace the parm with the default def. */
6567 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6568 update_stmt (parcopy_stmt
);
6572 /* Declare local variables needed in CHILD_CFUN. */
6573 block
= DECL_INITIAL (child_fn
);
6574 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
6575 /* The gimplifier could record temporaries in parallel/task block
6576 rather than in containing function's local_decls chain,
6577 which would mean cgraph missed finalizing them. Do it now. */
6578 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
6579 if (TREE_CODE (t
) == VAR_DECL
6581 && !DECL_EXTERNAL (t
))
6582 varpool_node::finalize_decl (t
);
6583 DECL_SAVED_TREE (child_fn
) = NULL
;
6584 /* We'll create a CFG for child_fn, so no gimple body is needed. */
6585 gimple_set_body (child_fn
, NULL
);
6586 TREE_USED (block
) = 1;
6588 /* Reset DECL_CONTEXT on function arguments. */
6589 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
6590 DECL_CONTEXT (t
) = child_fn
;
6592 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
6593 so that it can be moved to the child function. */
6594 gsi
= gsi_last_bb (entry_bb
);
6595 stmt
= gsi_stmt (gsi
);
6596 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
6597 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
6598 e
= split_block (entry_bb
, stmt
);
6599 gsi_remove (&gsi
, true);
6602 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
6603 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6606 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
6607 gcc_assert (e2
->dest
== region
->exit
);
6608 remove_edge (BRANCH_EDGE (entry_bb
));
6609 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
6610 gsi
= gsi_last_bb (region
->exit
);
6611 gcc_assert (!gsi_end_p (gsi
)
6612 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6613 gsi_remove (&gsi
, true);
6616 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
6619 gsi
= gsi_last_bb (exit_bb
);
6620 gcc_assert (!gsi_end_p (gsi
)
6621 && (gimple_code (gsi_stmt (gsi
))
6622 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
6623 stmt
= gimple_build_return (NULL
);
6624 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
6625 gsi_remove (&gsi
, true);
6628 /* Move the parallel region into CHILD_CFUN. */
6630 if (gimple_in_ssa_p (cfun
))
6632 init_tree_ssa (child_cfun
);
6633 init_ssa_operands (child_cfun
);
6634 child_cfun
->gimple_df
->in_ssa_p
= true;
6638 block
= gimple_block (entry_stmt
);
6640 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
6642 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
6645 basic_block dest_bb
= e2
->dest
;
6647 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
6649 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
6651 /* When the OMP expansion process cannot guarantee an up-to-date
6652 loop tree arrange for the child function to fixup loops. */
6653 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
6654 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
6656 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
6657 num
= vec_safe_length (child_cfun
->local_decls
);
6658 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
6660 t
= (*child_cfun
->local_decls
)[srcidx
];
6661 if (DECL_CONTEXT (t
) == cfun
->decl
)
6663 if (srcidx
!= dstidx
)
6664 (*child_cfun
->local_decls
)[dstidx
] = t
;
6668 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
6670 /* Inform the callgraph about the new function. */
6671 child_cfun
->curr_properties
= cfun
->curr_properties
;
6672 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
6673 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
6674 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
6675 node
->parallelized_function
= 1;
6676 cgraph_node::add_new_function (child_fn
, true);
6678 /* Fix the callgraph edges for child_cfun. Those for cfun will be
6679 fixed in a following pass. */
6680 push_cfun (child_cfun
);
6682 optimize_omp_library_calls (entry_stmt
);
6683 cgraph_edge::rebuild_edges ();
6685 /* Some EH regions might become dead, see PR34608. If
6686 pass_cleanup_cfg isn't the first pass to happen with the
6687 new child, these dead EH edges might cause problems.
6688 Clean them up now. */
6689 if (flag_exceptions
)
6692 bool changed
= false;
6694 FOR_EACH_BB_FN (bb
, cfun
)
6695 changed
|= gimple_purge_dead_eh_edges (bb
);
6697 cleanup_tree_cfg ();
6699 if (gimple_in_ssa_p (cfun
))
6700 update_ssa (TODO_update_ssa
);
6701 #ifdef ENABLE_CHECKING
6702 if (!loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
6703 verify_loop_structure ();
6708 /* Emit a library call to launch the children threads. */
6710 expand_cilk_for_call (new_bb
,
6711 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
6712 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
6713 expand_parallel_call (region
, new_bb
,
6714 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
6716 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
6717 if (gimple_in_ssa_p (cfun
))
6718 update_ssa (TODO_update_ssa_only_virtuals
);
6722 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
6723 of the combined collapse > 1 loop constructs, generate code like:
6724 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
6729 count3 = (adj + N32 - N31) / STEP3;
6730 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
6735 count2 = (adj + N22 - N21) / STEP2;
6736 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
6741 count1 = (adj + N12 - N11) / STEP1;
6742 count = count1 * count2 * count3;
6743 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
6745 and set ZERO_ITER_BB to that bb. If this isn't the outermost
6746 of the combined loop constructs, just initialize COUNTS array
6747 from the _looptemp_ clauses. */
6749 /* NOTE: It *could* be better to moosh all of the BBs together,
6750 creating one larger BB with all the computation and the unexpected
6751 jump at the end. I.e.
6753 bool zero3, zero2, zero1, zero;
6756 count3 = (N32 - N31) /[cl] STEP3;
6758 count2 = (N22 - N21) /[cl] STEP2;
6760 count1 = (N12 - N11) /[cl] STEP1;
6761 zero = zero3 || zero2 || zero1;
6762 count = count1 * count2 * count3;
6763 if (__builtin_expect(zero, false)) goto zero_iter_bb;
6765 After all, we expect the zero=false, and thus we expect to have to
6766 evaluate all of the comparison expressions, so short-circuiting
6767 oughtn't be a win. Since the condition isn't protecting a
6768 denominator, we're not concerned about divide-by-zero, so we can
6769 fully evaluate count even if a numerator turned out to be wrong.
6771 It seems like putting this all together would create much better
6772 scheduling opportunities, and less pressure on the chip's branch
6776 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
6777 basic_block
&entry_bb
, tree
*counts
,
6778 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
6779 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
6780 basic_block
&l2_dom_bb
)
6782 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
6786 /* Collapsed loops need work for expansion into SSA form. */
6787 gcc_assert (!gimple_in_ssa_p (cfun
));
6789 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
6790 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
6792 gcc_assert (fd
->ordered
== 0);
6793 /* First two _looptemp_ clauses are for istart/iend, counts[0]
6794 isn't supposed to be handled, as the inner loop doesn't
6796 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6797 OMP_CLAUSE__LOOPTEMP_
);
6798 gcc_assert (innerc
);
6799 for (i
= 0; i
< fd
->collapse
; i
++)
6801 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6802 OMP_CLAUSE__LOOPTEMP_
);
6803 gcc_assert (innerc
);
6805 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
6807 counts
[0] = NULL_TREE
;
6812 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
6814 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6815 counts
[i
] = NULL_TREE
;
6816 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
6817 fold_convert (itype
, fd
->loops
[i
].n1
),
6818 fold_convert (itype
, fd
->loops
[i
].n2
));
6819 if (t
&& integer_zerop (t
))
6821 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
6822 counts
[i
] = build_int_cst (type
, 0);
6826 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
6828 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6830 if (i
>= fd
->collapse
&& counts
[i
])
6832 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
6833 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
6834 fold_convert (itype
, fd
->loops
[i
].n1
),
6835 fold_convert (itype
, fd
->loops
[i
].n2
)))
6836 == NULL_TREE
|| !integer_onep (t
)))
6840 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
6841 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
6842 true, GSI_SAME_STMT
);
6843 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
6844 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
6845 true, GSI_SAME_STMT
);
6846 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
6847 NULL_TREE
, NULL_TREE
);
6848 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
6849 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6850 expand_omp_regimplify_p
, NULL
, NULL
)
6851 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6852 expand_omp_regimplify_p
, NULL
, NULL
))
6854 *gsi
= gsi_for_stmt (cond_stmt
);
6855 gimple_regimplify_operands (cond_stmt
, gsi
);
6857 e
= split_block (entry_bb
, cond_stmt
);
6858 basic_block
&zero_iter_bb
6859 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
6860 int &first_zero_iter
6861 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
6862 if (zero_iter_bb
== NULL
)
6864 gassign
*assign_stmt
;
6865 first_zero_iter
= i
;
6866 zero_iter_bb
= create_empty_bb (entry_bb
);
6867 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
6868 *gsi
= gsi_after_labels (zero_iter_bb
);
6869 if (i
< fd
->collapse
)
6870 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
6871 build_zero_cst (type
));
6874 counts
[i
] = create_tmp_reg (type
, ".count");
6876 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
6878 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
6879 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
6882 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
6883 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6884 e
->flags
= EDGE_TRUE_VALUE
;
6885 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
6886 if (l2_dom_bb
== NULL
)
6887 l2_dom_bb
= entry_bb
;
6889 *gsi
= gsi_last_bb (entry_bb
);
6892 if (POINTER_TYPE_P (itype
))
6893 itype
= signed_type_for (itype
);
6894 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
6896 t
= fold_build2 (PLUS_EXPR
, itype
,
6897 fold_convert (itype
, fd
->loops
[i
].step
), t
);
6898 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
6899 fold_convert (itype
, fd
->loops
[i
].n2
));
6900 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
6901 fold_convert (itype
, fd
->loops
[i
].n1
));
6902 /* ?? We could probably use CEIL_DIV_EXPR instead of
6903 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
6904 generate the same code in the end because generically we
6905 don't know that the values involved must be negative for
6907 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
6908 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6909 fold_build1 (NEGATE_EXPR
, itype
, t
),
6910 fold_build1 (NEGATE_EXPR
, itype
,
6911 fold_convert (itype
,
6912 fd
->loops
[i
].step
)));
6914 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
6915 fold_convert (itype
, fd
->loops
[i
].step
));
6916 t
= fold_convert (type
, t
);
6917 if (TREE_CODE (t
) == INTEGER_CST
)
6921 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
6922 counts
[i
] = create_tmp_reg (type
, ".count");
6923 expand_omp_build_assign (gsi
, counts
[i
], t
);
6925 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
6930 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
6931 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
6937 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
6939 V3 = N31 + (T % count3) * STEP3;
6941 V2 = N21 + (T % count2) * STEP2;
6943 V1 = N11 + T * STEP1;
6944 if this loop doesn't have an inner loop construct combined with it.
6945 If it does have an inner loop construct combined with it and the
6946 iteration count isn't known constant, store values from counts array
6947 into its _looptemp_ temporaries instead. */
6950 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
6951 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
6954 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6956 /* If fd->loop.n2 is constant, then no propagation of the counts
6957 is needed, they are constant. */
6958 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
6961 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
6962 ? gimple_omp_taskreg_clauses (inner_stmt
)
6963 : gimple_omp_for_clauses (inner_stmt
);
6964 /* First two _looptemp_ clauses are for istart/iend, counts[0]
6965 isn't supposed to be handled, as the inner loop doesn't
6967 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6968 gcc_assert (innerc
);
6969 for (i
= 0; i
< fd
->collapse
; i
++)
6971 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6972 OMP_CLAUSE__LOOPTEMP_
);
6973 gcc_assert (innerc
);
6976 tree tem
= OMP_CLAUSE_DECL (innerc
);
6977 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
6978 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
6979 false, GSI_CONTINUE_LINKING
);
6980 gassign
*stmt
= gimple_build_assign (tem
, t
);
6981 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
6987 tree type
= TREE_TYPE (fd
->loop
.v
);
6988 tree tem
= create_tmp_reg (type
, ".tem");
6989 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
6990 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
6992 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
6994 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
6996 if (POINTER_TYPE_P (vtype
))
6997 itype
= signed_type_for (vtype
);
6999 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7002 t
= fold_convert (itype
, t
);
7003 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7004 fold_convert (itype
, fd
->loops
[i
].step
));
7005 if (POINTER_TYPE_P (vtype
))
7006 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7008 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7009 t
= force_gimple_operand_gsi (gsi
, t
,
7010 DECL_P (fd
->loops
[i
].v
)
7011 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7013 GSI_CONTINUE_LINKING
);
7014 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7015 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7018 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7019 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7020 false, GSI_CONTINUE_LINKING
);
7021 stmt
= gimple_build_assign (tem
, t
);
7022 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7028 /* Helper function for expand_omp_for_*. Generate code like:
7031 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7035 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7042 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7043 basic_block body_bb
)
7045 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7047 gimple_stmt_iterator gsi
;
7053 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7055 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7057 bb
= create_empty_bb (last_bb
);
7058 add_bb_to_loop (bb
, last_bb
->loop_father
);
7059 gsi
= gsi_start_bb (bb
);
7061 if (i
< fd
->collapse
- 1)
7063 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7064 e
->probability
= REG_BR_PROB_BASE
/ 8;
7066 t
= fd
->loops
[i
+ 1].n1
;
7067 t
= force_gimple_operand_gsi (&gsi
, t
,
7068 DECL_P (fd
->loops
[i
+ 1].v
)
7069 && TREE_ADDRESSABLE (fd
->loops
[i
7072 GSI_CONTINUE_LINKING
);
7073 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7074 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7079 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7081 if (POINTER_TYPE_P (vtype
))
7082 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7084 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7085 t
= force_gimple_operand_gsi (&gsi
, t
,
7086 DECL_P (fd
->loops
[i
].v
)
7087 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7088 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7089 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7090 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7094 t
= fd
->loops
[i
].n2
;
7095 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7096 false, GSI_CONTINUE_LINKING
);
7097 tree v
= fd
->loops
[i
].v
;
7098 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7099 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7100 false, GSI_CONTINUE_LINKING
);
7101 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7102 stmt
= gimple_build_cond_empty (t
);
7103 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7104 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7105 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7108 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7116 /* Expand #pragma omp ordered depend(source). */
7119 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7120 tree
*counts
, location_t loc
)
7122 enum built_in_function source_ix
7123 = fd
->iter_type
== long_integer_type_node
7124 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7126 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7127 build_fold_addr_expr (counts
[fd
->ordered
]));
7128 gimple_set_location (g
, loc
);
7129 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7132 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7135 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7136 tree
*counts
, tree c
, location_t loc
)
7138 auto_vec
<tree
, 10> args
;
7139 enum built_in_function sink_ix
7140 = fd
->iter_type
== long_integer_type_node
7141 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7142 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7144 gimple_stmt_iterator gsi2
= *gsi
;
7145 bool warned_step
= false;
7147 for (i
= 0; i
< fd
->ordered
; i
++)
7149 off
= TREE_PURPOSE (deps
);
7150 if (!integer_zerop (off
))
7152 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7153 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7154 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7155 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7156 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7157 "lexically later iteration");
7160 deps
= TREE_CHAIN (deps
);
7162 /* If all offsets corresponding to the collapsed loops are zero,
7163 this depend clause can be ignored. FIXME: but there is still a
7164 flush needed. We need to emit one __sync_synchronize () for it
7165 though (perhaps conditionally)? Solve this together with the
7166 conservative dependence folding optimization.
7167 if (i >= fd->collapse)
7170 deps
= OMP_CLAUSE_DECL (c
);
7172 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7173 edge e2
= split_block_after_labels (e1
->dest
);
7175 *gsi
= gsi_after_labels (e1
->dest
);
7176 for (i
= 0; i
< fd
->ordered
; i
++)
7178 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7179 if (POINTER_TYPE_P (itype
))
7182 deps
= TREE_CHAIN (deps
);
7183 off
= TREE_PURPOSE (deps
);
7184 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7186 if (integer_zerop (off
))
7187 t
= boolean_true_node
;
7191 tree co
= fold_convert_loc (loc
, itype
, off
);
7192 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7194 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7195 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7196 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7197 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7200 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7201 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7202 fd
->loops
[i
].v
, co
);
7204 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7205 fd
->loops
[i
].v
, co
);
7206 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7208 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7209 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7212 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7215 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7216 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7219 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
7223 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
7227 off
= fold_convert_loc (loc
, itype
, off
);
7229 if (fd
->loops
[i
].cond_code
== LT_EXPR
7230 ? !integer_onep (fd
->loops
[i
].step
)
7231 : !integer_minus_onep (fd
->loops
[i
].step
))
7233 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7234 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
7235 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7238 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
7239 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
7240 build_int_cst (itype
, 0));
7241 if (integer_zerop (t
) && !warned_step
)
7243 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
7244 "in the iteration space");
7247 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
7251 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7257 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7258 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7259 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
7261 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7262 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
7263 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7266 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
7267 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7268 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
7269 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
7270 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7273 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
7275 if (i
< fd
->collapse
- 1)
7277 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
7282 off
= unshare_expr (off
);
7283 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
7284 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
7285 true, GSI_SAME_STMT
);
7288 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
7289 gimple_set_location (g
, loc
);
7290 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7292 *gsi
= gsi_last_bb (e1
->src
);
7293 cond
= unshare_expr (cond
);
7294 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
7295 GSI_CONTINUE_LINKING
);
7296 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
7297 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
7298 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7299 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7300 e1
->flags
= EDGE_TRUE_VALUE
;
7301 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
7303 *gsi
= gsi_after_labels (e2
->dest
);
7306 /* Expand all #pragma omp ordered depend(source) and
7307 #pragma omp ordered depend(sink:...) constructs in the current
7308 #pragma omp for ordered(n) region. */
7311 expand_omp_ordered_source_sink (struct omp_region
*region
,
7312 struct omp_for_data
*fd
, tree
*counts
,
7313 basic_block cont_bb
)
7315 struct omp_region
*inner
;
7317 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
7318 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
7319 counts
[i
] = NULL_TREE
;
7320 else if (i
>= fd
->collapse
&& !cont_bb
)
7321 counts
[i
] = build_zero_cst (fd
->iter_type
);
7322 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
7323 && integer_onep (fd
->loops
[i
].step
))
7324 counts
[i
] = NULL_TREE
;
7326 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
7328 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
7329 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
7330 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
7332 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
7333 if (inner
->type
== GIMPLE_OMP_ORDERED
)
7335 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
7336 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
7337 location_t loc
= gimple_location (ord_stmt
);
7339 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7340 c
; c
= OMP_CLAUSE_CHAIN (c
))
7341 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
7344 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
7345 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7346 c
; c
= OMP_CLAUSE_CHAIN (c
))
7347 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
7348 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
7349 gsi_remove (&gsi
, true);
7353 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
7357 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
7358 basic_block cont_bb
, basic_block body_bb
,
7359 bool ordered_lastprivate
)
7361 if (fd
->ordered
== fd
->collapse
)
7366 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7367 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7369 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7370 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
7371 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
7372 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7373 size_int (i
- fd
->collapse
+ 1),
7374 NULL_TREE
, NULL_TREE
);
7375 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7380 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
7382 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
7383 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7384 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7385 fold_convert (type
, fd
->loops
[i
].n1
));
7387 expand_omp_build_assign (&gsi
, counts
[i
],
7388 build_zero_cst (fd
->iter_type
));
7389 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7390 size_int (i
- fd
->collapse
+ 1),
7391 NULL_TREE
, NULL_TREE
);
7392 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7393 if (!gsi_end_p (gsi
))
7396 gsi
= gsi_last_bb (body_bb
);
7397 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
7398 basic_block new_body
= e1
->dest
;
7399 if (body_bb
== cont_bb
)
7402 basic_block new_header
;
7403 if (EDGE_COUNT (cont_bb
->preds
) > 0)
7405 gsi
= gsi_last_bb (cont_bb
);
7406 if (POINTER_TYPE_P (type
))
7407 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
7408 fold_convert (sizetype
,
7409 fd
->loops
[i
].step
));
7411 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
7412 fold_convert (type
, fd
->loops
[i
].step
));
7413 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7416 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
7417 build_int_cst (fd
->iter_type
, 1));
7418 expand_omp_build_assign (&gsi
, counts
[i
], t
);
7423 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7424 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7425 t
= fold_convert (fd
->iter_type
, t
);
7426 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7427 true, GSI_SAME_STMT
);
7429 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7430 size_int (i
- fd
->collapse
+ 1),
7431 NULL_TREE
, NULL_TREE
);
7432 expand_omp_build_assign (&gsi
, aref
, t
);
7434 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
7435 new_header
= e2
->dest
;
7438 new_header
= cont_bb
;
7439 gsi
= gsi_after_labels (new_header
);
7440 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
7441 true, GSI_SAME_STMT
);
7443 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
7444 true, NULL_TREE
, true, GSI_SAME_STMT
);
7445 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
7446 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
7447 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
7450 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
7451 e3
->flags
= EDGE_FALSE_VALUE
;
7452 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7453 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
7454 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7456 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
7457 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
7461 struct loop
*loop
= alloc_loop ();
7462 loop
->header
= new_header
;
7463 loop
->latch
= e2
->src
;
7464 add_loop (loop
, body_bb
->loop_father
);
7468 /* If there are any lastprivate clauses and it is possible some loops
7469 might have zero iterations, ensure all the decls are initialized,
7470 otherwise we could crash evaluating C++ class iterators with lastprivate
7472 bool need_inits
= false;
7473 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
7476 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7477 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7478 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7479 fold_convert (type
, fd
->loops
[i
].n1
));
7483 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7484 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
7486 fold_convert (type
, fd
->loops
[i
].n1
),
7487 fold_convert (type
, fd
->loops
[i
].n2
));
7488 if (!integer_onep (this_cond
))
7496 /* A subroutine of expand_omp_for. Generate code for a parallel
7497 loop with any schedule. Given parameters:
7499 for (V = N1; V cond N2; V += STEP) BODY;
7501 where COND is "<" or ">", we generate pseudocode
7503 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
7504 if (more) goto L0; else goto L3;
7511 if (V cond iend) goto L1; else goto L2;
7513 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
7516 If this is a combined omp parallel loop, instead of the call to
7517 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
7518 If this is gimple_omp_for_combined_p loop, then instead of assigning
7519 V and iend in L0 we assign the first two _looptemp_ clause decls of the
7520 inner GIMPLE_OMP_FOR and V += STEP; and
7521 if (V cond iend) goto L1; else goto L2; are removed.
7523 For collapsed loops, given parameters:
7525 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7526 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7527 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7530 we generate pseudocode
7532 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
7537 count3 = (adj + N32 - N31) / STEP3;
7538 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
7543 count2 = (adj + N22 - N21) / STEP2;
7544 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
7549 count1 = (adj + N12 - N11) / STEP1;
7550 count = count1 * count2 * count3;
7555 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
7556 if (more) goto L0; else goto L3;
7560 V3 = N31 + (T % count3) * STEP3;
7562 V2 = N21 + (T % count2) * STEP2;
7564 V1 = N11 + T * STEP1;
7569 if (V < iend) goto L10; else goto L2;
7572 if (V3 cond3 N32) goto L1; else goto L11;
7576 if (V2 cond2 N22) goto L1; else goto L12;
7582 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
7588 expand_omp_for_generic (struct omp_region
*region
,
7589 struct omp_for_data
*fd
,
7590 enum built_in_function start_fn
,
7591 enum built_in_function next_fn
,
7594 tree type
, istart0
, iend0
, iend
;
7595 tree t
, vmain
, vback
, bias
= NULL_TREE
;
7596 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
7597 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
7598 gimple_stmt_iterator gsi
;
7599 gassign
*assign_stmt
;
7600 bool in_combined_parallel
= is_combined_parallel (region
);
7601 bool broken_loop
= region
->cont
== NULL
;
7603 tree
*counts
= NULL
;
7605 bool ordered_lastprivate
= false;
7607 gcc_assert (!broken_loop
|| !in_combined_parallel
);
7608 gcc_assert (fd
->iter_type
== long_integer_type_node
7609 || !in_combined_parallel
);
7611 entry_bb
= region
->entry
;
7612 cont_bb
= region
->cont
;
7614 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7615 gcc_assert (broken_loop
7616 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7617 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
7618 l1_bb
= single_succ (l0_bb
);
7621 l2_bb
= create_empty_bb (cont_bb
);
7622 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
7623 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
7625 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7629 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7630 exit_bb
= region
->exit
;
7632 gsi
= gsi_last_bb (entry_bb
);
7634 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7636 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
7637 OMP_CLAUSE_LASTPRIVATE
))
7638 ordered_lastprivate
= false;
7639 if (fd
->collapse
> 1 || fd
->ordered
)
7641 int first_zero_iter1
= -1, first_zero_iter2
= -1;
7642 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
7644 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
7645 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7646 zero_iter1_bb
, first_zero_iter1
,
7647 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
7651 /* Some counts[i] vars might be uninitialized if
7652 some loop has zero iterations. But the body shouldn't
7653 be executed in that case, so just avoid uninit warnings. */
7654 for (i
= first_zero_iter1
;
7655 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7656 if (SSA_VAR_P (counts
[i
]))
7657 TREE_NO_WARNING (counts
[i
]) = 1;
7659 e
= split_block (entry_bb
, gsi_stmt (gsi
));
7661 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
7662 gsi
= gsi_last_bb (entry_bb
);
7663 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
7664 get_immediate_dominator (CDI_DOMINATORS
,
7669 /* Some counts[i] vars might be uninitialized if
7670 some loop has zero iterations. But the body shouldn't
7671 be executed in that case, so just avoid uninit warnings. */
7672 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
7673 if (SSA_VAR_P (counts
[i
]))
7674 TREE_NO_WARNING (counts
[i
]) = 1;
7676 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
7680 e
= split_block (entry_bb
, gsi_stmt (gsi
));
7682 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
7683 gsi
= gsi_last_bb (entry_bb
);
7684 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
7685 get_immediate_dominator
7686 (CDI_DOMINATORS
, zero_iter2_bb
));
7689 if (fd
->collapse
== 1)
7691 counts
[0] = fd
->loop
.n2
;
7692 fd
->loop
= fd
->loops
[0];
7696 type
= TREE_TYPE (fd
->loop
.v
);
7697 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
7698 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
7699 TREE_ADDRESSABLE (istart0
) = 1;
7700 TREE_ADDRESSABLE (iend0
) = 1;
7702 /* See if we need to bias by LLONG_MIN. */
7703 if (fd
->iter_type
== long_long_unsigned_type_node
7704 && TREE_CODE (type
) == INTEGER_TYPE
7705 && !TYPE_UNSIGNED (type
)
7706 && fd
->ordered
== 0)
7710 if (fd
->loop
.cond_code
== LT_EXPR
)
7713 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
7717 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
7720 if (TREE_CODE (n1
) != INTEGER_CST
7721 || TREE_CODE (n2
) != INTEGER_CST
7722 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
7723 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
7726 gimple_stmt_iterator gsif
= gsi
;
7729 tree arr
= NULL_TREE
;
7730 if (in_combined_parallel
)
7732 gcc_assert (fd
->ordered
== 0);
7733 /* In a combined parallel loop, emit a call to
7734 GOMP_loop_foo_next. */
7735 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
7736 build_fold_addr_expr (istart0
),
7737 build_fold_addr_expr (iend0
));
7741 tree t0
, t1
, t2
, t3
, t4
;
7742 /* If this is not a combined parallel loop, emit a call to
7743 GOMP_loop_foo_start in ENTRY_BB. */
7744 t4
= build_fold_addr_expr (iend0
);
7745 t3
= build_fold_addr_expr (istart0
);
7748 t0
= build_int_cst (unsigned_type_node
,
7749 fd
->ordered
- fd
->collapse
+ 1);
7750 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
7752 - fd
->collapse
+ 1),
7754 DECL_NAMELESS (arr
) = 1;
7755 TREE_ADDRESSABLE (arr
) = 1;
7756 TREE_STATIC (arr
) = 1;
7757 vec
<constructor_elt
, va_gc
> *v
;
7758 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
7761 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
7764 if (idx
== 0 && fd
->collapse
> 1)
7767 c
= counts
[idx
+ fd
->collapse
- 1];
7768 tree purpose
= size_int (idx
);
7769 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
7770 if (TREE_CODE (c
) != INTEGER_CST
)
7771 TREE_STATIC (arr
) = 0;
7774 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
7775 if (!TREE_STATIC (arr
))
7776 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
7777 void_type_node
, arr
),
7778 true, NULL_TREE
, true, GSI_SAME_STMT
);
7779 t1
= build_fold_addr_expr (arr
);
7784 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
7787 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7790 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7791 OMP_CLAUSE__LOOPTEMP_
);
7792 gcc_assert (innerc
);
7793 t0
= OMP_CLAUSE_DECL (innerc
);
7794 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7795 OMP_CLAUSE__LOOPTEMP_
);
7796 gcc_assert (innerc
);
7797 t1
= OMP_CLAUSE_DECL (innerc
);
7799 if (POINTER_TYPE_P (TREE_TYPE (t0
))
7800 && TYPE_PRECISION (TREE_TYPE (t0
))
7801 != TYPE_PRECISION (fd
->iter_type
))
7803 /* Avoid casting pointers to integer of a different size. */
7804 tree itype
= signed_type_for (type
);
7805 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
7806 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
7810 t1
= fold_convert (fd
->iter_type
, t1
);
7811 t0
= fold_convert (fd
->iter_type
, t0
);
7815 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
7816 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
7819 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
7823 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
7824 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
7826 t
= build_call_expr (builtin_decl_explicit (start_fn
),
7827 5, t0
, t1
, t
, t3
, t4
);
7829 t
= build_call_expr (builtin_decl_explicit (start_fn
),
7830 6, t0
, t1
, t2
, t
, t3
, t4
);
7832 else if (fd
->ordered
)
7833 t
= build_call_expr (builtin_decl_explicit (start_fn
),
7836 t
= build_call_expr (builtin_decl_explicit (start_fn
),
7837 5, t0
, t1
, t2
, t3
, t4
);
7845 /* The GOMP_loop_ull_*start functions have additional boolean
7846 argument, true for < loops and false for > loops.
7847 In Fortran, the C bool type can be different from
7848 boolean_type_node. */
7849 bfn_decl
= builtin_decl_explicit (start_fn
);
7850 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
7851 t5
= build_int_cst (c_bool_type
,
7852 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
7855 tree bfn_decl
= builtin_decl_explicit (start_fn
);
7856 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
7857 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
7858 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
7861 t
= build_call_expr (builtin_decl_explicit (start_fn
),
7862 6, t5
, t0
, t1
, t2
, t3
, t4
);
7865 if (TREE_TYPE (t
) != boolean_type_node
)
7866 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
7867 t
, build_int_cst (TREE_TYPE (t
), 0));
7868 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7869 true, GSI_SAME_STMT
);
7870 if (arr
&& !TREE_STATIC (arr
))
7872 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
7873 TREE_THIS_VOLATILE (clobber
) = 1;
7874 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
7877 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7879 /* Remove the GIMPLE_OMP_FOR statement. */
7880 gsi_remove (&gsi
, true);
7882 if (gsi_end_p (gsif
))
7883 gsif
= gsi_after_labels (gsi_bb (gsif
));
7886 /* Iteration setup for sequential loop goes in L0_BB. */
7887 tree startvar
= fd
->loop
.v
;
7888 tree endvar
= NULL_TREE
;
7890 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7892 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
7893 && gimple_omp_for_kind (inner_stmt
)
7894 == GF_OMP_FOR_KIND_SIMD
);
7895 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
7896 OMP_CLAUSE__LOOPTEMP_
);
7897 gcc_assert (innerc
);
7898 startvar
= OMP_CLAUSE_DECL (innerc
);
7899 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7900 OMP_CLAUSE__LOOPTEMP_
);
7901 gcc_assert (innerc
);
7902 endvar
= OMP_CLAUSE_DECL (innerc
);
7905 gsi
= gsi_start_bb (l0_bb
);
7907 if (fd
->ordered
&& fd
->collapse
== 1)
7908 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
7909 fold_convert (fd
->iter_type
, fd
->loop
.step
));
7911 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
7912 if (fd
->ordered
&& fd
->collapse
== 1)
7914 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
7915 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
7916 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7919 t
= fold_convert (TREE_TYPE (startvar
), t
);
7920 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
7926 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
7927 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
7928 t
= fold_convert (TREE_TYPE (startvar
), t
);
7930 t
= force_gimple_operand_gsi (&gsi
, t
,
7932 && TREE_ADDRESSABLE (startvar
),
7933 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7934 assign_stmt
= gimple_build_assign (startvar
, t
);
7935 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7938 if (fd
->ordered
&& fd
->collapse
== 1)
7939 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
7940 fold_convert (fd
->iter_type
, fd
->loop
.step
));
7942 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
7943 if (fd
->ordered
&& fd
->collapse
== 1)
7945 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
7946 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
7947 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7950 t
= fold_convert (TREE_TYPE (startvar
), t
);
7951 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
7957 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
7958 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
7959 t
= fold_convert (TREE_TYPE (startvar
), t
);
7961 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7962 false, GSI_CONTINUE_LINKING
);
7965 assign_stmt
= gimple_build_assign (endvar
, iend
);
7966 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7967 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
7968 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
7970 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
7971 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7973 /* Handle linear clause adjustments. */
7974 tree itercnt
= NULL_TREE
;
7975 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
7976 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
7977 c
; c
= OMP_CLAUSE_CHAIN (c
))
7978 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
7979 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
7981 tree d
= OMP_CLAUSE_DECL (c
);
7982 bool is_ref
= is_reference (d
);
7983 tree t
= d
, a
, dest
;
7985 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
7986 tree type
= TREE_TYPE (t
);
7987 if (POINTER_TYPE_P (type
))
7989 dest
= unshare_expr (t
);
7990 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
7991 expand_omp_build_assign (&gsif
, v
, t
);
7992 if (itercnt
== NULL_TREE
)
7995 tree n1
= fd
->loop
.n1
;
7996 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
7999 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8001 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8003 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8005 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8006 itercnt
, fd
->loop
.step
);
8007 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8009 GSI_CONTINUE_LINKING
);
8011 a
= fold_build2 (MULT_EXPR
, type
,
8012 fold_convert (type
, itercnt
),
8013 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8014 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8015 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8016 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8017 false, GSI_CONTINUE_LINKING
);
8018 assign_stmt
= gimple_build_assign (dest
, t
);
8019 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8021 if (fd
->collapse
> 1)
8022 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8026 /* Until now, counts array contained number of iterations or
8027 variable containing it for ith loop. From now on, we need
8028 those counts only for collapsed loops, and only for the 2nd
8029 till the last collapsed one. Move those one element earlier,
8030 we'll use counts[fd->collapse - 1] for the first source/sink
8031 iteration counter and so on and counts[fd->ordered]
8032 as the array holding the current counter values for
8034 if (fd
->collapse
> 1)
8035 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8039 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8041 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8043 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8044 fold_convert (type
, fd
->loops
[i
].n1
),
8045 fold_convert (type
, fd
->loops
[i
].n2
));
8046 if (!integer_onep (this_cond
))
8049 if (i
< fd
->ordered
)
8052 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8053 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8054 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8055 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8056 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8057 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8058 make_edge (cont_bb
, l1_bb
, 0);
8059 l2_bb
= create_empty_bb (cont_bb
);
8060 broken_loop
= false;
8063 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8064 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8065 ordered_lastprivate
);
8066 if (counts
[fd
->collapse
- 1])
8068 gcc_assert (fd
->collapse
== 1);
8069 gsi
= gsi_last_bb (l0_bb
);
8070 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8072 gsi
= gsi_last_bb (cont_bb
);
8073 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8074 build_int_cst (fd
->iter_type
, 1));
8075 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8076 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8077 size_zero_node
, NULL_TREE
, NULL_TREE
);
8078 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8079 t
= counts
[fd
->collapse
- 1];
8081 else if (fd
->collapse
> 1)
8085 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8086 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8087 t
= fold_convert (fd
->iter_type
, t
);
8089 gsi
= gsi_last_bb (l0_bb
);
8090 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8091 size_zero_node
, NULL_TREE
, NULL_TREE
);
8092 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8093 false, GSI_CONTINUE_LINKING
);
8094 expand_omp_build_assign (&gsi
, aref
, t
, true);
8099 /* Code to control the increment and predicate for the sequential
8100 loop goes in the CONT_BB. */
8101 gsi
= gsi_last_bb (cont_bb
);
8102 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8103 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8104 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8105 vback
= gimple_omp_continue_control_def (cont_stmt
);
8107 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8109 if (POINTER_TYPE_P (type
))
8110 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8112 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8113 t
= force_gimple_operand_gsi (&gsi
, t
,
8115 && TREE_ADDRESSABLE (vback
),
8116 NULL_TREE
, true, GSI_SAME_STMT
);
8117 assign_stmt
= gimple_build_assign (vback
, t
);
8118 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8120 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8122 if (fd
->collapse
> 1)
8126 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8127 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8128 t
= fold_convert (fd
->iter_type
, t
);
8130 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8131 counts
[fd
->ordered
], size_zero_node
,
8132 NULL_TREE
, NULL_TREE
);
8133 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8134 true, GSI_SAME_STMT
);
8135 expand_omp_build_assign (&gsi
, aref
, t
);
8138 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8139 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8141 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8142 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8145 /* Remove GIMPLE_OMP_CONTINUE. */
8146 gsi_remove (&gsi
, true);
8148 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8149 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8151 /* Emit code to get the next parallel iteration in L2_BB. */
8152 gsi
= gsi_start_bb (l2_bb
);
8154 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8155 build_fold_addr_expr (istart0
),
8156 build_fold_addr_expr (iend0
));
8157 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8158 false, GSI_CONTINUE_LINKING
);
8159 if (TREE_TYPE (t
) != boolean_type_node
)
8160 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8161 t
, build_int_cst (TREE_TYPE (t
), 0));
8162 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8163 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8166 /* Add the loop cleanup function. */
8167 gsi
= gsi_last_bb (exit_bb
);
8168 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8169 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8170 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8171 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8173 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8174 gcall
*call_stmt
= gimple_build_call (t
, 0);
8175 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8176 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8177 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8180 tree arr
= counts
[fd
->ordered
];
8181 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8182 TREE_THIS_VOLATILE (clobber
) = 1;
8183 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8186 gsi_remove (&gsi
, true);
8188 /* Connect the new blocks. */
8189 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8190 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8196 e
= find_edge (cont_bb
, l3_bb
);
8197 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8199 phis
= phi_nodes (l3_bb
);
8200 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8202 gimple
*phi
= gsi_stmt (gsi
);
8203 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8204 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8208 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8209 e
= find_edge (cont_bb
, l1_bb
);
8212 e
= BRANCH_EDGE (cont_bb
);
8213 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8215 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8220 else if (fd
->collapse
> 1)
8223 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
8226 e
->flags
= EDGE_TRUE_VALUE
;
8229 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
8230 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
8234 e
= find_edge (cont_bb
, l2_bb
);
8235 e
->flags
= EDGE_FALLTHRU
;
8237 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
8239 if (gimple_in_ssa_p (cfun
))
8241 /* Add phis to the outer loop that connect to the phis in the inner,
8242 original loop, and move the loop entry value of the inner phi to
8243 the loop entry value of the outer phi. */
8245 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
8247 source_location locus
;
8249 gphi
*exit_phi
= psi
.phi ();
8251 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
8252 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
8254 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
8255 edge latch_to_l1
= find_edge (latch
, l1_bb
);
8257 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
8259 tree t
= gimple_phi_result (exit_phi
);
8260 tree new_res
= copy_ssa_name (t
, NULL
);
8261 nphi
= create_phi_node (new_res
, l0_bb
);
8263 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
8264 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
8265 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
8266 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
8267 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
8269 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
8270 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
8272 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
8276 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
8277 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
8278 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
8279 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
8280 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
8281 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
8282 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
8283 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
8285 /* We enter expand_omp_for_generic with a loop. This original loop may
8286 have its own loop struct, or it may be part of an outer loop struct
8287 (which may be the fake loop). */
8288 struct loop
*outer_loop
= entry_bb
->loop_father
;
8289 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
8291 add_bb_to_loop (l2_bb
, outer_loop
);
8293 /* We've added a new loop around the original loop. Allocate the
8294 corresponding loop struct. */
8295 struct loop
*new_loop
= alloc_loop ();
8296 new_loop
->header
= l0_bb
;
8297 new_loop
->latch
= l2_bb
;
8298 add_loop (new_loop
, outer_loop
);
8300 /* Allocate a loop structure for the original loop unless we already
8302 if (!orig_loop_has_loop_struct
8303 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8305 struct loop
*orig_loop
= alloc_loop ();
8306 orig_loop
->header
= l1_bb
;
8307 /* The loop may have multiple latches. */
8308 add_loop (orig_loop
, new_loop
);
8314 /* A subroutine of expand_omp_for. Generate code for a parallel
8315 loop with static schedule and no specified chunk size. Given
8318 for (V = N1; V cond N2; V += STEP) BODY;
8320 where COND is "<" or ">", we generate pseudocode
8322 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
8327 if ((__typeof (V)) -1 > 0 && cond is >)
8328 n = -(adj + N2 - N1) / -STEP;
8330 n = (adj + N2 - N1) / STEP;
8333 if (threadid < tt) goto L3; else goto L4;
8338 s0 = q * threadid + tt;
8341 if (s0 >= e0) goto L2; else goto L0;
8347 if (V cond e) goto L1;
8352 expand_omp_for_static_nochunk (struct omp_region
*region
,
8353 struct omp_for_data
*fd
,
8356 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
8357 tree type
, itype
, vmain
, vback
;
8358 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
8359 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
8361 gimple_stmt_iterator gsi
;
8363 bool broken_loop
= region
->cont
== NULL
;
8364 tree
*counts
= NULL
;
8367 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
8368 != GF_OMP_FOR_KIND_OACC_LOOP
)
8371 itype
= type
= TREE_TYPE (fd
->loop
.v
);
8372 if (POINTER_TYPE_P (type
))
8373 itype
= signed_type_for (type
);
8375 entry_bb
= region
->entry
;
8376 cont_bb
= region
->cont
;
8377 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8378 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8379 gcc_assert (broken_loop
8380 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
8381 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8382 body_bb
= single_succ (seq_start_bb
);
8385 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
8386 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
8387 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8389 exit_bb
= region
->exit
;
8391 /* Iteration space partitioning goes in ENTRY_BB. */
8392 gsi
= gsi_last_bb (entry_bb
);
8393 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8395 if (fd
->collapse
> 1)
8397 int first_zero_iter
= -1, dummy
= -1;
8398 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
8400 counts
= XALLOCAVEC (tree
, fd
->collapse
);
8401 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8402 fin_bb
, first_zero_iter
,
8403 dummy_bb
, dummy
, l2_dom_bb
);
8406 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8407 t
= integer_one_node
;
8409 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
8410 fold_convert (type
, fd
->loop
.n1
),
8411 fold_convert (type
, fd
->loop
.n2
));
8412 if (fd
->collapse
== 1
8413 && TYPE_UNSIGNED (type
)
8414 && (t
== NULL_TREE
|| !integer_onep (t
)))
8416 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
8417 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
8418 true, GSI_SAME_STMT
);
8419 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
8420 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
8421 true, GSI_SAME_STMT
);
8422 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
8423 NULL_TREE
, NULL_TREE
);
8424 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8425 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
8426 expand_omp_regimplify_p
, NULL
, NULL
)
8427 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
8428 expand_omp_regimplify_p
, NULL
, NULL
))
8430 gsi
= gsi_for_stmt (cond_stmt
);
8431 gimple_regimplify_operands (cond_stmt
, &gsi
);
8433 ep
= split_block (entry_bb
, cond_stmt
);
8434 ep
->flags
= EDGE_TRUE_VALUE
;
8435 entry_bb
= ep
->dest
;
8436 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
8437 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
8438 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
8439 if (gimple_in_ssa_p (cfun
))
8441 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
8442 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
8443 !gsi_end_p (gpi
); gsi_next (&gpi
))
8445 gphi
*phi
= gpi
.phi ();
8446 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
8447 ep
, UNKNOWN_LOCATION
);
8450 gsi
= gsi_last_bb (entry_bb
);
8453 switch (gimple_omp_for_kind (fd
->for_stmt
))
8455 case GF_OMP_FOR_KIND_FOR
:
8456 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
8457 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8459 case GF_OMP_FOR_KIND_DISTRIBUTE
:
8460 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
8461 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
8463 case GF_OMP_FOR_KIND_OACC_LOOP
:
8464 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
8465 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
8470 nthreads
= build_call_expr (nthreads
, 0);
8471 nthreads
= fold_convert (itype
, nthreads
);
8472 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
8473 true, GSI_SAME_STMT
);
8474 threadid
= build_call_expr (threadid
, 0);
8475 threadid
= fold_convert (itype
, threadid
);
8476 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
8477 true, GSI_SAME_STMT
);
8481 step
= fd
->loop
.step
;
8482 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8484 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8485 OMP_CLAUSE__LOOPTEMP_
);
8486 gcc_assert (innerc
);
8487 n1
= OMP_CLAUSE_DECL (innerc
);
8488 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8489 OMP_CLAUSE__LOOPTEMP_
);
8490 gcc_assert (innerc
);
8491 n2
= OMP_CLAUSE_DECL (innerc
);
8493 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
8494 true, NULL_TREE
, true, GSI_SAME_STMT
);
8495 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
8496 true, NULL_TREE
, true, GSI_SAME_STMT
);
8497 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
8498 true, NULL_TREE
, true, GSI_SAME_STMT
);
8500 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
8501 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
8502 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
8503 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
8504 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
8505 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
8506 fold_build1 (NEGATE_EXPR
, itype
, t
),
8507 fold_build1 (NEGATE_EXPR
, itype
, step
));
8509 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
8510 t
= fold_convert (itype
, t
);
8511 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8513 q
= create_tmp_reg (itype
, "q");
8514 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
8515 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
8516 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
8518 tt
= create_tmp_reg (itype
, "tt");
8519 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
8520 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
8521 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
8523 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
8524 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8525 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8527 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
8528 gsi
= gsi_last_bb (second_bb
);
8529 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8531 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
8533 gassign
*assign_stmt
8534 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
8535 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8537 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
8538 gsi
= gsi_last_bb (third_bb
);
8539 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8541 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
8542 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
8543 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8545 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
8546 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8548 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
8549 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8551 /* Remove the GIMPLE_OMP_FOR statement. */
8552 gsi_remove (&gsi
, true);
8554 /* Setup code for sequential iteration goes in SEQ_START_BB. */
8555 gsi
= gsi_start_bb (seq_start_bb
);
8557 tree startvar
= fd
->loop
.v
;
8558 tree endvar
= NULL_TREE
;
8560 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8562 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
8563 ? gimple_omp_parallel_clauses (inner_stmt
)
8564 : gimple_omp_for_clauses (inner_stmt
);
8565 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
8566 gcc_assert (innerc
);
8567 startvar
= OMP_CLAUSE_DECL (innerc
);
8568 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8569 OMP_CLAUSE__LOOPTEMP_
);
8570 gcc_assert (innerc
);
8571 endvar
= OMP_CLAUSE_DECL (innerc
);
8573 t
= fold_convert (itype
, s0
);
8574 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
8575 if (POINTER_TYPE_P (type
))
8576 t
= fold_build_pointer_plus (n1
, t
);
8578 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
8579 t
= fold_convert (TREE_TYPE (startvar
), t
);
8580 t
= force_gimple_operand_gsi (&gsi
, t
,
8582 && TREE_ADDRESSABLE (startvar
),
8583 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8584 assign_stmt
= gimple_build_assign (startvar
, t
);
8585 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8587 t
= fold_convert (itype
, e0
);
8588 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
8589 if (POINTER_TYPE_P (type
))
8590 t
= fold_build_pointer_plus (n1
, t
);
8592 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
8593 t
= fold_convert (TREE_TYPE (startvar
), t
);
8594 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8595 false, GSI_CONTINUE_LINKING
);
8598 assign_stmt
= gimple_build_assign (endvar
, e
);
8599 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8600 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
8601 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
8603 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
8604 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8606 /* Handle linear clause adjustments. */
8607 tree itercnt
= NULL_TREE
;
8608 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8609 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8610 c
; c
= OMP_CLAUSE_CHAIN (c
))
8611 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8612 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8614 tree d
= OMP_CLAUSE_DECL (c
);
8615 bool is_ref
= is_reference (d
);
8616 tree t
= d
, a
, dest
;
8618 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8619 if (itercnt
== NULL_TREE
)
8621 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8623 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
8624 fold_convert (itype
, n1
),
8625 fold_convert (itype
, fd
->loop
.n1
));
8626 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
8627 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
8628 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8630 GSI_CONTINUE_LINKING
);
8635 tree type
= TREE_TYPE (t
);
8636 if (POINTER_TYPE_P (type
))
8638 a
= fold_build2 (MULT_EXPR
, type
,
8639 fold_convert (type
, itercnt
),
8640 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8641 dest
= unshare_expr (t
);
8642 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8643 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
8644 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8645 false, GSI_CONTINUE_LINKING
);
8646 assign_stmt
= gimple_build_assign (dest
, t
);
8647 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8649 if (fd
->collapse
> 1)
8650 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8654 /* The code controlling the sequential loop replaces the
8655 GIMPLE_OMP_CONTINUE. */
8656 gsi
= gsi_last_bb (cont_bb
);
8657 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8658 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8659 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8660 vback
= gimple_omp_continue_control_def (cont_stmt
);
8662 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8664 if (POINTER_TYPE_P (type
))
8665 t
= fold_build_pointer_plus (vmain
, step
);
8667 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
8668 t
= force_gimple_operand_gsi (&gsi
, t
,
8670 && TREE_ADDRESSABLE (vback
),
8671 NULL_TREE
, true, GSI_SAME_STMT
);
8672 assign_stmt
= gimple_build_assign (vback
, t
);
8673 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8675 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8676 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
8678 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8681 /* Remove the GIMPLE_OMP_CONTINUE statement. */
8682 gsi_remove (&gsi
, true);
8684 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8685 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
8688 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
8689 gsi
= gsi_last_bb (exit_bb
);
8690 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8692 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
8693 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
8694 gcc_checking_assert (t
== NULL_TREE
);
8696 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
8698 gsi_remove (&gsi
, true);
8700 /* Connect all the blocks. */
8701 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
8702 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
8703 ep
= find_edge (entry_bb
, second_bb
);
8704 ep
->flags
= EDGE_TRUE_VALUE
;
8705 ep
->probability
= REG_BR_PROB_BASE
/ 4;
8706 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
8707 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
8711 ep
= find_edge (cont_bb
, body_bb
);
8714 ep
= BRANCH_EDGE (cont_bb
);
8715 gcc_assert (single_succ (ep
->dest
) == body_bb
);
8717 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8722 else if (fd
->collapse
> 1)
8725 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
8728 ep
->flags
= EDGE_TRUE_VALUE
;
8729 find_edge (cont_bb
, fin_bb
)->flags
8730 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
8733 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
8734 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
8735 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
8737 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
8738 recompute_dominator (CDI_DOMINATORS
, body_bb
));
8739 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
8740 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
8742 struct loop
*loop
= body_bb
->loop_father
;
8743 if (loop
!= entry_bb
->loop_father
)
8745 gcc_assert (loop
->header
== body_bb
);
8746 gcc_assert (broken_loop
8747 || loop
->latch
== region
->cont
8748 || single_pred (loop
->latch
) == region
->cont
);
8752 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
8754 loop
= alloc_loop ();
8755 loop
->header
= body_bb
;
8756 if (collapse_bb
== NULL
)
8757 loop
->latch
= cont_bb
;
8758 add_loop (loop
, body_bb
->loop_father
);
8762 /* Return phi in E->DEST with ARG on edge E. */
8765 find_phi_with_arg_on_edge (tree arg
, edge e
)
8767 basic_block bb
= e
->dest
;
8769 for (gphi_iterator gpi
= gsi_start_phis (bb
);
8773 gphi
*phi
= gpi
.phi ();
8774 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
8781 /* A subroutine of expand_omp_for. Generate code for a parallel
8782 loop with static schedule and a specified chunk size. Given
8785 for (V = N1; V cond N2; V += STEP) BODY;
8787 where COND is "<" or ">", we generate pseudocode
8789 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
8794 if ((__typeof (V)) -1 > 0 && cond is >)
8795 n = -(adj + N2 - N1) / -STEP;
8797 n = (adj + N2 - N1) / STEP;
8799 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
8800 here so that V is defined
8801 if the loop is not entered
8803 s0 = (trip * nthreads + threadid) * CHUNK;
8804 e0 = min(s0 + CHUNK, n);
8805 if (s0 < n) goto L1; else goto L4;
8812 if (V cond e) goto L2; else goto L3;
8820 expand_omp_for_static_chunk (struct omp_region
*region
,
8821 struct omp_for_data
*fd
, gimple
*inner_stmt
)
8823 tree n
, s0
, e0
, e
, t
;
8824 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
8825 tree type
, itype
, vmain
, vback
, vextra
;
8826 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
8827 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
8828 gimple_stmt_iterator gsi
;
8830 bool broken_loop
= region
->cont
== NULL
;
8831 tree
*counts
= NULL
;
8834 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
8835 != GF_OMP_FOR_KIND_OACC_LOOP
)
8838 itype
= type
= TREE_TYPE (fd
->loop
.v
);
8839 if (POINTER_TYPE_P (type
))
8840 itype
= signed_type_for (type
);
8842 entry_bb
= region
->entry
;
8843 se
= split_block (entry_bb
, last_stmt (entry_bb
));
8845 iter_part_bb
= se
->dest
;
8846 cont_bb
= region
->cont
;
8847 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
8848 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
8849 gcc_assert (broken_loop
8850 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
8851 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
8852 body_bb
= single_succ (seq_start_bb
);
8855 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
8856 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
8857 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8858 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
8860 exit_bb
= region
->exit
;
8862 /* Trip and adjustment setup goes in ENTRY_BB. */
8863 gsi
= gsi_last_bb (entry_bb
);
8864 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8866 if (fd
->collapse
> 1)
8868 int first_zero_iter
= -1, dummy
= -1;
8869 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
8871 counts
= XALLOCAVEC (tree
, fd
->collapse
);
8872 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8873 fin_bb
, first_zero_iter
,
8874 dummy_bb
, dummy
, l2_dom_bb
);
8877 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8878 t
= integer_one_node
;
8880 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
8881 fold_convert (type
, fd
->loop
.n1
),
8882 fold_convert (type
, fd
->loop
.n2
));
8883 if (fd
->collapse
== 1
8884 && TYPE_UNSIGNED (type
)
8885 && (t
== NULL_TREE
|| !integer_onep (t
)))
8887 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
8888 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
8889 true, GSI_SAME_STMT
);
8890 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
8891 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
8892 true, GSI_SAME_STMT
);
8893 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
8894 NULL_TREE
, NULL_TREE
);
8895 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8896 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
8897 expand_omp_regimplify_p
, NULL
, NULL
)
8898 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
8899 expand_omp_regimplify_p
, NULL
, NULL
))
8901 gsi
= gsi_for_stmt (cond_stmt
);
8902 gimple_regimplify_operands (cond_stmt
, &gsi
);
8904 se
= split_block (entry_bb
, cond_stmt
);
8905 se
->flags
= EDGE_TRUE_VALUE
;
8906 entry_bb
= se
->dest
;
8907 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
8908 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
8909 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
8910 if (gimple_in_ssa_p (cfun
))
8912 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
8913 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
8914 !gsi_end_p (gpi
); gsi_next (&gpi
))
8916 gphi
*phi
= gpi
.phi ();
8917 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
8918 se
, UNKNOWN_LOCATION
);
8921 gsi
= gsi_last_bb (entry_bb
);
8924 switch (gimple_omp_for_kind (fd
->for_stmt
))
8926 case GF_OMP_FOR_KIND_FOR
:
8927 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
8928 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8930 case GF_OMP_FOR_KIND_DISTRIBUTE
:
8931 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
8932 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
8934 case GF_OMP_FOR_KIND_OACC_LOOP
:
8935 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
8936 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
8941 nthreads
= build_call_expr (nthreads
, 0);
8942 nthreads
= fold_convert (itype
, nthreads
);
8943 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
8944 true, GSI_SAME_STMT
);
8945 threadid
= build_call_expr (threadid
, 0);
8946 threadid
= fold_convert (itype
, threadid
);
8947 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
8948 true, GSI_SAME_STMT
);
8952 step
= fd
->loop
.step
;
8953 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8955 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8956 OMP_CLAUSE__LOOPTEMP_
);
8957 gcc_assert (innerc
);
8958 n1
= OMP_CLAUSE_DECL (innerc
);
8959 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8960 OMP_CLAUSE__LOOPTEMP_
);
8961 gcc_assert (innerc
);
8962 n2
= OMP_CLAUSE_DECL (innerc
);
8964 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
8965 true, NULL_TREE
, true, GSI_SAME_STMT
);
8966 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
8967 true, NULL_TREE
, true, GSI_SAME_STMT
);
8968 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
8969 true, NULL_TREE
, true, GSI_SAME_STMT
);
8970 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
8971 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
8973 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
8976 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
8977 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
8978 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
8979 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
8980 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
8981 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
8982 fold_build1 (NEGATE_EXPR
, itype
, t
),
8983 fold_build1 (NEGATE_EXPR
, itype
, step
));
8985 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
8986 t
= fold_convert (itype
, t
);
8987 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8988 true, GSI_SAME_STMT
);
8990 trip_var
= create_tmp_reg (itype
, ".trip");
8991 if (gimple_in_ssa_p (cfun
))
8993 trip_init
= make_ssa_name (trip_var
);
8994 trip_main
= make_ssa_name (trip_var
);
8995 trip_back
= make_ssa_name (trip_var
);
8999 trip_init
= trip_var
;
9000 trip_main
= trip_var
;
9001 trip_back
= trip_var
;
9004 gassign
*assign_stmt
9005 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9006 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9008 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9009 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9010 if (POINTER_TYPE_P (type
))
9011 t
= fold_build_pointer_plus (n1
, t
);
9013 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9014 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9015 true, GSI_SAME_STMT
);
9017 /* Remove the GIMPLE_OMP_FOR. */
9018 gsi_remove (&gsi
, true);
9020 gimple_stmt_iterator gsif
= gsi
;
9022 /* Iteration space partitioning goes in ITER_PART_BB. */
9023 gsi
= gsi_last_bb (iter_part_bb
);
9025 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9026 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9027 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9028 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9029 false, GSI_CONTINUE_LINKING
);
9031 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9032 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9033 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9034 false, GSI_CONTINUE_LINKING
);
9036 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9037 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9039 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9040 gsi
= gsi_start_bb (seq_start_bb
);
9042 tree startvar
= fd
->loop
.v
;
9043 tree endvar
= NULL_TREE
;
9045 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9047 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9048 ? gimple_omp_parallel_clauses (inner_stmt
)
9049 : gimple_omp_for_clauses (inner_stmt
);
9050 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9051 gcc_assert (innerc
);
9052 startvar
= OMP_CLAUSE_DECL (innerc
);
9053 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9054 OMP_CLAUSE__LOOPTEMP_
);
9055 gcc_assert (innerc
);
9056 endvar
= OMP_CLAUSE_DECL (innerc
);
9059 t
= fold_convert (itype
, s0
);
9060 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9061 if (POINTER_TYPE_P (type
))
9062 t
= fold_build_pointer_plus (n1
, t
);
9064 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9065 t
= fold_convert (TREE_TYPE (startvar
), t
);
9066 t
= force_gimple_operand_gsi (&gsi
, t
,
9068 && TREE_ADDRESSABLE (startvar
),
9069 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9070 assign_stmt
= gimple_build_assign (startvar
, t
);
9071 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9073 t
= fold_convert (itype
, e0
);
9074 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9075 if (POINTER_TYPE_P (type
))
9076 t
= fold_build_pointer_plus (n1
, t
);
9078 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9079 t
= fold_convert (TREE_TYPE (startvar
), t
);
9080 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9081 false, GSI_CONTINUE_LINKING
);
9084 assign_stmt
= gimple_build_assign (endvar
, e
);
9085 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9086 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9087 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9089 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9090 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9092 /* Handle linear clause adjustments. */
9093 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9094 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9095 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9096 c
; c
= OMP_CLAUSE_CHAIN (c
))
9097 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9098 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9100 tree d
= OMP_CLAUSE_DECL (c
);
9101 bool is_ref
= is_reference (d
);
9102 tree t
= d
, a
, dest
;
9104 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9105 tree type
= TREE_TYPE (t
);
9106 if (POINTER_TYPE_P (type
))
9108 dest
= unshare_expr (t
);
9109 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9110 expand_omp_build_assign (&gsif
, v
, t
);
9111 if (itercnt
== NULL_TREE
)
9113 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9116 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9117 fold_convert (itype
, fd
->loop
.n1
));
9118 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9121 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9124 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9125 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9127 GSI_CONTINUE_LINKING
);
9132 a
= fold_build2 (MULT_EXPR
, type
,
9133 fold_convert (type
, itercnt
),
9134 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9135 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9136 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9137 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9138 false, GSI_CONTINUE_LINKING
);
9139 assign_stmt
= gimple_build_assign (dest
, t
);
9140 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9142 if (fd
->collapse
> 1)
9143 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9147 /* The code controlling the sequential loop goes in CONT_BB,
9148 replacing the GIMPLE_OMP_CONTINUE. */
9149 gsi
= gsi_last_bb (cont_bb
);
9150 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9151 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9152 vback
= gimple_omp_continue_control_def (cont_stmt
);
9154 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9156 if (POINTER_TYPE_P (type
))
9157 t
= fold_build_pointer_plus (vmain
, step
);
9159 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9160 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9161 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9162 true, GSI_SAME_STMT
);
9163 assign_stmt
= gimple_build_assign (vback
, t
);
9164 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9166 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9167 t
= build2 (EQ_EXPR
, boolean_type_node
,
9168 build_int_cst (itype
, 0),
9169 build_int_cst (itype
, 1));
9171 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9172 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9174 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9177 /* Remove GIMPLE_OMP_CONTINUE. */
9178 gsi_remove (&gsi
, true);
9180 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9181 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9183 /* Trip update code goes into TRIP_UPDATE_BB. */
9184 gsi
= gsi_start_bb (trip_update_bb
);
9186 t
= build_int_cst (itype
, 1);
9187 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9188 assign_stmt
= gimple_build_assign (trip_back
, t
);
9189 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9192 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9193 gsi
= gsi_last_bb (exit_bb
);
9194 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9196 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9197 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
9198 gcc_checking_assert (t
== NULL_TREE
);
9200 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9202 gsi_remove (&gsi
, true);
9204 /* Connect the new blocks. */
9205 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
9206 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
9210 se
= find_edge (cont_bb
, body_bb
);
9213 se
= BRANCH_EDGE (cont_bb
);
9214 gcc_assert (single_succ (se
->dest
) == body_bb
);
9216 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9221 else if (fd
->collapse
> 1)
9224 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9227 se
->flags
= EDGE_TRUE_VALUE
;
9228 find_edge (cont_bb
, trip_update_bb
)->flags
9229 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9231 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
9234 if (gimple_in_ssa_p (cfun
))
9242 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
9244 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
9245 remove arguments of the phi nodes in fin_bb. We need to create
9246 appropriate phi nodes in iter_part_bb instead. */
9247 se
= find_edge (iter_part_bb
, fin_bb
);
9248 re
= single_succ_edge (trip_update_bb
);
9249 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
9250 ene
= single_succ_edge (entry_bb
);
9252 psi
= gsi_start_phis (fin_bb
);
9253 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
9254 gsi_next (&psi
), ++i
)
9257 source_location locus
;
9260 t
= gimple_phi_result (phi
);
9261 gcc_assert (t
== redirect_edge_var_map_result (vm
));
9263 if (!single_pred_p (fin_bb
))
9264 t
= copy_ssa_name (t
, phi
);
9266 nphi
= create_phi_node (t
, iter_part_bb
);
9268 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
9269 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
9271 /* A special case -- fd->loop.v is not yet computed in
9272 iter_part_bb, we need to use vextra instead. */
9273 if (t
== fd
->loop
.v
)
9275 add_phi_arg (nphi
, t
, ene
, locus
);
9276 locus
= redirect_edge_var_map_location (vm
);
9277 tree back_arg
= redirect_edge_var_map_def (vm
);
9278 add_phi_arg (nphi
, back_arg
, re
, locus
);
9279 edge ce
= find_edge (cont_bb
, body_bb
);
9282 ce
= BRANCH_EDGE (cont_bb
);
9283 gcc_assert (single_succ (ce
->dest
) == body_bb
);
9284 ce
= single_succ_edge (ce
->dest
);
9286 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
9287 gcc_assert (inner_loop_phi
!= NULL
);
9288 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
9289 find_edge (seq_start_bb
, body_bb
), locus
);
9291 if (!single_pred_p (fin_bb
))
9292 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
9294 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
9295 redirect_edge_var_map_clear (re
);
9296 if (single_pred_p (fin_bb
))
9299 psi
= gsi_start_phis (fin_bb
);
9300 if (gsi_end_p (psi
))
9302 remove_phi_node (&psi
, false);
9305 /* Make phi node for trip. */
9306 phi
= create_phi_node (trip_main
, iter_part_bb
);
9307 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
9309 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
9314 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
9315 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
9316 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
9317 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9318 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9319 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
9320 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
9321 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9322 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9326 struct loop
*loop
= body_bb
->loop_father
;
9327 struct loop
*trip_loop
= alloc_loop ();
9328 trip_loop
->header
= iter_part_bb
;
9329 trip_loop
->latch
= trip_update_bb
;
9330 add_loop (trip_loop
, iter_part_bb
->loop_father
);
9332 if (loop
!= entry_bb
->loop_father
)
9334 gcc_assert (loop
->header
== body_bb
);
9335 gcc_assert (loop
->latch
== region
->cont
9336 || single_pred (loop
->latch
) == region
->cont
);
9337 trip_loop
->inner
= loop
;
9341 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9343 loop
= alloc_loop ();
9344 loop
->header
= body_bb
;
9345 if (collapse_bb
== NULL
)
9346 loop
->latch
= cont_bb
;
9347 add_loop (loop
, trip_loop
);
9352 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
9354 for (V = N1; V cond N2; V += STEP) BODY;
9356 where COND is "<" or ">" or "!=", we generate pseudocode
9358 for (ind_var = low; ind_var < high; ind_var++)
9360 V = n1 + (ind_var * STEP)
9365 In the above pseudocode, low and high are function parameters of the
9366 child function. In the function below, we are inserting a temp.
9367 variable that will be making a call to two OMP functions that will not be
9368 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
9369 with _Cilk_for). These functions are replaced with low and high
9370 by the function that handles taskreg. */
9374 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
9376 bool broken_loop
= region
->cont
== NULL
;
9377 basic_block entry_bb
= region
->entry
;
9378 basic_block cont_bb
= region
->cont
;
9380 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9381 gcc_assert (broken_loop
9382 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
9383 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
9384 basic_block l1_bb
, l2_bb
;
9388 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
9389 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9390 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
9391 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9395 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
9396 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
9397 l2_bb
= single_succ (l1_bb
);
9399 basic_block exit_bb
= region
->exit
;
9400 basic_block l2_dom_bb
= NULL
;
9402 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
9404 /* Below statements until the "tree high_val = ..." are pseudo statements
9405 used to pass information to be used by expand_omp_taskreg.
9406 low_val and high_val will be replaced by the __low and __high
9407 parameter from the child function.
9409 The call_exprs part is a place-holder, it is mainly used
9410 to distinctly identify to the top-level part that this is
9411 where we should put low and high (reasoning given in header
9415 = gimple_omp_parallel_child_fn (
9416 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
9417 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
9418 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
9420 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
9422 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
9425 gcc_assert (low_val
&& high_val
);
9427 tree type
= TREE_TYPE (low_val
);
9428 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
9429 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9431 /* Not needed in SSA form right now. */
9432 gcc_assert (!gimple_in_ssa_p (cfun
));
9433 if (l2_dom_bb
== NULL
)
9439 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
9441 /* Replace the GIMPLE_OMP_FOR statement. */
9442 gsi_replace (&gsi
, stmt
, true);
9446 /* Code to control the increment goes in the CONT_BB. */
9447 gsi
= gsi_last_bb (cont_bb
);
9448 stmt
= gsi_stmt (gsi
);
9449 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
9450 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
9451 build_one_cst (type
));
9453 /* Replace GIMPLE_OMP_CONTINUE. */
9454 gsi_replace (&gsi
, stmt
, true);
9457 /* Emit the condition in L1_BB. */
9458 gsi
= gsi_after_labels (l1_bb
);
9459 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
9460 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
9462 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
9463 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
9464 fd
->loop
.n1
, fold_convert (sizetype
, t
));
9466 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
9467 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
9468 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
9469 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
9471 /* The condition is always '<' since the runtime will fill in the low
9473 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
9474 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
9476 /* Remove GIMPLE_OMP_RETURN. */
9477 gsi
= gsi_last_bb (exit_bb
);
9478 gsi_remove (&gsi
, true);
9480 /* Connect the new blocks. */
9481 remove_edge (FALLTHRU_EDGE (entry_bb
));
9486 remove_edge (BRANCH_EDGE (entry_bb
));
9487 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
9489 e
= BRANCH_EDGE (l1_bb
);
9490 ne
= FALLTHRU_EDGE (l1_bb
);
9491 e
->flags
= EDGE_TRUE_VALUE
;
9495 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
9497 ne
= single_succ_edge (l1_bb
);
9498 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9501 ne
->flags
= EDGE_FALSE_VALUE
;
9502 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9503 ne
->probability
= REG_BR_PROB_BASE
/ 8;
9505 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
9506 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
9507 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
9511 struct loop
*loop
= alloc_loop ();
9512 loop
->header
= l1_bb
;
9513 loop
->latch
= cont_bb
;
9514 add_loop (loop
, l1_bb
->loop_father
);
9515 loop
->safelen
= INT_MAX
;
9518 /* Pick the correct library function based on the precision of the
9519 induction variable type. */
9520 tree lib_fun
= NULL_TREE
;
9521 if (TYPE_PRECISION (type
) == 32)
9522 lib_fun
= cilk_for_32_fndecl
;
9523 else if (TYPE_PRECISION (type
) == 64)
9524 lib_fun
= cilk_for_64_fndecl
;
9528 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
9530 /* WS_ARGS contains the library function flavor to call:
9531 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
9532 user-defined grain value. If the user does not define one, then zero
9533 is passed in by the parser. */
9534 vec_alloc (region
->ws_args
, 2);
9535 region
->ws_args
->quick_push (lib_fun
);
9536 region
->ws_args
->quick_push (fd
->chunk_size
);
9539 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
9540 loop. Given parameters:
9542 for (V = N1; V cond N2; V += STEP) BODY;
9544 where COND is "<" or ">", we generate pseudocode
9552 if (V cond N2) goto L0; else goto L2;
9555 For collapsed loops, given parameters:
9557 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
9558 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
9559 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
9562 we generate pseudocode
9568 count3 = (adj + N32 - N31) / STEP3;
9573 count2 = (adj + N22 - N21) / STEP2;
9578 count1 = (adj + N12 - N11) / STEP1;
9579 count = count1 * count2 * count3;
9589 V2 += (V3 cond3 N32) ? 0 : STEP2;
9590 V3 = (V3 cond3 N32) ? V3 : N31;
9591 V1 += (V2 cond2 N22) ? 0 : STEP1;
9592 V2 = (V2 cond2 N22) ? V2 : N21;
9594 if (V < count) goto L0; else goto L2;
9600 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
9603 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
9604 gimple_stmt_iterator gsi
;
9607 bool broken_loop
= region
->cont
== NULL
;
9609 tree
*counts
= NULL
;
9611 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9612 OMP_CLAUSE_SAFELEN
);
9613 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9614 OMP_CLAUSE__SIMDUID_
);
9617 type
= TREE_TYPE (fd
->loop
.v
);
9618 entry_bb
= region
->entry
;
9619 cont_bb
= region
->cont
;
9620 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9621 gcc_assert (broken_loop
9622 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
9623 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
9626 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
9627 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9628 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
9629 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9633 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
9634 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
9635 l2_bb
= single_succ (l1_bb
);
9637 exit_bb
= region
->exit
;
9640 gsi
= gsi_last_bb (entry_bb
);
9642 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9643 /* Not needed in SSA form right now. */
9644 gcc_assert (!gimple_in_ssa_p (cfun
));
9645 if (fd
->collapse
> 1)
9647 int first_zero_iter
= -1, dummy
= -1;
9648 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
9650 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9651 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9652 zero_iter_bb
, first_zero_iter
,
9653 dummy_bb
, dummy
, l2_dom_bb
);
9655 if (l2_dom_bb
== NULL
)
9660 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9662 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9663 OMP_CLAUSE__LOOPTEMP_
);
9664 gcc_assert (innerc
);
9665 n1
= OMP_CLAUSE_DECL (innerc
);
9666 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9667 OMP_CLAUSE__LOOPTEMP_
);
9668 gcc_assert (innerc
);
9669 n2
= OMP_CLAUSE_DECL (innerc
);
9670 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
9671 fold_convert (type
, n1
));
9672 if (fd
->collapse
> 1)
9675 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
9681 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
9682 fold_convert (type
, fd
->loop
.n1
));
9683 if (fd
->collapse
> 1)
9684 for (i
= 0; i
< fd
->collapse
; i
++)
9686 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
9687 if (POINTER_TYPE_P (itype
))
9688 itype
= signed_type_for (itype
);
9689 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
9690 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
9694 /* Remove the GIMPLE_OMP_FOR statement. */
9695 gsi_remove (&gsi
, true);
9699 /* Code to control the increment goes in the CONT_BB. */
9700 gsi
= gsi_last_bb (cont_bb
);
9701 stmt
= gsi_stmt (gsi
);
9702 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
9704 if (POINTER_TYPE_P (type
))
9705 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
9707 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
9708 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
9710 if (fd
->collapse
> 1)
9712 i
= fd
->collapse
- 1;
9713 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
9715 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
9716 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
9720 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
9722 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
9725 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
9727 for (i
= fd
->collapse
- 1; i
> 0; i
--)
9729 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
9730 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
9731 if (POINTER_TYPE_P (itype2
))
9732 itype2
= signed_type_for (itype2
);
9733 t
= build3 (COND_EXPR
, itype2
,
9734 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
9736 fold_convert (itype
, fd
->loops
[i
].n2
)),
9737 build_int_cst (itype2
, 0),
9738 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
9739 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
9740 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
9742 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
9743 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
9745 t
= build3 (COND_EXPR
, itype
,
9746 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
9748 fold_convert (itype
, fd
->loops
[i
].n2
)),
9750 fold_convert (itype
, fd
->loops
[i
].n1
));
9751 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
9755 /* Remove GIMPLE_OMP_CONTINUE. */
9756 gsi_remove (&gsi
, true);
9759 /* Emit the condition in L1_BB. */
9760 gsi
= gsi_start_bb (l1_bb
);
9762 t
= fold_convert (type
, n2
);
9763 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9764 false, GSI_CONTINUE_LINKING
);
9765 tree v
= fd
->loop
.v
;
9766 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
9767 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
9768 false, GSI_CONTINUE_LINKING
);
9769 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
9770 cond_stmt
= gimple_build_cond_empty (t
);
9771 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
9772 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
9774 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
9777 gsi
= gsi_for_stmt (cond_stmt
);
9778 gimple_regimplify_operands (cond_stmt
, &gsi
);
9781 /* Remove GIMPLE_OMP_RETURN. */
9782 gsi
= gsi_last_bb (exit_bb
);
9783 gsi_remove (&gsi
, true);
9785 /* Connect the new blocks. */
9786 remove_edge (FALLTHRU_EDGE (entry_bb
));
9790 remove_edge (BRANCH_EDGE (entry_bb
));
9791 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
9793 e
= BRANCH_EDGE (l1_bb
);
9794 ne
= FALLTHRU_EDGE (l1_bb
);
9795 e
->flags
= EDGE_TRUE_VALUE
;
9799 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
9801 ne
= single_succ_edge (l1_bb
);
9802 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9805 ne
->flags
= EDGE_FALSE_VALUE
;
9806 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9807 ne
->probability
= REG_BR_PROB_BASE
/ 8;
9809 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
9810 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
9811 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
9815 struct loop
*loop
= alloc_loop ();
9816 loop
->header
= l1_bb
;
9817 loop
->latch
= cont_bb
;
9818 add_loop (loop
, l1_bb
->loop_father
);
9819 if (safelen
== NULL_TREE
)
9820 loop
->safelen
= INT_MAX
;
9823 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
9824 if (TREE_CODE (safelen
) != INTEGER_CST
)
9826 else if (!tree_fits_uhwi_p (safelen
)
9827 || tree_to_uhwi (safelen
) > INT_MAX
)
9828 loop
->safelen
= INT_MAX
;
9830 loop
->safelen
= tree_to_uhwi (safelen
);
9831 if (loop
->safelen
== 1)
9836 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
9837 cfun
->has_simduid_loops
= true;
9839 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
9841 if ((flag_tree_loop_vectorize
9842 || (!global_options_set
.x_flag_tree_loop_vectorize
9843 && !global_options_set
.x_flag_tree_vectorize
))
9844 && flag_tree_loop_optimize
9845 && loop
->safelen
> 1)
9847 loop
->force_vectorize
= true;
9848 cfun
->has_force_vectorize_loops
= true;
9852 cfun
->has_simduid_loops
= true;
9855 /* Taskloop construct is represented after gimplification with
9856 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
9857 in between them. This routine expands the outer GIMPLE_OMP_FOR,
9858 which should just compute all the needed loop temporaries
9859 for GIMPLE_OMP_TASK. */
9862 expand_omp_taskloop_for_outer (struct omp_region
*region
,
9863 struct omp_for_data
*fd
,
9866 tree type
, bias
= NULL_TREE
;
9867 basic_block entry_bb
, cont_bb
, exit_bb
;
9868 gimple_stmt_iterator gsi
;
9869 gassign
*assign_stmt
;
9870 tree
*counts
= NULL
;
9873 gcc_assert (inner_stmt
);
9874 gcc_assert (region
->cont
);
9875 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
9876 && gimple_omp_task_taskloop_p (inner_stmt
));
9877 type
= TREE_TYPE (fd
->loop
.v
);
9879 /* See if we need to bias by LLONG_MIN. */
9880 if (fd
->iter_type
== long_long_unsigned_type_node
9881 && TREE_CODE (type
) == INTEGER_TYPE
9882 && !TYPE_UNSIGNED (type
))
9886 if (fd
->loop
.cond_code
== LT_EXPR
)
9889 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
9893 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
9896 if (TREE_CODE (n1
) != INTEGER_CST
9897 || TREE_CODE (n2
) != INTEGER_CST
9898 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
9899 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
9902 entry_bb
= region
->entry
;
9903 cont_bb
= region
->cont
;
9904 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9905 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
9906 exit_bb
= region
->exit
;
9908 gsi
= gsi_last_bb (entry_bb
);
9909 gimple
*for_stmt
= gsi_stmt (gsi
);
9910 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
9911 if (fd
->collapse
> 1)
9913 int first_zero_iter
= -1, dummy
= -1;
9914 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
9916 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9917 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9918 zero_iter_bb
, first_zero_iter
,
9919 dummy_bb
, dummy
, l2_dom_bb
);
9923 /* Some counts[i] vars might be uninitialized if
9924 some loop has zero iterations. But the body shouldn't
9925 be executed in that case, so just avoid uninit warnings. */
9926 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
9927 if (SSA_VAR_P (counts
[i
]))
9928 TREE_NO_WARNING (counts
[i
]) = 1;
9930 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
9932 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
9933 gsi
= gsi_last_bb (entry_bb
);
9934 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
9935 get_immediate_dominator (CDI_DOMINATORS
,
9943 if (POINTER_TYPE_P (TREE_TYPE (t0
))
9944 && TYPE_PRECISION (TREE_TYPE (t0
))
9945 != TYPE_PRECISION (fd
->iter_type
))
9947 /* Avoid casting pointers to integer of a different size. */
9948 tree itype
= signed_type_for (type
);
9949 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
9950 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
9954 t1
= fold_convert (fd
->iter_type
, t1
);
9955 t0
= fold_convert (fd
->iter_type
, t0
);
9959 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
9960 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
9963 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
9964 OMP_CLAUSE__LOOPTEMP_
);
9965 gcc_assert (innerc
);
9966 tree startvar
= OMP_CLAUSE_DECL (innerc
);
9967 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
9968 gcc_assert (innerc
);
9969 tree endvar
= OMP_CLAUSE_DECL (innerc
);
9970 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
9972 gcc_assert (innerc
);
9973 for (i
= 1; i
< fd
->collapse
; i
++)
9975 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9976 OMP_CLAUSE__LOOPTEMP_
);
9977 gcc_assert (innerc
);
9979 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9980 OMP_CLAUSE__LOOPTEMP_
);
9983 /* If needed (inner taskloop has lastprivate clause), propagate
9984 down the total number of iterations. */
9985 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
9987 GSI_CONTINUE_LINKING
);
9988 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9989 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9993 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
9994 GSI_CONTINUE_LINKING
);
9995 assign_stmt
= gimple_build_assign (startvar
, t0
);
9996 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9998 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
9999 GSI_CONTINUE_LINKING
);
10000 assign_stmt
= gimple_build_assign (endvar
, t1
);
10001 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10002 if (fd
->collapse
> 1)
10003 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10005 /* Remove the GIMPLE_OMP_FOR statement. */
10006 gsi
= gsi_for_stmt (for_stmt
);
10007 gsi_remove (&gsi
, true);
10009 gsi
= gsi_last_bb (cont_bb
);
10010 gsi_remove (&gsi
, true);
10012 gsi
= gsi_last_bb (exit_bb
);
10013 gsi_remove (&gsi
, true);
10015 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10016 remove_edge (BRANCH_EDGE (entry_bb
));
10017 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10018 remove_edge (BRANCH_EDGE (cont_bb
));
10019 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10020 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10021 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10024 /* Taskloop construct is represented after gimplification with
10025 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10026 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10027 GOMP_taskloop{,_ull} function arranges for each task to be given just
10028 a single range of iterations. */
10031 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10032 struct omp_for_data
*fd
,
10033 gimple
*inner_stmt
)
10035 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10036 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10037 basic_block fin_bb
;
10038 gimple_stmt_iterator gsi
;
10040 bool broken_loop
= region
->cont
== NULL
;
10041 tree
*counts
= NULL
;
10044 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10045 if (POINTER_TYPE_P (type
))
10046 itype
= signed_type_for (type
);
10048 /* See if we need to bias by LLONG_MIN. */
10049 if (fd
->iter_type
== long_long_unsigned_type_node
10050 && TREE_CODE (type
) == INTEGER_TYPE
10051 && !TYPE_UNSIGNED (type
))
10055 if (fd
->loop
.cond_code
== LT_EXPR
)
10058 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10062 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10065 if (TREE_CODE (n1
) != INTEGER_CST
10066 || TREE_CODE (n2
) != INTEGER_CST
10067 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10068 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10071 entry_bb
= region
->entry
;
10072 cont_bb
= region
->cont
;
10073 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10074 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10075 gcc_assert (broken_loop
10076 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10077 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10080 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10081 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10083 exit_bb
= region
->exit
;
10085 /* Iteration space partitioning goes in ENTRY_BB. */
10086 gsi
= gsi_last_bb (entry_bb
);
10087 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10089 if (fd
->collapse
> 1)
10091 int first_zero_iter
= -1, dummy
= -1;
10092 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10094 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10095 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10096 fin_bb
, first_zero_iter
,
10097 dummy_bb
, dummy
, l2_dom_bb
);
10101 t
= integer_one_node
;
10103 step
= fd
->loop
.step
;
10104 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10105 OMP_CLAUSE__LOOPTEMP_
);
10106 gcc_assert (innerc
);
10107 n1
= OMP_CLAUSE_DECL (innerc
);
10108 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10109 gcc_assert (innerc
);
10110 n2
= OMP_CLAUSE_DECL (innerc
);
10113 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10114 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10116 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10117 true, NULL_TREE
, true, GSI_SAME_STMT
);
10118 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10119 true, NULL_TREE
, true, GSI_SAME_STMT
);
10120 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10121 true, NULL_TREE
, true, GSI_SAME_STMT
);
10123 tree startvar
= fd
->loop
.v
;
10124 tree endvar
= NULL_TREE
;
10126 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10128 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10129 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10130 gcc_assert (innerc
);
10131 startvar
= OMP_CLAUSE_DECL (innerc
);
10132 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10133 OMP_CLAUSE__LOOPTEMP_
);
10134 gcc_assert (innerc
);
10135 endvar
= OMP_CLAUSE_DECL (innerc
);
10137 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10138 t
= force_gimple_operand_gsi (&gsi
, t
,
10140 && TREE_ADDRESSABLE (startvar
),
10141 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10142 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10143 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10145 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10146 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10147 false, GSI_CONTINUE_LINKING
);
10150 assign_stmt
= gimple_build_assign (endvar
, e
);
10151 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10152 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10153 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10155 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10156 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10158 if (fd
->collapse
> 1)
10159 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10163 /* The code controlling the sequential loop replaces the
10164 GIMPLE_OMP_CONTINUE. */
10165 gsi
= gsi_last_bb (cont_bb
);
10166 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10167 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10168 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10169 vback
= gimple_omp_continue_control_def (cont_stmt
);
10171 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10173 if (POINTER_TYPE_P (type
))
10174 t
= fold_build_pointer_plus (vmain
, step
);
10176 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10177 t
= force_gimple_operand_gsi (&gsi
, t
,
10179 && TREE_ADDRESSABLE (vback
),
10180 NULL_TREE
, true, GSI_SAME_STMT
);
10181 assign_stmt
= gimple_build_assign (vback
, t
);
10182 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10184 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10185 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10187 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10190 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10191 gsi_remove (&gsi
, true);
10193 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10194 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10197 /* Remove the GIMPLE_OMP_FOR statement. */
10198 gsi
= gsi_for_stmt (fd
->for_stmt
);
10199 gsi_remove (&gsi
, true);
10201 /* Remove the GIMPLE_OMP_RETURN statement. */
10202 gsi
= gsi_last_bb (exit_bb
);
10203 gsi_remove (&gsi
, true);
10205 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10207 remove_edge (BRANCH_EDGE (entry_bb
));
10210 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
10211 region
->outer
->cont
= NULL
;
10214 /* Connect all the blocks. */
10217 ep
= find_edge (cont_bb
, body_bb
);
10218 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10223 else if (fd
->collapse
> 1)
10226 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10229 ep
->flags
= EDGE_TRUE_VALUE
;
10230 find_edge (cont_bb
, fin_bb
)->flags
10231 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10234 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10235 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10237 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10238 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10240 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
10242 struct loop
*loop
= alloc_loop ();
10243 loop
->header
= body_bb
;
10244 if (collapse_bb
== NULL
)
10245 loop
->latch
= cont_bb
;
10246 add_loop (loop
, body_bb
->loop_father
);
10250 /* Expand the OMP loop defined by REGION. */
10253 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
10255 struct omp_for_data fd
;
10256 struct omp_for_data_loop
*loops
;
10259 = (struct omp_for_data_loop
*)
10260 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
10261 * sizeof (struct omp_for_data_loop
));
10262 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
10264 region
->sched_kind
= fd
.sched_kind
;
10266 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
10267 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
10268 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
10271 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
10272 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
10273 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
10276 /* If there isn't a continue then this is a degerate case where
10277 the introduction of abnormal edges during lowering will prevent
10278 original loops from being detected. Fix that up. */
10279 loops_state_set (LOOPS_NEED_FIXUP
);
10281 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
10282 expand_omp_simd (region
, &fd
);
10283 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
10284 expand_cilk_for (region
, &fd
);
10285 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
10287 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
10288 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
10290 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
10292 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
10293 && !fd
.have_ordered
)
10295 if (fd
.chunk_size
== NULL
)
10296 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
10298 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
10302 int fn_index
, start_ix
, next_ix
;
10304 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
10305 == GF_OMP_FOR_KIND_FOR
);
10306 if (fd
.chunk_size
== NULL
10307 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
10308 fd
.chunk_size
= integer_zero_node
;
10309 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
10310 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
10311 ? 3 : fd
.sched_kind
;
10313 fn_index
+= fd
.have_ordered
* 4;
10315 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
10317 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
10318 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
10319 if (fd
.iter_type
== long_long_unsigned_type_node
)
10321 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
10322 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
10323 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
10324 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
10326 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
10327 (enum built_in_function
) next_ix
, inner_stmt
);
10330 if (gimple_in_ssa_p (cfun
))
10331 update_ssa (TODO_update_ssa_only_virtuals
);
10335 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
10337 v = GOMP_sections_start (n);
10354 v = GOMP_sections_next ();
10359 If this is a combined parallel sections, replace the call to
10360 GOMP_sections_start with call to GOMP_sections_next. */
10363 expand_omp_sections (struct omp_region
*region
)
10365 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
10367 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
10368 gimple_stmt_iterator si
, switch_si
;
10369 gomp_sections
*sections_stmt
;
10371 gomp_continue
*cont
;
10374 struct omp_region
*inner
;
10376 bool exit_reachable
= region
->cont
!= NULL
;
10378 gcc_assert (region
->exit
!= NULL
);
10379 entry_bb
= region
->entry
;
10380 l0_bb
= single_succ (entry_bb
);
10381 l1_bb
= region
->cont
;
10382 l2_bb
= region
->exit
;
10383 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
10384 l2
= gimple_block_label (l2_bb
);
10387 /* This can happen if there are reductions. */
10388 len
= EDGE_COUNT (l0_bb
->succs
);
10389 gcc_assert (len
> 0);
10390 e
= EDGE_SUCC (l0_bb
, len
- 1);
10391 si
= gsi_last_bb (e
->dest
);
10394 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
10395 l2
= gimple_block_label (e
->dest
);
10397 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
10399 si
= gsi_last_bb (e
->dest
);
10401 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
10403 l2
= gimple_block_label (e
->dest
);
10408 if (exit_reachable
)
10409 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
10411 default_bb
= create_empty_bb (l0_bb
);
10413 /* We will build a switch() with enough cases for all the
10414 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
10415 and a default case to abort if something goes wrong. */
10416 len
= EDGE_COUNT (l0_bb
->succs
);
10418 /* Use vec::quick_push on label_vec throughout, since we know the size
10420 auto_vec
<tree
> label_vec (len
);
10422 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
10423 GIMPLE_OMP_SECTIONS statement. */
10424 si
= gsi_last_bb (entry_bb
);
10425 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
10426 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
10427 vin
= gimple_omp_sections_control (sections_stmt
);
10428 if (!is_combined_parallel (region
))
10430 /* If we are not inside a combined parallel+sections region,
10431 call GOMP_sections_start. */
10432 t
= build_int_cst (unsigned_type_node
, len
- 1);
10433 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
10434 stmt
= gimple_build_call (u
, 1, t
);
10438 /* Otherwise, call GOMP_sections_next. */
10439 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
10440 stmt
= gimple_build_call (u
, 0);
10442 gimple_call_set_lhs (stmt
, vin
);
10443 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
10444 gsi_remove (&si
, true);
10446 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
10448 switch_si
= gsi_last_bb (l0_bb
);
10449 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
10450 if (exit_reachable
)
10452 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
10453 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
10454 vmain
= gimple_omp_continue_control_use (cont
);
10455 vnext
= gimple_omp_continue_control_def (cont
);
10463 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
10464 label_vec
.quick_push (t
);
10467 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
10468 for (inner
= region
->inner
, casei
= 1;
10470 inner
= inner
->next
, i
++, casei
++)
10472 basic_block s_entry_bb
, s_exit_bb
;
10474 /* Skip optional reduction region. */
10475 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
10482 s_entry_bb
= inner
->entry
;
10483 s_exit_bb
= inner
->exit
;
10485 t
= gimple_block_label (s_entry_bb
);
10486 u
= build_int_cst (unsigned_type_node
, casei
);
10487 u
= build_case_label (u
, NULL
, t
);
10488 label_vec
.quick_push (u
);
10490 si
= gsi_last_bb (s_entry_bb
);
10491 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
10492 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
10493 gsi_remove (&si
, true);
10494 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
10496 if (s_exit_bb
== NULL
)
10499 si
= gsi_last_bb (s_exit_bb
);
10500 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
10501 gsi_remove (&si
, true);
10503 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
10506 /* Error handling code goes in DEFAULT_BB. */
10507 t
= gimple_block_label (default_bb
);
10508 u
= build_case_label (NULL
, NULL
, t
);
10509 make_edge (l0_bb
, default_bb
, 0);
10510 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
10512 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
10513 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
10514 gsi_remove (&switch_si
, true);
10516 si
= gsi_start_bb (default_bb
);
10517 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
10518 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
10520 if (exit_reachable
)
10524 /* Code to get the next section goes in L1_BB. */
10525 si
= gsi_last_bb (l1_bb
);
10526 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
10528 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
10529 stmt
= gimple_build_call (bfn_decl
, 0);
10530 gimple_call_set_lhs (stmt
, vnext
);
10531 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
10532 gsi_remove (&si
, true);
10534 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
10537 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
10538 si
= gsi_last_bb (l2_bb
);
10539 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
10540 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
10541 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
10542 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
10544 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
10545 stmt
= gimple_build_call (t
, 0);
10546 if (gimple_omp_return_lhs (gsi_stmt (si
)))
10547 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
10548 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
10549 gsi_remove (&si
, true);
10551 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
10555 /* Expand code for an OpenMP single directive. We've already expanded
10556 much of the code, here we simply place the GOMP_barrier call. */
10559 expand_omp_single (struct omp_region
*region
)
10561 basic_block entry_bb
, exit_bb
;
10562 gimple_stmt_iterator si
;
10564 entry_bb
= region
->entry
;
10565 exit_bb
= region
->exit
;
10567 si
= gsi_last_bb (entry_bb
);
10568 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
10569 gsi_remove (&si
, true);
10570 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10572 si
= gsi_last_bb (exit_bb
);
10573 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
10575 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
10576 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
10578 gsi_remove (&si
, true);
10579 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
10583 /* Generic expansion for OpenMP synchronization directives: master,
10584 ordered and critical. All we need to do here is remove the entry
10585 and exit markers for REGION. */
10588 expand_omp_synch (struct omp_region
*region
)
10590 basic_block entry_bb
, exit_bb
;
10591 gimple_stmt_iterator si
;
10593 entry_bb
= region
->entry
;
10594 exit_bb
= region
->exit
;
10596 si
= gsi_last_bb (entry_bb
);
10597 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
10598 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
10599 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
10600 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
10601 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
10602 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
10603 gsi_remove (&si
, true);
10604 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10608 si
= gsi_last_bb (exit_bb
);
10609 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
10610 gsi_remove (&si
, true);
10611 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
10615 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
10616 operation as a normal volatile load. */
10619 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
10620 tree loaded_val
, int index
)
10622 enum built_in_function tmpbase
;
10623 gimple_stmt_iterator gsi
;
10624 basic_block store_bb
;
10627 tree decl
, call
, type
, itype
;
10629 gsi
= gsi_last_bb (load_bb
);
10630 stmt
= gsi_stmt (gsi
);
10631 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
10632 loc
= gimple_location (stmt
);
10634 /* ??? If the target does not implement atomic_load_optab[mode], and mode
10635 is smaller than word size, then expand_atomic_load assumes that the load
10636 is atomic. We could avoid the builtin entirely in this case. */
10638 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
10639 decl
= builtin_decl_explicit (tmpbase
);
10640 if (decl
== NULL_TREE
)
10643 type
= TREE_TYPE (loaded_val
);
10644 itype
= TREE_TYPE (TREE_TYPE (decl
));
10646 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
10647 build_int_cst (NULL
,
10648 gimple_omp_atomic_seq_cst_p (stmt
)
10650 : MEMMODEL_RELAXED
));
10651 if (!useless_type_conversion_p (type
, itype
))
10652 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
10653 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
10655 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10656 gsi_remove (&gsi
, true);
10658 store_bb
= single_succ (load_bb
);
10659 gsi
= gsi_last_bb (store_bb
);
10660 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
10661 gsi_remove (&gsi
, true);
10663 if (gimple_in_ssa_p (cfun
))
10664 update_ssa (TODO_update_ssa_no_phi
);
10669 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
10670 operation as a normal volatile store. */
10673 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
10674 tree loaded_val
, tree stored_val
, int index
)
10676 enum built_in_function tmpbase
;
10677 gimple_stmt_iterator gsi
;
10678 basic_block store_bb
= single_succ (load_bb
);
10681 tree decl
, call
, type
, itype
;
10682 machine_mode imode
;
10685 gsi
= gsi_last_bb (load_bb
);
10686 stmt
= gsi_stmt (gsi
);
10687 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
10689 /* If the load value is needed, then this isn't a store but an exchange. */
10690 exchange
= gimple_omp_atomic_need_value_p (stmt
);
10692 gsi
= gsi_last_bb (store_bb
);
10693 stmt
= gsi_stmt (gsi
);
10694 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
10695 loc
= gimple_location (stmt
);
10697 /* ??? If the target does not implement atomic_store_optab[mode], and mode
10698 is smaller than word size, then expand_atomic_store assumes that the store
10699 is atomic. We could avoid the builtin entirely in this case. */
10701 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
10702 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
10703 decl
= builtin_decl_explicit (tmpbase
);
10704 if (decl
== NULL_TREE
)
10707 type
= TREE_TYPE (stored_val
);
10709 /* Dig out the type of the function's second argument. */
10710 itype
= TREE_TYPE (decl
);
10711 itype
= TYPE_ARG_TYPES (itype
);
10712 itype
= TREE_CHAIN (itype
);
10713 itype
= TREE_VALUE (itype
);
10714 imode
= TYPE_MODE (itype
);
10716 if (exchange
&& !can_atomic_exchange_p (imode
, true))
10719 if (!useless_type_conversion_p (itype
, type
))
10720 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
10721 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
10722 build_int_cst (NULL
,
10723 gimple_omp_atomic_seq_cst_p (stmt
)
10725 : MEMMODEL_RELAXED
));
10728 if (!useless_type_conversion_p (type
, itype
))
10729 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
10730 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
10733 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10734 gsi_remove (&gsi
, true);
10736 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
10737 gsi
= gsi_last_bb (load_bb
);
10738 gsi_remove (&gsi
, true);
10740 if (gimple_in_ssa_p (cfun
))
10741 update_ssa (TODO_update_ssa_no_phi
);
10746 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
10747 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
10748 size of the data type, and thus usable to find the index of the builtin
10749 decl. Returns false if the expression is not of the proper form. */
10752 expand_omp_atomic_fetch_op (basic_block load_bb
,
10753 tree addr
, tree loaded_val
,
10754 tree stored_val
, int index
)
10756 enum built_in_function oldbase
, newbase
, tmpbase
;
10757 tree decl
, itype
, call
;
10759 basic_block store_bb
= single_succ (load_bb
);
10760 gimple_stmt_iterator gsi
;
10763 enum tree_code code
;
10764 bool need_old
, need_new
;
10765 machine_mode imode
;
10768 /* We expect to find the following sequences:
10771 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
10774 val = tmp OP something; (or: something OP tmp)
10775 GIMPLE_OMP_STORE (val)
10777 ???FIXME: Allow a more flexible sequence.
10778 Perhaps use data flow to pick the statements.
10782 gsi
= gsi_after_labels (store_bb
);
10783 stmt
= gsi_stmt (gsi
);
10784 loc
= gimple_location (stmt
);
10785 if (!is_gimple_assign (stmt
))
10788 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
10790 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
10791 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
10792 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
10793 gcc_checking_assert (!need_old
|| !need_new
);
10795 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
10798 /* Check for one of the supported fetch-op operations. */
10799 code
= gimple_assign_rhs_code (stmt
);
10803 case POINTER_PLUS_EXPR
:
10804 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
10805 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
10808 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
10809 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
10812 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
10813 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
10816 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
10817 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
10820 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
10821 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
10827 /* Make sure the expression is of the proper form. */
10828 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
10829 rhs
= gimple_assign_rhs2 (stmt
);
10830 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
10831 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
10832 rhs
= gimple_assign_rhs1 (stmt
);
10836 tmpbase
= ((enum built_in_function
)
10837 ((need_new
? newbase
: oldbase
) + index
+ 1));
10838 decl
= builtin_decl_explicit (tmpbase
);
10839 if (decl
== NULL_TREE
)
10841 itype
= TREE_TYPE (TREE_TYPE (decl
));
10842 imode
= TYPE_MODE (itype
);
10844 /* We could test all of the various optabs involved, but the fact of the
10845 matter is that (with the exception of i486 vs i586 and xadd) all targets
10846 that support any atomic operaton optab also implements compare-and-swap.
10847 Let optabs.c take care of expanding any compare-and-swap loop. */
10848 if (!can_compare_and_swap_p (imode
, true))
10851 gsi
= gsi_last_bb (load_bb
);
10852 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
10854 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
10855 It only requires that the operation happen atomically. Thus we can
10856 use the RELAXED memory model. */
10857 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
10858 fold_convert_loc (loc
, itype
, rhs
),
10859 build_int_cst (NULL
,
10860 seq_cst
? MEMMODEL_SEQ_CST
10861 : MEMMODEL_RELAXED
));
10863 if (need_old
|| need_new
)
10865 lhs
= need_old
? loaded_val
: stored_val
;
10866 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
10867 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
10870 call
= fold_convert_loc (loc
, void_type_node
, call
);
10871 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10872 gsi_remove (&gsi
, true);
10874 gsi
= gsi_last_bb (store_bb
);
10875 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
10876 gsi_remove (&gsi
, true);
10877 gsi
= gsi_last_bb (store_bb
);
10878 gsi_remove (&gsi
, true);
10880 if (gimple_in_ssa_p (cfun
))
10881 update_ssa (TODO_update_ssa_no_phi
);
10886 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
10890 newval = rhs; // with oldval replacing *addr in rhs
10891 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
10892 if (oldval != newval)
10895 INDEX is log2 of the size of the data type, and thus usable to find the
10896 index of the builtin decl. */
10899 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
10900 tree addr
, tree loaded_val
, tree stored_val
,
10903 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
10904 tree type
, itype
, cmpxchg
, iaddr
;
10905 gimple_stmt_iterator si
;
10906 basic_block loop_header
= single_succ (load_bb
);
10907 gimple
*phi
, *stmt
;
10909 enum built_in_function fncode
;
10911 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
10912 order to use the RELAXED memory model effectively. */
10913 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
10915 cmpxchg
= builtin_decl_explicit (fncode
);
10916 if (cmpxchg
== NULL_TREE
)
10918 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
10919 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
10921 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
10924 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
10925 si
= gsi_last_bb (load_bb
);
10926 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
10928 /* For floating-point values, we'll need to view-convert them to integers
10929 so that we can perform the atomic compare and swap. Simplify the
10930 following code by always setting up the "i"ntegral variables. */
10931 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
10935 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
10938 = force_gimple_operand_gsi (&si
,
10939 fold_convert (TREE_TYPE (iaddr
), addr
),
10940 false, NULL_TREE
, true, GSI_SAME_STMT
);
10941 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
10942 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
10943 loadedi
= create_tmp_var (itype
);
10944 if (gimple_in_ssa_p (cfun
))
10945 loadedi
= make_ssa_name (loadedi
);
10950 loadedi
= loaded_val
;
10953 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
10954 tree loaddecl
= builtin_decl_explicit (fncode
);
10957 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
10958 build_call_expr (loaddecl
, 2, iaddr
,
10959 build_int_cst (NULL_TREE
,
10960 MEMMODEL_RELAXED
)));
10962 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
10963 build_int_cst (TREE_TYPE (iaddr
), 0));
10966 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
10969 /* Move the value to the LOADEDI temporary. */
10970 if (gimple_in_ssa_p (cfun
))
10972 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
10973 phi
= create_phi_node (loadedi
, loop_header
);
10974 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
10978 gsi_insert_before (&si
,
10979 gimple_build_assign (loadedi
, initial
),
10981 if (loadedi
!= loaded_val
)
10983 gimple_stmt_iterator gsi2
;
10986 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
10987 gsi2
= gsi_start_bb (loop_header
);
10988 if (gimple_in_ssa_p (cfun
))
10991 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
10992 true, GSI_SAME_STMT
);
10993 stmt
= gimple_build_assign (loaded_val
, x
);
10994 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
10998 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
10999 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
11000 true, GSI_SAME_STMT
);
11003 gsi_remove (&si
, true);
11005 si
= gsi_last_bb (store_bb
);
11006 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
11009 storedi
= stored_val
;
11012 force_gimple_operand_gsi (&si
,
11013 build1 (VIEW_CONVERT_EXPR
, itype
,
11014 stored_val
), true, NULL_TREE
, true,
11017 /* Build the compare&swap statement. */
11018 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
11019 new_storedi
= force_gimple_operand_gsi (&si
,
11020 fold_convert (TREE_TYPE (loadedi
),
11023 true, GSI_SAME_STMT
);
11025 if (gimple_in_ssa_p (cfun
))
11026 old_vali
= loadedi
;
11029 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
11030 stmt
= gimple_build_assign (old_vali
, loadedi
);
11031 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11033 stmt
= gimple_build_assign (loadedi
, new_storedi
);
11034 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11037 /* Note that we always perform the comparison as an integer, even for
11038 floating point. This allows the atomic operation to properly
11039 succeed even with NaNs and -0.0. */
11040 stmt
= gimple_build_cond_empty
11041 (build2 (NE_EXPR
, boolean_type_node
,
11042 new_storedi
, old_vali
));
11043 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11046 e
= single_succ_edge (store_bb
);
11047 e
->flags
&= ~EDGE_FALLTHRU
;
11048 e
->flags
|= EDGE_FALSE_VALUE
;
11050 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
11052 /* Copy the new value to loadedi (we already did that before the condition
11053 if we are not in SSA). */
11054 if (gimple_in_ssa_p (cfun
))
11056 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
11057 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
11060 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
11061 gsi_remove (&si
, true);
11063 struct loop
*loop
= alloc_loop ();
11064 loop
->header
= loop_header
;
11065 loop
->latch
= store_bb
;
11066 add_loop (loop
, loop_header
->loop_father
);
11068 if (gimple_in_ssa_p (cfun
))
11069 update_ssa (TODO_update_ssa_no_phi
);
11074 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
11076 GOMP_atomic_start ();
11078 GOMP_atomic_end ();
11080 The result is not globally atomic, but works so long as all parallel
11081 references are within #pragma omp atomic directives. According to
11082 responses received from omp@openmp.org, appears to be within spec.
11083 Which makes sense, since that's how several other compilers handle
11084 this situation as well.
11085 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
11086 expanding. STORED_VAL is the operand of the matching
11087 GIMPLE_OMP_ATOMIC_STORE.
11090 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
11091 loaded_val = *addr;
11094 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
11095 *addr = stored_val;
11099 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
11100 tree addr
, tree loaded_val
, tree stored_val
)
11102 gimple_stmt_iterator si
;
11106 si
= gsi_last_bb (load_bb
);
11107 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11109 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
11110 t
= build_call_expr (t
, 0);
11111 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11113 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
11114 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11115 gsi_remove (&si
, true);
11117 si
= gsi_last_bb (store_bb
);
11118 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
11120 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
11122 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11124 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
11125 t
= build_call_expr (t
, 0);
11126 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11127 gsi_remove (&si
, true);
11129 if (gimple_in_ssa_p (cfun
))
11130 update_ssa (TODO_update_ssa_no_phi
);
11134 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
11135 using expand_omp_atomic_fetch_op. If it failed, we try to
11136 call expand_omp_atomic_pipeline, and if it fails too, the
11137 ultimate fallback is wrapping the operation in a mutex
11138 (expand_omp_atomic_mutex). REGION is the atomic region built
11139 by build_omp_regions_1(). */
11142 expand_omp_atomic (struct omp_region
*region
)
11144 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
11145 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
11146 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
11147 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
11148 tree addr
= gimple_omp_atomic_load_rhs (load
);
11149 tree stored_val
= gimple_omp_atomic_store_val (store
);
11150 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
11151 HOST_WIDE_INT index
;
11153 /* Make sure the type is one of the supported sizes. */
11154 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
11155 index
= exact_log2 (index
);
11156 if (index
>= 0 && index
<= 4)
11158 unsigned int align
= TYPE_ALIGN_UNIT (type
);
11160 /* __sync builtins require strict data alignment. */
11161 if (exact_log2 (align
) >= index
)
11164 if (loaded_val
== stored_val
11165 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
11166 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
11167 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
11168 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
11171 /* Atomic store. */
11172 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
11173 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
11174 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
11175 && store_bb
== single_succ (load_bb
)
11176 && first_stmt (store_bb
) == store
11177 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
11178 stored_val
, index
))
11181 /* When possible, use specialized atomic update functions. */
11182 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
11183 && store_bb
== single_succ (load_bb
)
11184 && expand_omp_atomic_fetch_op (load_bb
, addr
,
11185 loaded_val
, stored_val
, index
))
11188 /* If we don't have specialized __sync builtins, try and implement
11189 as a compare and swap loop. */
11190 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
11191 loaded_val
, stored_val
, index
))
11196 /* The ultimate fallback is wrapping the operation in a mutex. */
11197 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
11201 /* Encode an oacc launc argument. This matches the GOMP_LAUNCH_PACK
11202 macro on gomp-constants.h. We do not check for overflow. */
11205 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
11209 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
11212 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
11213 device
, build_int_cst (unsigned_type_node
,
11214 GOMP_LAUNCH_DEVICE_SHIFT
));
11215 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
11220 /* Look for compute grid dimension clauses and convert to an attribute
11221 attached to FN. This permits the target-side code to (a) massage
11222 the dimensions, (b) emit that data and (c) optimize. Non-constant
11223 dimensions are pushed onto ARGS.
11225 The attribute value is a TREE_LIST. A set of dimensions is
11226 represented as a list of INTEGER_CST. Those that are runtime
11227 expres are represented as an INTEGER_CST of zero.
11229 TOOO. Normally the attribute will just contain a single such list. If
11230 however it contains a list of lists, this will represent the use of
11231 device_type. Each member of the outer list is an assoc list of
11232 dimensions, keyed by the device type. The first entry will be the
11233 default. Well, that's the plan. */
11235 #define OACC_FN_ATTRIB "oacc function"
11237 /* Replace any existing oacc fn attribute with updated dimensions. */
11240 replace_oacc_fn_attrib (tree fn
, tree dims
)
11242 tree ident
= get_identifier (OACC_FN_ATTRIB
);
11243 tree attribs
= DECL_ATTRIBUTES (fn
);
11245 /* If we happen to be present as the first attrib, drop it. */
11246 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
11247 attribs
= TREE_CHAIN (attribs
);
11248 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
11251 /* Scan CLAUSES for launch dimensions and attach them to the oacc
11252 function attribute. Push any that are non-constant onto the ARGS
11253 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
11256 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
11258 /* Must match GOMP_DIM ordering. */
11259 static const omp_clause_code ids
[]
11260 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
11261 OMP_CLAUSE_VECTOR_LENGTH
};
11263 tree dims
[GOMP_DIM_MAX
];
11264 tree attr
= NULL_TREE
;
11265 unsigned non_const
= 0;
11267 for (ix
= GOMP_DIM_MAX
; ix
--;)
11269 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
11270 tree dim
= NULL_TREE
;
11273 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
11275 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
11277 dim
= integer_zero_node
;
11278 non_const
|= GOMP_DIM_MASK (ix
);
11280 attr
= tree_cons (NULL_TREE
, dim
, attr
);
11283 replace_oacc_fn_attrib (fn
, attr
);
11287 /* Push a dynamic argument set. */
11288 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
11289 NULL_TREE
, non_const
));
11290 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
11291 if (non_const
& GOMP_DIM_MASK (ix
))
11292 args
->safe_push (dims
[ix
]);
11296 /* Retrieve the oacc function attrib and return it. Non-oacc
11297 functions will return NULL. */
11300 get_oacc_fn_attrib (tree fn
)
11302 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
11305 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
11308 expand_omp_target (struct omp_region
*region
)
11310 basic_block entry_bb
, exit_bb
, new_bb
;
11311 struct function
*child_cfun
;
11312 tree child_fn
, block
, t
;
11313 gimple_stmt_iterator gsi
;
11314 gomp_target
*entry_stmt
;
11317 bool offloaded
, data_region
;
11319 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
11320 new_bb
= region
->entry
;
11322 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
11323 switch (gimple_omp_target_kind (entry_stmt
))
11325 case GF_OMP_TARGET_KIND_REGION
:
11326 case GF_OMP_TARGET_KIND_UPDATE
:
11327 case GF_OMP_TARGET_KIND_ENTER_DATA
:
11328 case GF_OMP_TARGET_KIND_EXIT_DATA
:
11329 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11330 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11331 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11332 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11333 data_region
= false;
11335 case GF_OMP_TARGET_KIND_DATA
:
11336 case GF_OMP_TARGET_KIND_OACC_DATA
:
11337 data_region
= true;
11340 gcc_unreachable ();
11343 child_fn
= NULL_TREE
;
11347 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
11348 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
11351 /* Supported by expand_omp_taskreg, but not here. */
11352 if (child_cfun
!= NULL
)
11353 gcc_checking_assert (!child_cfun
->cfg
);
11354 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
11356 entry_bb
= region
->entry
;
11357 exit_bb
= region
->exit
;
11361 unsigned srcidx
, dstidx
, num
;
11363 /* If the offloading region needs data sent from the parent
11364 function, then the very first statement (except possible
11365 tree profile counter updates) of the offloading body
11366 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
11367 &.OMP_DATA_O is passed as an argument to the child function,
11368 we need to replace it with the argument as seen by the child
11371 In most cases, this will end up being the identity assignment
11372 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
11373 a function call that has been inlined, the original PARM_DECL
11374 .OMP_DATA_I may have been converted into a different local
11375 variable. In which case, we need to keep the assignment. */
11376 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
11379 basic_block entry_succ_bb
= single_succ (entry_bb
);
11380 gimple_stmt_iterator gsi
;
11382 gimple
*tgtcopy_stmt
= NULL
;
11383 tree sender
= TREE_VEC_ELT (data_arg
, 0);
11385 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
11387 gcc_assert (!gsi_end_p (gsi
));
11388 stmt
= gsi_stmt (gsi
);
11389 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
11392 if (gimple_num_ops (stmt
) == 2)
11394 tree arg
= gimple_assign_rhs1 (stmt
);
11396 /* We're ignoring the subcode because we're
11397 effectively doing a STRIP_NOPS. */
11399 if (TREE_CODE (arg
) == ADDR_EXPR
11400 && TREE_OPERAND (arg
, 0) == sender
)
11402 tgtcopy_stmt
= stmt
;
11408 gcc_assert (tgtcopy_stmt
!= NULL
);
11409 arg
= DECL_ARGUMENTS (child_fn
);
11411 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
11412 gsi_remove (&gsi
, true);
11415 /* Declare local variables needed in CHILD_CFUN. */
11416 block
= DECL_INITIAL (child_fn
);
11417 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
11418 /* The gimplifier could record temporaries in the offloading block
11419 rather than in containing function's local_decls chain,
11420 which would mean cgraph missed finalizing them. Do it now. */
11421 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
11422 if (TREE_CODE (t
) == VAR_DECL
11424 && !DECL_EXTERNAL (t
))
11425 varpool_node::finalize_decl (t
);
11426 DECL_SAVED_TREE (child_fn
) = NULL
;
11427 /* We'll create a CFG for child_fn, so no gimple body is needed. */
11428 gimple_set_body (child_fn
, NULL
);
11429 TREE_USED (block
) = 1;
11431 /* Reset DECL_CONTEXT on function arguments. */
11432 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
11433 DECL_CONTEXT (t
) = child_fn
;
11435 /* Split ENTRY_BB at GIMPLE_*,
11436 so that it can be moved to the child function. */
11437 gsi
= gsi_last_bb (entry_bb
);
11438 stmt
= gsi_stmt (gsi
);
11440 && gimple_code (stmt
) == gimple_code (entry_stmt
));
11441 e
= split_block (entry_bb
, stmt
);
11442 gsi_remove (&gsi
, true);
11443 entry_bb
= e
->dest
;
11444 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11446 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
11449 gsi
= gsi_last_bb (exit_bb
);
11450 gcc_assert (!gsi_end_p (gsi
)
11451 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11452 stmt
= gimple_build_return (NULL
);
11453 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
11454 gsi_remove (&gsi
, true);
11457 /* Move the offloading region into CHILD_CFUN. */
11459 block
= gimple_block (entry_stmt
);
11461 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
11463 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
11464 /* When the OMP expansion process cannot guarantee an up-to-date
11465 loop tree arrange for the child function to fixup loops. */
11466 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
11467 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
11469 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
11470 num
= vec_safe_length (child_cfun
->local_decls
);
11471 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
11473 t
= (*child_cfun
->local_decls
)[srcidx
];
11474 if (DECL_CONTEXT (t
) == cfun
->decl
)
11476 if (srcidx
!= dstidx
)
11477 (*child_cfun
->local_decls
)[dstidx
] = t
;
11481 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
11483 /* Inform the callgraph about the new function. */
11484 child_cfun
->curr_properties
= cfun
->curr_properties
;
11485 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
11486 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
11487 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
11488 node
->parallelized_function
= 1;
11489 cgraph_node::add_new_function (child_fn
, true);
11491 #ifdef ENABLE_OFFLOADING
11492 /* Add the new function to the offload table. */
11493 vec_safe_push (offload_funcs
, child_fn
);
11496 /* Fix the callgraph edges for child_cfun. Those for cfun will be
11497 fixed in a following pass. */
11498 push_cfun (child_cfun
);
11499 cgraph_edge::rebuild_edges ();
11501 #ifdef ENABLE_OFFLOADING
11502 /* Prevent IPA from removing child_fn as unreachable, since there are no
11503 refs from the parent function to child_fn in offload LTO mode. */
11504 cgraph_node::get (child_fn
)->mark_force_output ();
11507 /* Some EH regions might become dead, see PR34608. If
11508 pass_cleanup_cfg isn't the first pass to happen with the
11509 new child, these dead EH edges might cause problems.
11510 Clean them up now. */
11511 if (flag_exceptions
)
11514 bool changed
= false;
11516 FOR_EACH_BB_FN (bb
, cfun
)
11517 changed
|= gimple_purge_dead_eh_edges (bb
);
11519 cleanup_tree_cfg ();
11521 #ifdef ENABLE_CHECKING
11522 if (!loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
11523 verify_loop_structure ();
11528 /* Emit a library call to launch the offloading region, or do data
11530 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
11531 enum built_in_function start_ix
;
11532 location_t clause_loc
;
11533 unsigned int flags_i
= 0;
11535 switch (gimple_omp_target_kind (entry_stmt
))
11537 case GF_OMP_TARGET_KIND_REGION
:
11538 start_ix
= BUILT_IN_GOMP_TARGET
;
11540 case GF_OMP_TARGET_KIND_DATA
:
11541 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
11543 case GF_OMP_TARGET_KIND_UPDATE
:
11544 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
11546 case GF_OMP_TARGET_KIND_ENTER_DATA
:
11547 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
11549 case GF_OMP_TARGET_KIND_EXIT_DATA
:
11550 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
11551 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
11553 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11554 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11555 start_ix
= BUILT_IN_GOACC_PARALLEL
;
11557 case GF_OMP_TARGET_KIND_OACC_DATA
:
11558 start_ix
= BUILT_IN_GOACC_DATA_START
;
11560 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11561 start_ix
= BUILT_IN_GOACC_UPDATE
;
11563 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11564 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
11567 gcc_unreachable ();
11570 clauses
= gimple_omp_target_clauses (entry_stmt
);
11572 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
11573 library choose) and there is no conditional. */
11575 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
11577 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
11579 cond
= OMP_CLAUSE_IF_EXPR (c
);
11581 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
11584 /* Even if we pass it to all library function calls, it is currently only
11585 defined/used for the OpenMP target ones. */
11586 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
11587 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
11588 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
11589 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
11591 device
= OMP_CLAUSE_DEVICE_ID (c
);
11592 clause_loc
= OMP_CLAUSE_LOCATION (c
);
11595 clause_loc
= gimple_location (entry_stmt
);
11597 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
11599 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
11601 /* Ensure 'device' is of the correct type. */
11602 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
11604 /* If we found the clause 'if (cond)', build
11605 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
11608 cond
= gimple_boolify (cond
);
11610 basic_block cond_bb
, then_bb
, else_bb
;
11614 tmp_var
= create_tmp_var (TREE_TYPE (device
));
11616 e
= split_block_after_labels (new_bb
);
11619 gsi
= gsi_last_bb (new_bb
);
11621 e
= split_block (new_bb
, gsi_stmt (gsi
));
11627 then_bb
= create_empty_bb (cond_bb
);
11628 else_bb
= create_empty_bb (then_bb
);
11629 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
11630 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
11632 stmt
= gimple_build_cond_empty (cond
);
11633 gsi
= gsi_last_bb (cond_bb
);
11634 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
11636 gsi
= gsi_start_bb (then_bb
);
11637 stmt
= gimple_build_assign (tmp_var
, device
);
11638 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
11640 gsi
= gsi_start_bb (else_bb
);
11641 stmt
= gimple_build_assign (tmp_var
,
11642 build_int_cst (integer_type_node
,
11643 GOMP_DEVICE_HOST_FALLBACK
));
11644 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
11646 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
11647 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
11648 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
11649 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
11650 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
11651 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
11656 gsi
= gsi_last_bb (new_bb
);
11657 t
= gimple_omp_target_data_arg (entry_stmt
);
11660 t1
= size_zero_node
;
11661 t2
= build_zero_cst (ptr_type_node
);
11667 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
11668 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
11669 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
11670 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
11671 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
11675 bool tagging
= false;
11676 /* The maximum number used by any start_ix, without varargs. */
11677 auto_vec
<tree
, 11> args
;
11678 args
.quick_push (device
);
11680 args
.quick_push (build_fold_addr_expr (child_fn
));
11681 args
.quick_push (t1
);
11682 args
.quick_push (t2
);
11683 args
.quick_push (t3
);
11684 args
.quick_push (t4
);
11687 case BUILT_IN_GOACC_DATA_START
:
11688 case BUILT_IN_GOMP_TARGET_DATA
:
11690 case BUILT_IN_GOMP_TARGET
:
11691 case BUILT_IN_GOMP_TARGET_UPDATE
:
11692 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
11693 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
11694 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
11696 depend
= OMP_CLAUSE_DECL (c
);
11698 depend
= build_int_cst (ptr_type_node
, 0);
11699 args
.quick_push (depend
);
11701 case BUILT_IN_GOACC_PARALLEL
:
11703 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
11707 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
11708 case BUILT_IN_GOACC_UPDATE
:
11710 tree t_async
= NULL_TREE
;
11712 /* If present, use the value specified by the respective
11713 clause, making sure that is of the correct type. */
11714 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
11716 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
11718 OMP_CLAUSE_ASYNC_EXPR (c
));
11720 /* Default values for t_async. */
11721 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
11723 build_int_cst (integer_type_node
,
11725 if (tagging
&& t_async
)
11727 unsigned HOST_WIDE_INT i_async
;
11729 if (TREE_CODE (t_async
) == INTEGER_CST
)
11731 /* See if we can pack the async arg in to the tag's
11733 i_async
= TREE_INT_CST_LOW (t_async
);
11735 if (i_async
< GOMP_LAUNCH_OP_MAX
)
11736 t_async
= NULL_TREE
;
11739 i_async
= GOMP_LAUNCH_OP_MAX
;
11740 args
.safe_push (oacc_launch_pack
11741 (GOMP_LAUNCH_ASYNC
, NULL_TREE
, i_async
));
11744 args
.safe_push (t_async
);
11746 /* Save the argument index, and ... */
11747 unsigned t_wait_idx
= args
.length ();
11748 unsigned num_waits
= 0;
11749 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
11751 /* ... push a placeholder. */
11752 args
.safe_push (integer_zero_node
);
11754 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
11755 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
11757 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
11759 OMP_CLAUSE_WAIT_EXPR (c
)));
11763 if (!tagging
|| num_waits
)
11767 /* Now that we know the number, update the placeholder. */
11769 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
11771 len
= build_int_cst (integer_type_node
, num_waits
);
11772 len
= fold_convert_loc (gimple_location (entry_stmt
),
11773 unsigned_type_node
, len
);
11774 args
[t_wait_idx
] = len
;
11779 gcc_unreachable ();
11782 /* Push terminal marker - zero. */
11783 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
11785 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
11786 gimple_set_location (g
, gimple_location (entry_stmt
));
11787 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11790 g
= gsi_stmt (gsi
);
11791 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
11792 gsi_remove (&gsi
, true);
11794 if (data_region
&& region
->exit
)
11796 gsi
= gsi_last_bb (region
->exit
);
11797 g
= gsi_stmt (gsi
);
11798 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
11799 gsi_remove (&gsi
, true);
11804 /* Expand the parallel region tree rooted at REGION. Expansion
11805 proceeds in depth-first order. Innermost regions are expanded
11806 first. This way, parallel regions that require a new function to
11807 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
11808 internal dependencies in their body. */
11811 expand_omp (struct omp_region
*region
)
11815 location_t saved_location
;
11816 gimple
*inner_stmt
= NULL
;
11818 /* First, determine whether this is a combined parallel+workshare
11820 if (region
->type
== GIMPLE_OMP_PARALLEL
)
11821 determine_parallel_type (region
);
11823 if (region
->type
== GIMPLE_OMP_FOR
11824 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
11825 inner_stmt
= last_stmt (region
->inner
->entry
);
11828 expand_omp (region
->inner
);
11830 saved_location
= input_location
;
11831 if (gimple_has_location (last_stmt (region
->entry
)))
11832 input_location
= gimple_location (last_stmt (region
->entry
));
11834 switch (region
->type
)
11836 case GIMPLE_OMP_PARALLEL
:
11837 case GIMPLE_OMP_TASK
:
11838 expand_omp_taskreg (region
);
11841 case GIMPLE_OMP_FOR
:
11842 expand_omp_for (region
, inner_stmt
);
11845 case GIMPLE_OMP_SECTIONS
:
11846 expand_omp_sections (region
);
11849 case GIMPLE_OMP_SECTION
:
11850 /* Individual omp sections are handled together with their
11851 parent GIMPLE_OMP_SECTIONS region. */
11854 case GIMPLE_OMP_SINGLE
:
11855 expand_omp_single (region
);
11858 case GIMPLE_OMP_ORDERED
:
11860 gomp_ordered
*ord_stmt
11861 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
11862 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
11863 OMP_CLAUSE_DEPEND
))
11865 /* We'll expand these when expanding corresponding
11866 worksharing region with ordered(n) clause. */
11867 gcc_assert (region
->outer
11868 && region
->outer
->type
== GIMPLE_OMP_FOR
);
11869 region
->ord_stmt
= ord_stmt
;
11874 case GIMPLE_OMP_MASTER
:
11875 case GIMPLE_OMP_TASKGROUP
:
11876 case GIMPLE_OMP_CRITICAL
:
11877 case GIMPLE_OMP_TEAMS
:
11878 expand_omp_synch (region
);
11881 case GIMPLE_OMP_ATOMIC_LOAD
:
11882 expand_omp_atomic (region
);
11885 case GIMPLE_OMP_TARGET
:
11886 expand_omp_target (region
);
11890 gcc_unreachable ();
11893 input_location
= saved_location
;
11894 region
= region
->next
;
11899 /* Helper for build_omp_regions. Scan the dominator tree starting at
11900 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
11901 true, the function ends once a single tree is built (otherwise, whole
11902 forest of OMP constructs may be built). */
11905 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
11908 gimple_stmt_iterator gsi
;
11912 gsi
= gsi_last_bb (bb
);
11913 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
11915 struct omp_region
*region
;
11916 enum gimple_code code
;
11918 stmt
= gsi_stmt (gsi
);
11919 code
= gimple_code (stmt
);
11920 if (code
== GIMPLE_OMP_RETURN
)
11922 /* STMT is the return point out of region PARENT. Mark it
11923 as the exit point and make PARENT the immediately
11924 enclosing region. */
11925 gcc_assert (parent
);
11928 parent
= parent
->outer
;
11930 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
11932 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
11933 GIMPLE_OMP_RETURN, but matches with
11934 GIMPLE_OMP_ATOMIC_LOAD. */
11935 gcc_assert (parent
);
11936 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
11939 parent
= parent
->outer
;
11941 else if (code
== GIMPLE_OMP_CONTINUE
)
11943 gcc_assert (parent
);
11946 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
11948 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
11949 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
11953 region
= new_omp_region (bb
, code
, parent
);
11955 if (code
== GIMPLE_OMP_TARGET
)
11957 switch (gimple_omp_target_kind (stmt
))
11959 case GF_OMP_TARGET_KIND_REGION
:
11960 case GF_OMP_TARGET_KIND_DATA
:
11961 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11962 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11963 case GF_OMP_TARGET_KIND_OACC_DATA
:
11965 case GF_OMP_TARGET_KIND_UPDATE
:
11966 case GF_OMP_TARGET_KIND_ENTER_DATA
:
11967 case GF_OMP_TARGET_KIND_EXIT_DATA
:
11968 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11969 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11970 /* ..., other than for those stand-alone directives... */
11974 gcc_unreachable ();
11977 else if (code
== GIMPLE_OMP_ORDERED
11978 && find_omp_clause (gimple_omp_ordered_clauses
11979 (as_a
<gomp_ordered
*> (stmt
)),
11980 OMP_CLAUSE_DEPEND
))
11981 /* #pragma omp ordered depend is also just a stand-alone
11984 /* ..., this directive becomes the parent for a new region. */
11990 if (single_tree
&& !parent
)
11993 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
11995 son
= next_dom_son (CDI_DOMINATORS
, son
))
11996 build_omp_regions_1 (son
, parent
, single_tree
);
11999 /* Builds the tree of OMP regions rooted at ROOT, storing it to
12000 root_omp_region. */
12003 build_omp_regions_root (basic_block root
)
12005 gcc_assert (root_omp_region
== NULL
);
12006 build_omp_regions_1 (root
, NULL
, true);
12007 gcc_assert (root_omp_region
!= NULL
);
12010 /* Expands omp construct (and its subconstructs) starting in HEAD. */
12013 omp_expand_local (basic_block head
)
12015 build_omp_regions_root (head
);
12016 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
12018 fprintf (dump_file
, "\nOMP region tree\n\n");
12019 dump_omp_region (dump_file
, root_omp_region
, 0);
12020 fprintf (dump_file
, "\n");
12023 remove_exit_barriers (root_omp_region
);
12024 expand_omp (root_omp_region
);
12026 free_omp_regions ();
12029 /* Scan the CFG and build a tree of OMP regions. Return the root of
12030 the OMP region tree. */
12033 build_omp_regions (void)
12035 gcc_assert (root_omp_region
== NULL
);
12036 calculate_dominance_info (CDI_DOMINATORS
);
12037 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
12040 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
12042 static unsigned int
12043 execute_expand_omp (void)
12045 build_omp_regions ();
12047 if (!root_omp_region
)
12052 fprintf (dump_file
, "\nOMP region tree\n\n");
12053 dump_omp_region (dump_file
, root_omp_region
, 0);
12054 fprintf (dump_file
, "\n");
12057 remove_exit_barriers (root_omp_region
);
12059 expand_omp (root_omp_region
);
12061 #ifdef ENABLE_CHECKING
12062 if (!loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12063 verify_loop_structure ();
12065 cleanup_tree_cfg ();
12067 free_omp_regions ();
12072 /* OMP expansion -- the default pass, run before creation of SSA form. */
12076 const pass_data pass_data_expand_omp
=
12078 GIMPLE_PASS
, /* type */
12079 "ompexp", /* name */
12080 OPTGROUP_NONE
, /* optinfo_flags */
12081 TV_NONE
, /* tv_id */
12082 PROP_gimple_any
, /* properties_required */
12083 PROP_gimple_eomp
, /* properties_provided */
12084 0, /* properties_destroyed */
12085 0, /* todo_flags_start */
12086 0, /* todo_flags_finish */
12089 class pass_expand_omp
: public gimple_opt_pass
12092 pass_expand_omp (gcc::context
*ctxt
)
12093 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
12096 /* opt_pass methods: */
12097 virtual unsigned int execute (function
*)
12099 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
12100 || flag_openmp_simd
!= 0)
12101 && !seen_error ());
12103 /* This pass always runs, to provide PROP_gimple_eomp.
12104 But often, there is nothing to do. */
12108 return execute_expand_omp ();
12111 }; // class pass_expand_omp
12113 } // anon namespace
12116 make_pass_expand_omp (gcc::context
*ctxt
)
12118 return new pass_expand_omp (ctxt
);
12123 const pass_data pass_data_expand_omp_ssa
=
12125 GIMPLE_PASS
, /* type */
12126 "ompexpssa", /* name */
12127 OPTGROUP_NONE
, /* optinfo_flags */
12128 TV_NONE
, /* tv_id */
12129 PROP_cfg
| PROP_ssa
, /* properties_required */
12130 PROP_gimple_eomp
, /* properties_provided */
12131 0, /* properties_destroyed */
12132 0, /* todo_flags_start */
12133 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
12136 class pass_expand_omp_ssa
: public gimple_opt_pass
12139 pass_expand_omp_ssa (gcc::context
*ctxt
)
12140 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
12143 /* opt_pass methods: */
12144 virtual bool gate (function
*fun
)
12146 return !(fun
->curr_properties
& PROP_gimple_eomp
);
12148 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
12150 }; // class pass_expand_omp_ssa
12152 } // anon namespace
12155 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
12157 return new pass_expand_omp_ssa (ctxt
);
12160 /* Routines to lower OMP directives into OMP-GIMPLE. */
12162 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
12163 convert it to gimple. */
12165 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
12169 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
12171 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
12172 gimple_seq_add_stmt (seq
, stmt
);
12176 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12177 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
12178 gimplify_assign (t
, rdest
, seq
);
12181 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12182 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
12183 gimplify_assign (t
, idest
, seq
);
12186 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
12187 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
12188 gimplify_assign (t
, rsrc
, seq
);
12191 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
12192 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
12193 gimplify_assign (t
, isrc
, seq
);
12196 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12197 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12200 if (op
== PLUS_EXPR
)
12202 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
12203 gimple_seq_add_stmt (seq
, stmt
);
12205 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
12206 gimple_seq_add_stmt (seq
, stmt
);
12208 else if (op
== MULT_EXPR
)
12210 /* Let x = a + ib = dest, y = c + id = src.
12211 x * y = (ac - bd) + i(ad + bc) */
12212 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12213 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12214 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12215 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
12217 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
12218 gimple_seq_add_stmt (seq
, stmt
);
12220 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
12221 gimple_seq_add_stmt (seq
, stmt
);
12223 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
12224 gimple_seq_add_stmt (seq
, stmt
);
12226 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
12227 gimple_seq_add_stmt (seq
, stmt
);
12229 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
12230 gimple_seq_add_stmt (seq
, stmt
);
12232 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
12233 gimple_seq_add_stmt (seq
, stmt
);
12236 gcc_unreachable ();
12238 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
12239 gimplify_assign (dest
, result
, seq
);
12242 /* Helper function to initialize local data for the reduction arrays.
12243 The reduction arrays need to be placed inside the calling function
12244 for accelerators, or else the host won't be able to preform the final
12248 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
12249 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
12255 /* Find the innermost OpenACC parallel context. */
12256 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
12257 && (gimple_omp_target_kind (ctx
->stmt
)
12258 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
12262 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
12263 && (gimple_omp_target_kind (octx
->stmt
)
12264 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
12266 /* Extract the clauses. */
12267 oc
= gimple_omp_target_clauses (octx
->stmt
);
12269 /* Find the last outer clause. */
12270 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
12273 /* Allocate arrays for each reduction variable. */
12274 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
12276 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
12279 tree var
= OMP_CLAUSE_DECL (c
);
12280 tree type
= get_base_type (var
);
12281 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
12285 /* Calculate size of the reduction array. */
12286 t
= create_tmp_var (TREE_TYPE (nthreads
));
12287 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
12288 fold_convert (TREE_TYPE (nthreads
),
12289 TYPE_SIZE_UNIT (type
)));
12290 gimple_seq_add_stmt (stmt_seqp
, stmt
);
12292 size
= create_tmp_var (sizetype
);
12293 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
12295 /* Now allocate memory for it. */
12296 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
12297 stmt
= gimple_build_call (call
, 1, size
);
12298 gimple_call_set_lhs (stmt
, array
);
12299 gimple_seq_add_stmt (stmt_seqp
, stmt
);
12301 /* Map this array into the accelerator. */
12303 /* Add the reduction array to the list of clauses. */
12305 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
12306 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
12307 OMP_CLAUSE_DECL (t
) = x
;
12308 OMP_CLAUSE_CHAIN (t
) = NULL
;
12310 OMP_CLAUSE_CHAIN (oc
) = t
;
12312 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
12313 OMP_CLAUSE_SIZE (t
) = size
;
12318 /* Helper function to process the array of partial reductions. Nthreads
12319 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
12320 cannot be used here, because nthreads on the host may be different than
12321 on the accelerator. */
12324 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
12325 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
12327 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
12330 /* Create for loop.
12332 let var = the original reduction variable
12333 let array = reduction variable array
12335 for (i = 0; i < nthreads; i++)
12339 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
12340 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
12341 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
12343 /* Create and initialize an index variable. */
12344 tree ix
= create_tmp_var (sizetype
);
12345 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
12348 /* Insert the loop header label here. */
12349 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
12351 /* Exit loop if ix >= nthreads. */
12352 x
= create_tmp_var (sizetype
);
12353 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
12354 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
12355 gimple_seq_add_stmt (stmt_seqp
, stmt
);
12357 /* Insert the loop body label here. */
12358 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
12360 /* Collapse each reduction array, one element at a time. */
12361 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
12363 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
12366 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
12368 /* reduction(-:var) sums up the partial results, so it acts
12369 identically to reduction(+:var). */
12370 if (reduction_code
== MINUS_EXPR
)
12371 reduction_code
= PLUS_EXPR
;
12373 /* Set up reduction variable var. */
12374 var
= OMP_CLAUSE_DECL (c
);
12375 type
= get_base_type (var
);
12376 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
12377 (OMP_CLAUSE_DECL (c
)), ctx
);
12379 /* Calculate the array offset. */
12380 tree offset
= create_tmp_var (sizetype
);
12381 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
12382 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
12383 gimple_seq_add_stmt (stmt_seqp
, stmt
);
12385 tree ptr
= create_tmp_var (TREE_TYPE (array
));
12386 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
12387 gimple_seq_add_stmt (stmt_seqp
, stmt
);
12389 /* Extract array[ix] into mem. */
12390 tree mem
= create_tmp_var (type
);
12391 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
12393 /* Find the original reduction variable. */
12394 if (is_reference (var
))
12395 var
= build_simple_mem_ref (var
);
12397 tree t
= create_tmp_var (type
);
12399 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
12400 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
12402 /* var = var op mem */
12403 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
12405 case TRUTH_ANDIF_EXPR
:
12406 case TRUTH_ORIF_EXPR
:
12407 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
12409 gimplify_and_add (t
, stmt_seqp
);
12412 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
12413 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
12417 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
12418 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
12419 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
12422 /* Increment the induction variable. */
12423 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
12424 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
12425 gimple_seq_add_stmt (stmt_seqp
, stmt
);
12427 /* Go back to the top of the loop. */
12428 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
12430 /* Place the loop exit label here. */
12431 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
12434 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
12435 scan that for reductions. */
12438 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
12439 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
12441 gimple_stmt_iterator gsi
;
12442 gimple_seq inner
= NULL
;
12444 /* A collapse clause may have inserted a new bind block. */
12445 gsi
= gsi_start (*body
);
12446 while (!gsi_end_p (gsi
))
12448 gimple
*stmt
= gsi_stmt (gsi
);
12449 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
12451 inner
= gimple_bind_body (bind_stmt
);
12453 gsi
= gsi_start (*body
);
12455 else if (dyn_cast
<gomp_for
*> (stmt
))
12461 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
12463 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
12465 bool reduction_found
= false;
12467 gimple
*stmt
= gsi_stmt (gsi
);
12469 switch (gimple_code (stmt
))
12471 case GIMPLE_OMP_FOR
:
12472 clauses
= gimple_omp_for_clauses (stmt
);
12474 /* Search for a reduction clause. */
12475 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
12476 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
12478 reduction_found
= true;
12482 if (!reduction_found
)
12485 ctx
= maybe_lookup_ctx (stmt
);
12488 /* Extract the number of threads. */
12489 nthreads
= create_tmp_var (sizetype
);
12490 t
= oacc_max_threads (ctx
);
12491 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
12493 /* Determine if this is kernel will be executed on the host. */
12494 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
12495 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
12496 stmt
= gimple_build_call (call
, 0);
12497 gimple_call_set_lhs (stmt
, acc_device
);
12498 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
12500 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
12501 acc_device_host
= create_tmp_var (integer_type_node
,
12502 ".acc_device_host");
12503 gimplify_assign (acc_device_host
,
12504 build_int_cst (integer_type_node
,
12508 enter
= create_artificial_label (UNKNOWN_LOCATION
);
12509 exit
= create_artificial_label (UNKNOWN_LOCATION
);
12511 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
12513 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
12514 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
12515 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
12518 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
12520 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
12522 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
12525 // Scan for other directives which support reduction here.
12531 /* If ctx is a worksharing context inside of a cancellable parallel
12532 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
12533 and conditional branch to parallel's cancel_label to handle
12534 cancellation in the implicit barrier. */
12537 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
12539 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
12540 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
12541 if (gimple_omp_return_nowait_p (omp_return
))
12544 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
12545 && ctx
->outer
->cancellable
)
12547 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
12548 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
12549 tree lhs
= create_tmp_var (c_bool_type
);
12550 gimple_omp_return_set_lhs (omp_return
, lhs
);
12551 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
12552 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
12553 fold_convert (c_bool_type
,
12554 boolean_false_node
),
12555 ctx
->outer
->cancel_label
, fallthru_label
);
12556 gimple_seq_add_stmt (body
, g
);
12557 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
12561 /* Lower the OpenMP sections directive in the current statement in GSI_P.
12562 CTX is the enclosing OMP context for the current statement. */
12565 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12567 tree block
, control
;
12568 gimple_stmt_iterator tgsi
;
12569 gomp_sections
*stmt
;
12571 gbind
*new_stmt
, *bind
;
12572 gimple_seq ilist
, dlist
, olist
, new_body
;
12574 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
12576 push_gimplify_context ();
12580 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
12581 &ilist
, &dlist
, ctx
, NULL
);
12583 new_body
= gimple_omp_body (stmt
);
12584 gimple_omp_set_body (stmt
, NULL
);
12585 tgsi
= gsi_start (new_body
);
12586 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
12591 sec_start
= gsi_stmt (tgsi
);
12592 sctx
= maybe_lookup_ctx (sec_start
);
12595 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
12596 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
12597 GSI_CONTINUE_LINKING
);
12598 gimple_omp_set_body (sec_start
, NULL
);
12600 if (gsi_one_before_end_p (tgsi
))
12602 gimple_seq l
= NULL
;
12603 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
12605 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
12606 gimple_omp_section_set_last (sec_start
);
12609 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
12610 GSI_CONTINUE_LINKING
);
12613 block
= make_node (BLOCK
);
12614 bind
= gimple_build_bind (NULL
, new_body
, block
);
12617 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
12619 block
= make_node (BLOCK
);
12620 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
12621 gsi_replace (gsi_p
, new_stmt
, true);
12623 pop_gimplify_context (new_stmt
);
12624 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
12625 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
12626 if (BLOCK_VARS (block
))
12627 TREE_USED (block
) = 1;
12630 gimple_seq_add_seq (&new_body
, ilist
);
12631 gimple_seq_add_stmt (&new_body
, stmt
);
12632 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
12633 gimple_seq_add_stmt (&new_body
, bind
);
12635 control
= create_tmp_var (unsigned_type_node
, ".section");
12636 t
= gimple_build_omp_continue (control
, control
);
12637 gimple_omp_sections_set_control (stmt
, control
);
12638 gimple_seq_add_stmt (&new_body
, t
);
12640 gimple_seq_add_seq (&new_body
, olist
);
12641 if (ctx
->cancellable
)
12642 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
12643 gimple_seq_add_seq (&new_body
, dlist
);
12645 new_body
= maybe_catch_exception (new_body
);
12647 t
= gimple_build_omp_return
12648 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
12649 OMP_CLAUSE_NOWAIT
));
12650 gimple_seq_add_stmt (&new_body
, t
);
12651 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
12653 gimple_bind_set_body (new_stmt
, new_body
);
12657 /* A subroutine of lower_omp_single. Expand the simple form of
12658 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
12660 if (GOMP_single_start ())
12662 [ GOMP_barrier (); ] -> unless 'nowait' is present.
12664 FIXME. It may be better to delay expanding the logic of this until
12665 pass_expand_omp. The expanded logic may make the job more difficult
12666 to a synchronization analysis pass. */
12669 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
12671 location_t loc
= gimple_location (single_stmt
);
12672 tree tlabel
= create_artificial_label (loc
);
12673 tree flabel
= create_artificial_label (loc
);
12674 gimple
*call
, *cond
;
12677 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
12678 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
12679 call
= gimple_build_call (decl
, 0);
12680 gimple_call_set_lhs (call
, lhs
);
12681 gimple_seq_add_stmt (pre_p
, call
);
12683 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
12684 fold_convert_loc (loc
, TREE_TYPE (lhs
),
12685 boolean_true_node
),
12687 gimple_seq_add_stmt (pre_p
, cond
);
12688 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
12689 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
12690 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
12694 /* A subroutine of lower_omp_single. Expand the simple form of
12695 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
12697 #pragma omp single copyprivate (a, b, c)
12699 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
12702 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
12708 GOMP_single_copy_end (©out);
12719 FIXME. It may be better to delay expanding the logic of this until
12720 pass_expand_omp. The expanded logic may make the job more difficult
12721 to a synchronization analysis pass. */
12724 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
12727 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
12728 gimple_seq copyin_seq
;
12729 location_t loc
= gimple_location (single_stmt
);
12731 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
12733 ptr_type
= build_pointer_type (ctx
->record_type
);
12734 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
12736 l0
= create_artificial_label (loc
);
12737 l1
= create_artificial_label (loc
);
12738 l2
= create_artificial_label (loc
);
12740 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
12741 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
12742 t
= fold_convert_loc (loc
, ptr_type
, t
);
12743 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
12745 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
12746 build_int_cst (ptr_type
, 0));
12747 t
= build3 (COND_EXPR
, void_type_node
, t
,
12748 build_and_jump (&l0
), build_and_jump (&l1
));
12749 gimplify_and_add (t
, pre_p
);
12751 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
12753 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
12756 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
12759 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
12760 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
12761 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
12762 gimplify_and_add (t
, pre_p
);
12764 t
= build_and_jump (&l2
);
12765 gimplify_and_add (t
, pre_p
);
12767 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
12769 gimple_seq_add_seq (pre_p
, copyin_seq
);
12771 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
12775 /* Expand code for an OpenMP single directive. */
12778 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12782 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
12784 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
12786 push_gimplify_context ();
12788 block
= make_node (BLOCK
);
12789 bind
= gimple_build_bind (NULL
, NULL
, block
);
12790 gsi_replace (gsi_p
, bind
, true);
12793 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
12794 &bind_body
, &dlist
, ctx
, NULL
);
12795 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
12797 gimple_seq_add_stmt (&bind_body
, single_stmt
);
12799 if (ctx
->record_type
)
12800 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
12802 lower_omp_single_simple (single_stmt
, &bind_body
);
12804 gimple_omp_set_body (single_stmt
, NULL
);
12806 gimple_seq_add_seq (&bind_body
, dlist
);
12808 bind_body
= maybe_catch_exception (bind_body
);
12810 t
= gimple_build_omp_return
12811 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
12812 OMP_CLAUSE_NOWAIT
));
12813 gimple_seq_add_stmt (&bind_body_tail
, t
);
12814 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
12815 if (ctx
->record_type
)
12817 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
12818 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
12819 TREE_THIS_VOLATILE (clobber
) = 1;
12820 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
12821 clobber
), GSI_SAME_STMT
);
12823 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
12824 gimple_bind_set_body (bind
, bind_body
);
12826 pop_gimplify_context (bind
);
12828 gimple_bind_append_vars (bind
, ctx
->block_vars
);
12829 BLOCK_VARS (block
) = ctx
->block_vars
;
12830 if (BLOCK_VARS (block
))
12831 TREE_USED (block
) = 1;
12835 /* Expand code for an OpenMP master directive. */
12838 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12840 tree block
, lab
= NULL
, x
, bfn_decl
;
12841 gimple
*stmt
= gsi_stmt (*gsi_p
);
12843 location_t loc
= gimple_location (stmt
);
12846 push_gimplify_context ();
12848 block
= make_node (BLOCK
);
12849 bind
= gimple_build_bind (NULL
, NULL
, block
);
12850 gsi_replace (gsi_p
, bind
, true);
12851 gimple_bind_add_stmt (bind
, stmt
);
12853 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
12854 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
12855 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
12856 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
12858 gimplify_and_add (x
, &tseq
);
12859 gimple_bind_add_seq (bind
, tseq
);
12861 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
12862 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
12863 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
12864 gimple_omp_set_body (stmt
, NULL
);
12866 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
12868 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
12870 pop_gimplify_context (bind
);
12872 gimple_bind_append_vars (bind
, ctx
->block_vars
);
12873 BLOCK_VARS (block
) = ctx
->block_vars
;
12877 /* Expand code for an OpenMP taskgroup directive. */
12880 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
12882 gimple
*stmt
= gsi_stmt (*gsi_p
);
12885 tree block
= make_node (BLOCK
);
12887 bind
= gimple_build_bind (NULL
, NULL
, block
);
12888 gsi_replace (gsi_p
, bind
, true);
12889 gimple_bind_add_stmt (bind
, stmt
);
12891 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
12893 gimple_bind_add_stmt (bind
, x
);
12895 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
12896 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
12897 gimple_omp_set_body (stmt
, NULL
);
12899 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
12901 gimple_bind_append_vars (bind
, ctx
->block_vars
);
12902 BLOCK_VARS (block
) = ctx
->block_vars
;
12906 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
12909 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
12912 struct omp_for_data fd
;
12913 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
12916 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
12917 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
12918 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
12922 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
12923 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
12924 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
12925 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
12927 /* Merge depend clauses from multiple adjacent
12928 #pragma omp ordered depend(sink:...) constructs
12929 into one #pragma omp ordered depend(sink:...), so that
12930 we can optimize them together. */
12931 gimple_stmt_iterator gsi
= *gsi_p
;
12933 while (!gsi_end_p (gsi
))
12935 gimple
*stmt
= gsi_stmt (gsi
);
12936 if (is_gimple_debug (stmt
)
12937 || gimple_code (stmt
) == GIMPLE_NOP
)
12942 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
12944 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
12945 c
= gimple_omp_ordered_clauses (ord_stmt2
);
12947 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
12948 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
12951 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
12953 gsi_remove (&gsi
, true);
12957 /* Canonicalize sink dependence clauses into one folded clause if
12960 The basic algorithm is to create a sink vector whose first
12961 element is the GCD of all the first elements, and whose remaining
12962 elements are the minimum of the subsequent columns.
12964 We ignore dependence vectors whose first element is zero because
12965 such dependencies are known to be executed by the same thread.
12967 We take into account the direction of the loop, so a minimum
12968 becomes a maximum if the loop is iterating forwards. We also
12969 ignore sink clauses where the loop direction is unknown, or where
12970 the offsets are clearly invalid because they are not a multiple
12971 of the loop increment.
12975 #pragma omp for ordered(2)
12976 for (i=0; i < N; ++i)
12977 for (j=0; j < M; ++j)
12979 #pragma omp ordered \
12980 depend(sink:i-8,j-2) \
12981 depend(sink:i,j-1) \ // Completely ignored because i+0.
12982 depend(sink:i-4,j-3) \
12983 depend(sink:i-6,j-4)
12984 #pragma omp ordered depend(source)
12989 depend(sink:-gcd(8,4,6),-min(2,3,4))
12994 /* FIXME: Computing GCD's where the first element is zero is
12995 non-trivial in the presence of collapsed loops. Do this later. */
12996 if (fd
.collapse
> 1)
12999 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
13000 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
13001 tree folded_dep
= NULL_TREE
;
13002 /* TRUE if the first dimension's offset is negative. */
13003 bool neg_offset_p
= false;
13005 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13007 while ((c
= *list_p
) != NULL
)
13009 bool remove
= false;
13011 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
13012 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13013 goto next_ordered_clause
;
13016 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
13017 vec
&& TREE_CODE (vec
) == TREE_LIST
;
13018 vec
= TREE_CHAIN (vec
), ++i
)
13020 gcc_assert (i
< len
);
13022 /* extract_omp_for_data has canonicalized the condition. */
13023 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
13024 || fd
.loops
[i
].cond_code
== GT_EXPR
);
13025 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
13026 bool maybe_lexically_later
= true;
13028 /* While the committee makes up its mind, bail if we have any
13029 non-constant steps. */
13030 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
13031 goto lower_omp_ordered_ret
;
13033 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
13034 if (POINTER_TYPE_P (itype
))
13036 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
13037 TYPE_PRECISION (itype
),
13038 TYPE_SIGN (itype
));
13040 /* Ignore invalid offsets that are not multiples of the step. */
13041 if (!wi::multiple_of_p
13042 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
13045 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
13046 "ignoring sink clause with offset that is not "
13047 "a multiple of the loop step");
13049 goto next_ordered_clause
;
13052 /* Calculate the first dimension. The first dimension of
13053 the folded dependency vector is the GCD of the first
13054 elements, while ignoring any first elements whose offset
13058 /* Ignore dependence vectors whose first dimension is 0. */
13062 goto next_ordered_clause
;
13066 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
13068 error_at (OMP_CLAUSE_LOCATION (c
),
13069 "first offset must be in opposite direction "
13070 "of loop iterations");
13071 goto lower_omp_ordered_ret
;
13075 neg_offset_p
= forward
;
13076 /* Initialize the first time around. */
13077 if (folded_dep
== NULL_TREE
)
13080 folded_deps
[0] = offset
;
13083 folded_deps
[0] = wi::gcd (folded_deps
[0],
13087 /* Calculate minimum for the remaining dimensions. */
13090 folded_deps
[len
+ i
- 1] = offset
;
13091 if (folded_dep
== c
)
13092 folded_deps
[i
] = offset
;
13093 else if (maybe_lexically_later
13094 && !wi::eq_p (folded_deps
[i
], offset
))
13096 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
13100 for (j
= 1; j
<= i
; j
++)
13101 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
13104 maybe_lexically_later
= false;
13108 gcc_assert (i
== len
);
13112 next_ordered_clause
:
13114 *list_p
= OMP_CLAUSE_CHAIN (c
);
13116 list_p
= &OMP_CLAUSE_CHAIN (c
);
13122 folded_deps
[0] = -folded_deps
[0];
13124 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
13125 if (POINTER_TYPE_P (itype
))
13128 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
13129 = wide_int_to_tree (itype
, folded_deps
[0]);
13130 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
13131 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
13134 lower_omp_ordered_ret
:
13136 /* Ordered without clauses is #pragma omp threads, while we want
13137 a nop instead if we remove all clauses. */
13138 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
13139 gsi_replace (gsi_p
, gimple_build_nop (), true);
13143 /* Expand code for an OpenMP ordered directive. */
13146 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13149 gimple
*stmt
= gsi_stmt (*gsi_p
);
13150 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
13154 = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
), OMP_CLAUSE_SIMD
);
13156 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13157 OMP_CLAUSE_DEPEND
))
13159 /* FIXME: This is needs to be moved to the expansion to verify various
13160 conditions only testable on cfg with dominators computed, and also
13161 all the depend clauses to be merged still might need to be available
13162 for the runtime checks. */
13164 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
13168 push_gimplify_context ();
13170 block
= make_node (BLOCK
);
13171 bind
= gimple_build_bind (NULL
, NULL
, block
);
13172 gsi_replace (gsi_p
, bind
, true);
13173 gimple_bind_add_stmt (bind
, stmt
);
13177 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 0);
13178 cfun
->has_simduid_loops
= true;
13181 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
13183 gimple_bind_add_stmt (bind
, x
);
13185 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13186 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13187 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13188 gimple_omp_set_body (stmt
, NULL
);
13191 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 0);
13193 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
13195 gimple_bind_add_stmt (bind
, x
);
13197 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13199 pop_gimplify_context (bind
);
13201 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13202 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13206 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
13207 substitution of a couple of function calls. But in the NAMED case,
13208 requires that languages coordinate a symbol name. It is therefore
13209 best put here in common code. */
13211 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
13214 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13217 tree name
, lock
, unlock
;
13218 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
13220 location_t loc
= gimple_location (stmt
);
13223 name
= gimple_omp_critical_name (stmt
);
13228 if (!critical_name_mutexes
)
13229 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
13231 tree
*n
= critical_name_mutexes
->get (name
);
13236 decl
= create_tmp_var_raw (ptr_type_node
);
13238 new_str
= ACONCAT ((".gomp_critical_user_",
13239 IDENTIFIER_POINTER (name
), NULL
));
13240 DECL_NAME (decl
) = get_identifier (new_str
);
13241 TREE_PUBLIC (decl
) = 1;
13242 TREE_STATIC (decl
) = 1;
13243 DECL_COMMON (decl
) = 1;
13244 DECL_ARTIFICIAL (decl
) = 1;
13245 DECL_IGNORED_P (decl
) = 1;
13247 varpool_node::finalize_decl (decl
);
13249 critical_name_mutexes
->put (name
, decl
);
13254 /* If '#pragma omp critical' is inside offloaded region or
13255 inside function marked as offloadable, the symbol must be
13256 marked as offloadable too. */
13258 if (cgraph_node::get (current_function_decl
)->offloadable
)
13259 varpool_node::get_create (decl
)->offloadable
= 1;
13261 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
13262 if (is_gimple_omp_offloaded (octx
->stmt
))
13264 varpool_node::get_create (decl
)->offloadable
= 1;
13268 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
13269 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
13271 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
13272 unlock
= build_call_expr_loc (loc
, unlock
, 1,
13273 build_fold_addr_expr_loc (loc
, decl
));
13277 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
13278 lock
= build_call_expr_loc (loc
, lock
, 0);
13280 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
13281 unlock
= build_call_expr_loc (loc
, unlock
, 0);
13284 push_gimplify_context ();
13286 block
= make_node (BLOCK
);
13287 bind
= gimple_build_bind (NULL
, NULL
, block
);
13288 gsi_replace (gsi_p
, bind
, true);
13289 gimple_bind_add_stmt (bind
, stmt
);
13291 tbody
= gimple_bind_body (bind
);
13292 gimplify_and_add (lock
, &tbody
);
13293 gimple_bind_set_body (bind
, tbody
);
13295 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13296 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13297 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13298 gimple_omp_set_body (stmt
, NULL
);
13300 tbody
= gimple_bind_body (bind
);
13301 gimplify_and_add (unlock
, &tbody
);
13302 gimple_bind_set_body (bind
, tbody
);
13304 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13306 pop_gimplify_context (bind
);
13307 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13308 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13312 /* A subroutine of lower_omp_for. Generate code to emit the predicate
13313 for a lastprivate clause. Given a loop control predicate of (V
13314 cond N2), we gate the clause on (!(V cond N2)). The lowered form
13315 is appended to *DLIST, iterator initialization is appended to
13319 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
13320 gimple_seq
*dlist
, struct omp_context
*ctx
)
13322 tree clauses
, cond
, vinit
;
13323 enum tree_code cond_code
;
13326 cond_code
= fd
->loop
.cond_code
;
13327 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
13329 /* When possible, use a strict equality expression. This can let VRP
13330 type optimizations deduce the value and remove a copy. */
13331 if (tree_fits_shwi_p (fd
->loop
.step
))
13333 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
13334 if (step
== 1 || step
== -1)
13335 cond_code
= EQ_EXPR
;
13338 tree n2
= fd
->loop
.n2
;
13339 if (fd
->collapse
> 1
13340 && TREE_CODE (n2
) != INTEGER_CST
13341 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
13343 struct omp_context
*task_ctx
= NULL
;
13344 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
13346 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
13347 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
)
13349 struct omp_for_data outer_fd
;
13350 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
13351 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
13353 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
13354 task_ctx
= ctx
->outer
->outer
;
13356 else if (is_task_ctx (ctx
->outer
))
13357 task_ctx
= ctx
->outer
;
13362 = find_omp_clause (gimple_omp_task_clauses (task_ctx
->stmt
),
13363 OMP_CLAUSE__LOOPTEMP_
);
13364 gcc_assert (innerc
);
13365 for (i
= 0; i
< fd
->collapse
; i
++)
13367 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
13368 OMP_CLAUSE__LOOPTEMP_
);
13369 gcc_assert (innerc
);
13371 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
13372 OMP_CLAUSE__LOOPTEMP_
);
13374 n2
= fold_convert (TREE_TYPE (n2
),
13375 lookup_decl (OMP_CLAUSE_DECL (innerc
),
13379 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
13381 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
13383 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
13384 if (!gimple_seq_empty_p (stmts
))
13386 gimple_seq_add_seq (&stmts
, *dlist
);
13389 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
13390 vinit
= fd
->loop
.n1
;
13391 if (cond_code
== EQ_EXPR
13392 && tree_fits_shwi_p (fd
->loop
.n2
)
13393 && ! integer_zerop (fd
->loop
.n2
))
13394 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
13396 vinit
= unshare_expr (vinit
);
13398 /* Initialize the iterator variable, so that threads that don't execute
13399 any iterations don't execute the lastprivate clauses by accident. */
13400 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
13405 /* Lower code for an OMP loop directive. */
13408 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13410 tree
*rhs_p
, block
;
13411 struct omp_for_data fd
, *fdp
= NULL
;
13412 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
13414 gimple_seq omp_for_body
, body
, dlist
;
13417 push_gimplify_context ();
13419 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
13421 block
= make_node (BLOCK
);
13422 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
13423 /* Replace at gsi right away, so that 'stmt' is no member
13424 of a sequence anymore as we're going to add to a different
13426 gsi_replace (gsi_p
, new_stmt
, true);
13428 /* Move declaration of temporaries in the loop body before we make
13430 omp_for_body
= gimple_omp_body (stmt
);
13431 if (!gimple_seq_empty_p (omp_for_body
)
13432 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
13435 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
13436 tree vars
= gimple_bind_vars (inner_bind
);
13437 gimple_bind_append_vars (new_stmt
, vars
);
13438 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
13439 keep them on the inner_bind and it's block. */
13440 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
13441 if (gimple_bind_block (inner_bind
))
13442 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
13445 if (gimple_omp_for_combined_into_p (stmt
))
13447 extract_omp_for_data (stmt
, &fd
, NULL
);
13450 /* We need two temporaries with fd.loop.v type (istart/iend)
13451 and then (fd.collapse - 1) temporaries with the same
13452 type for count2 ... countN-1 vars if not constant. */
13454 tree type
= fd
.iter_type
;
13455 if (fd
.collapse
> 1
13456 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
13457 count
+= fd
.collapse
- 1;
13459 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
13460 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
13461 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
13462 tree clauses
= *pc
;
13465 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
13466 OMP_CLAUSE__LOOPTEMP_
);
13467 for (i
= 0; i
< count
; i
++)
13472 gcc_assert (outerc
);
13473 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
13474 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
13475 OMP_CLAUSE__LOOPTEMP_
);
13479 temp
= create_tmp_var (type
);
13480 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
13482 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
13483 OMP_CLAUSE_DECL (*pc
) = temp
;
13484 pc
= &OMP_CLAUSE_CHAIN (*pc
);
13489 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
13492 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
13494 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
13496 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13498 /* Lower the header expressions. At this point, we can assume that
13499 the header is of the form:
13501 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
13503 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
13504 using the .omp_data_s mapping, if needed. */
13505 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
13507 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
13508 if (!is_gimple_min_invariant (*rhs_p
))
13509 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
13511 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
13512 if (!is_gimple_min_invariant (*rhs_p
))
13513 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
13515 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
13516 if (!is_gimple_min_invariant (*rhs_p
))
13517 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
13520 /* Once lowered, extract the bounds and clauses. */
13521 extract_omp_for_data (stmt
, &fd
, NULL
);
13523 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
13525 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
13526 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
13527 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
13528 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
13530 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
13531 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
13532 OMP_CLAUSE_LINEAR_STEP (c
)
13533 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
13537 gimple_seq_add_stmt (&body
, stmt
);
13538 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
13540 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
13543 /* After the loop, add exit clauses. */
13544 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
13546 if (ctx
->cancellable
)
13547 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
13549 gimple_seq_add_seq (&body
, dlist
);
13551 body
= maybe_catch_exception (body
);
13553 /* Region exit marker goes at the end of the loop body. */
13554 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
13555 maybe_add_implicit_barrier_cancel (ctx
, &body
);
13556 pop_gimplify_context (new_stmt
);
13558 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
13559 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
13560 if (BLOCK_VARS (block
))
13561 TREE_USED (block
) = 1;
13563 gimple_bind_set_body (new_stmt
, body
);
13564 gimple_omp_set_body (stmt
, NULL
);
13565 gimple_omp_for_set_pre_body (stmt
, NULL
);
13568 /* Callback for walk_stmts. Check if the current statement only contains
13569 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
13572 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
13573 bool *handled_ops_p
,
13574 struct walk_stmt_info
*wi
)
13576 int *info
= (int *) wi
->info
;
13577 gimple
*stmt
= gsi_stmt (*gsi_p
);
13579 *handled_ops_p
= true;
13580 switch (gimple_code (stmt
))
13584 case GIMPLE_OMP_FOR
:
13585 case GIMPLE_OMP_SECTIONS
:
13586 *info
= *info
== 0 ? 1 : -1;
13595 struct omp_taskcopy_context
13597 /* This field must be at the beginning, as we do "inheritance": Some
13598 callback functions for tree-inline.c (e.g., omp_copy_decl)
13599 receive a copy_body_data pointer that is up-casted to an
13600 omp_context pointer. */
13606 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
13608 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
13610 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
13611 return create_tmp_var (TREE_TYPE (var
));
13617 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
13619 tree name
, new_fields
= NULL
, type
, f
;
13621 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
13622 name
= DECL_NAME (TYPE_NAME (orig_type
));
13623 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
13624 TYPE_DECL
, name
, type
);
13625 TYPE_NAME (type
) = name
;
13627 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
13629 tree new_f
= copy_node (f
);
13630 DECL_CONTEXT (new_f
) = type
;
13631 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
13632 TREE_CHAIN (new_f
) = new_fields
;
13633 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
13634 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
13635 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
13637 new_fields
= new_f
;
13638 tcctx
->cb
.decl_map
->put (f
, new_f
);
13640 TYPE_FIELDS (type
) = nreverse (new_fields
);
13641 layout_type (type
);
13645 /* Create task copyfn. */
13648 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
13650 struct function
*child_cfun
;
13651 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
13652 tree record_type
, srecord_type
, bind
, list
;
13653 bool record_needs_remap
= false, srecord_needs_remap
= false;
13655 struct omp_taskcopy_context tcctx
;
13656 location_t loc
= gimple_location (task_stmt
);
13658 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
13659 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
13660 gcc_assert (child_cfun
->cfg
== NULL
);
13661 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
13663 /* Reset DECL_CONTEXT on function arguments. */
13664 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13665 DECL_CONTEXT (t
) = child_fn
;
13667 /* Populate the function. */
13668 push_gimplify_context ();
13669 push_cfun (child_cfun
);
13671 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
13672 TREE_SIDE_EFFECTS (bind
) = 1;
13674 DECL_SAVED_TREE (child_fn
) = bind
;
13675 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
13677 /* Remap src and dst argument types if needed. */
13678 record_type
= ctx
->record_type
;
13679 srecord_type
= ctx
->srecord_type
;
13680 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
13681 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
13683 record_needs_remap
= true;
13686 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
13687 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
13689 srecord_needs_remap
= true;
13693 if (record_needs_remap
|| srecord_needs_remap
)
13695 memset (&tcctx
, '\0', sizeof (tcctx
));
13696 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
13697 tcctx
.cb
.dst_fn
= child_fn
;
13698 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
13699 gcc_checking_assert (tcctx
.cb
.src_node
);
13700 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
13701 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
13702 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
13703 tcctx
.cb
.eh_lp_nr
= 0;
13704 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
13705 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
13708 if (record_needs_remap
)
13709 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
13710 if (srecord_needs_remap
)
13711 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
13714 tcctx
.cb
.decl_map
= NULL
;
13716 arg
= DECL_ARGUMENTS (child_fn
);
13717 TREE_TYPE (arg
) = build_pointer_type (record_type
);
13718 sarg
= DECL_CHAIN (arg
);
13719 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
13721 /* First pass: initialize temporaries used in record_type and srecord_type
13722 sizes and field offsets. */
13723 if (tcctx
.cb
.decl_map
)
13724 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
13725 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
13729 decl
= OMP_CLAUSE_DECL (c
);
13730 p
= tcctx
.cb
.decl_map
->get (decl
);
13733 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
13734 sf
= (tree
) n
->value
;
13735 sf
= *tcctx
.cb
.decl_map
->get (sf
);
13736 src
= build_simple_mem_ref_loc (loc
, sarg
);
13737 src
= omp_build_component_ref (src
, sf
);
13738 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
13739 append_to_statement_list (t
, &list
);
13742 /* Second pass: copy shared var pointers and copy construct non-VLA
13743 firstprivate vars. */
13744 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
13745 switch (OMP_CLAUSE_CODE (c
))
13747 splay_tree_key key
;
13748 case OMP_CLAUSE_SHARED
:
13749 decl
= OMP_CLAUSE_DECL (c
);
13750 key
= (splay_tree_key
) decl
;
13751 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
13752 key
= (splay_tree_key
) &DECL_UID (decl
);
13753 n
= splay_tree_lookup (ctx
->field_map
, key
);
13756 f
= (tree
) n
->value
;
13757 if (tcctx
.cb
.decl_map
)
13758 f
= *tcctx
.cb
.decl_map
->get (f
);
13759 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
13760 sf
= (tree
) n
->value
;
13761 if (tcctx
.cb
.decl_map
)
13762 sf
= *tcctx
.cb
.decl_map
->get (sf
);
13763 src
= build_simple_mem_ref_loc (loc
, sarg
);
13764 src
= omp_build_component_ref (src
, sf
);
13765 dst
= build_simple_mem_ref_loc (loc
, arg
);
13766 dst
= omp_build_component_ref (dst
, f
);
13767 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
13768 append_to_statement_list (t
, &list
);
13770 case OMP_CLAUSE_FIRSTPRIVATE
:
13771 decl
= OMP_CLAUSE_DECL (c
);
13772 if (is_variable_sized (decl
))
13774 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
13777 f
= (tree
) n
->value
;
13778 if (tcctx
.cb
.decl_map
)
13779 f
= *tcctx
.cb
.decl_map
->get (f
);
13780 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
13783 sf
= (tree
) n
->value
;
13784 if (tcctx
.cb
.decl_map
)
13785 sf
= *tcctx
.cb
.decl_map
->get (sf
);
13786 src
= build_simple_mem_ref_loc (loc
, sarg
);
13787 src
= omp_build_component_ref (src
, sf
);
13788 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
13789 src
= build_simple_mem_ref_loc (loc
, src
);
13793 dst
= build_simple_mem_ref_loc (loc
, arg
);
13794 dst
= omp_build_component_ref (dst
, f
);
13795 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
13796 append_to_statement_list (t
, &list
);
13798 case OMP_CLAUSE_PRIVATE
:
13799 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
13801 decl
= OMP_CLAUSE_DECL (c
);
13802 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
13803 f
= (tree
) n
->value
;
13804 if (tcctx
.cb
.decl_map
)
13805 f
= *tcctx
.cb
.decl_map
->get (f
);
13806 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
13809 sf
= (tree
) n
->value
;
13810 if (tcctx
.cb
.decl_map
)
13811 sf
= *tcctx
.cb
.decl_map
->get (sf
);
13812 src
= build_simple_mem_ref_loc (loc
, sarg
);
13813 src
= omp_build_component_ref (src
, sf
);
13814 if (use_pointer_for_field (decl
, NULL
))
13815 src
= build_simple_mem_ref_loc (loc
, src
);
13819 dst
= build_simple_mem_ref_loc (loc
, arg
);
13820 dst
= omp_build_component_ref (dst
, f
);
13821 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
13822 append_to_statement_list (t
, &list
);
13828 /* Last pass: handle VLA firstprivates. */
13829 if (tcctx
.cb
.decl_map
)
13830 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
13831 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
13835 decl
= OMP_CLAUSE_DECL (c
);
13836 if (!is_variable_sized (decl
))
13838 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
13841 f
= (tree
) n
->value
;
13842 f
= *tcctx
.cb
.decl_map
->get (f
);
13843 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
13844 ind
= DECL_VALUE_EXPR (decl
);
13845 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
13846 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
13847 n
= splay_tree_lookup (ctx
->sfield_map
,
13848 (splay_tree_key
) TREE_OPERAND (ind
, 0));
13849 sf
= (tree
) n
->value
;
13850 sf
= *tcctx
.cb
.decl_map
->get (sf
);
13851 src
= build_simple_mem_ref_loc (loc
, sarg
);
13852 src
= omp_build_component_ref (src
, sf
);
13853 src
= build_simple_mem_ref_loc (loc
, src
);
13854 dst
= build_simple_mem_ref_loc (loc
, arg
);
13855 dst
= omp_build_component_ref (dst
, f
);
13856 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
13857 append_to_statement_list (t
, &list
);
13858 n
= splay_tree_lookup (ctx
->field_map
,
13859 (splay_tree_key
) TREE_OPERAND (ind
, 0));
13860 df
= (tree
) n
->value
;
13861 df
= *tcctx
.cb
.decl_map
->get (df
);
13862 ptr
= build_simple_mem_ref_loc (loc
, arg
);
13863 ptr
= omp_build_component_ref (ptr
, df
);
13864 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
13865 build_fold_addr_expr_loc (loc
, dst
));
13866 append_to_statement_list (t
, &list
);
13869 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
13870 append_to_statement_list (t
, &list
);
13872 if (tcctx
.cb
.decl_map
)
13873 delete tcctx
.cb
.decl_map
;
13874 pop_gimplify_context (NULL
);
13875 BIND_EXPR_BODY (bind
) = list
;
13880 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
13884 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
13886 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
13887 gcc_assert (clauses
);
13888 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
13889 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
13890 switch (OMP_CLAUSE_DEPEND_KIND (c
))
13892 case OMP_CLAUSE_DEPEND_IN
:
13895 case OMP_CLAUSE_DEPEND_OUT
:
13896 case OMP_CLAUSE_DEPEND_INOUT
:
13899 case OMP_CLAUSE_DEPEND_SOURCE
:
13900 case OMP_CLAUSE_DEPEND_SINK
:
13903 gcc_unreachable ();
13905 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
13906 tree array
= create_tmp_var (type
);
13907 TREE_ADDRESSABLE (array
) = 1;
13908 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
13910 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
13911 gimple_seq_add_stmt (iseq
, g
);
13912 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
13914 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
13915 gimple_seq_add_stmt (iseq
, g
);
13916 for (i
= 0; i
< 2; i
++)
13918 if ((i
? n_in
: n_out
) == 0)
13920 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
13921 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
13922 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
13924 tree t
= OMP_CLAUSE_DECL (c
);
13925 t
= fold_convert (ptr_type_node
, t
);
13926 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
13927 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
13928 NULL_TREE
, NULL_TREE
);
13929 g
= gimple_build_assign (r
, t
);
13930 gimple_seq_add_stmt (iseq
, g
);
13933 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
13934 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
13935 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
13937 tree clobber
= build_constructor (type
, NULL
);
13938 TREE_THIS_VOLATILE (clobber
) = 1;
13939 g
= gimple_build_assign (array
, clobber
);
13940 gimple_seq_add_stmt (oseq
, g
);
13943 /* Lower the OpenMP parallel or task directive in the current statement
13944 in GSI_P. CTX holds context information for the directive. */
13947 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13951 gimple
*stmt
= gsi_stmt (*gsi_p
);
13952 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
13953 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
13954 location_t loc
= gimple_location (stmt
);
13956 clauses
= gimple_omp_taskreg_clauses (stmt
);
13958 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
13959 par_body
= gimple_bind_body (par_bind
);
13960 child_fn
= ctx
->cb
.dst_fn
;
13961 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
13962 && !gimple_omp_parallel_combined_p (stmt
))
13964 struct walk_stmt_info wi
;
13967 memset (&wi
, 0, sizeof (wi
));
13969 wi
.val_only
= true;
13970 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
13972 gimple_omp_parallel_set_combined_p (stmt
, true);
13974 gimple_seq dep_ilist
= NULL
;
13975 gimple_seq dep_olist
= NULL
;
13976 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
13977 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
13979 push_gimplify_context ();
13980 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
13981 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
13982 &dep_ilist
, &dep_olist
);
13985 if (ctx
->srecord_type
)
13986 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
13988 push_gimplify_context ();
13993 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
13994 lower_omp (&par_body
, ctx
);
13995 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
13996 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
13998 /* Declare all the variables created by mapping and the variables
13999 declared in the scope of the parallel body. */
14000 record_vars_into (ctx
->block_vars
, child_fn
);
14001 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
14003 if (ctx
->record_type
)
14006 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
14007 : ctx
->record_type
, ".omp_data_o");
14008 DECL_NAMELESS (ctx
->sender_decl
) = 1;
14009 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
14010 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
14015 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
14016 lower_send_shared_vars (&ilist
, &olist
, ctx
);
14018 if (ctx
->record_type
)
14020 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
14021 TREE_THIS_VOLATILE (clobber
) = 1;
14022 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
14026 /* Once all the expansions are done, sequence all the different
14027 fragments inside gimple_omp_body. */
14031 if (ctx
->record_type
)
14033 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14034 /* fixup_child_record_type might have changed receiver_decl's type. */
14035 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
14036 gimple_seq_add_stmt (&new_body
,
14037 gimple_build_assign (ctx
->receiver_decl
, t
));
14040 gimple_seq_add_seq (&new_body
, par_ilist
);
14041 gimple_seq_add_seq (&new_body
, par_body
);
14042 gimple_seq_add_seq (&new_body
, par_rlist
);
14043 if (ctx
->cancellable
)
14044 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14045 gimple_seq_add_seq (&new_body
, par_olist
);
14046 new_body
= maybe_catch_exception (new_body
);
14047 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
14048 gimple_seq_add_stmt (&new_body
,
14049 gimple_build_omp_continue (integer_zero_node
,
14050 integer_zero_node
));
14051 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
14052 gimple_omp_set_body (stmt
, new_body
);
14054 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
14055 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
14056 gimple_bind_add_seq (bind
, ilist
);
14057 gimple_bind_add_stmt (bind
, stmt
);
14058 gimple_bind_add_seq (bind
, olist
);
14060 pop_gimplify_context (NULL
);
14064 gimple_bind_add_seq (dep_bind
, dep_ilist
);
14065 gimple_bind_add_stmt (dep_bind
, bind
);
14066 gimple_bind_add_seq (dep_bind
, dep_olist
);
14067 pop_gimplify_context (dep_bind
);
14071 /* Lower the GIMPLE_OMP_TARGET in the current statement
14072 in GSI_P. CTX holds context information for the directive. */
14075 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14078 tree child_fn
, t
, c
;
14079 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
14080 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
14081 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
14082 location_t loc
= gimple_location (stmt
);
14083 bool offloaded
, data_region
;
14084 unsigned int map_cnt
= 0;
14085 bool has_depend
= false;
14087 offloaded
= is_gimple_omp_offloaded (stmt
);
14088 switch (gimple_omp_target_kind (stmt
))
14090 case GF_OMP_TARGET_KIND_REGION
:
14091 case GF_OMP_TARGET_KIND_UPDATE
:
14092 case GF_OMP_TARGET_KIND_ENTER_DATA
:
14093 case GF_OMP_TARGET_KIND_EXIT_DATA
:
14094 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
14095 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
14096 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
14097 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
14098 data_region
= false;
14100 case GF_OMP_TARGET_KIND_DATA
:
14101 case GF_OMP_TARGET_KIND_OACC_DATA
:
14102 data_region
= true;
14105 gcc_unreachable ();
14108 clauses
= gimple_omp_target_clauses (stmt
);
14110 gimple_seq dep_ilist
= NULL
;
14111 gimple_seq dep_olist
= NULL
;
14112 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14114 push_gimplify_context ();
14115 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14116 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
14117 &dep_ilist
, &dep_olist
);
14125 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
14126 tgt_body
= gimple_bind_body (tgt_bind
);
14128 else if (data_region
)
14129 tgt_body
= gimple_omp_body (stmt
);
14130 child_fn
= ctx
->cb
.dst_fn
;
14132 push_gimplify_context ();
14137 && is_gimple_omp_oacc (stmt
))
14138 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
14140 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14141 switch (OMP_CLAUSE_CODE (c
))
14147 case OMP_CLAUSE_MAP
:
14148 #ifdef ENABLE_CHECKING
14149 /* First check what we're prepared to handle in the following. */
14150 switch (OMP_CLAUSE_MAP_KIND (c
))
14152 case GOMP_MAP_ALLOC
:
14154 case GOMP_MAP_FROM
:
14155 case GOMP_MAP_TOFROM
:
14156 case GOMP_MAP_POINTER
:
14157 case GOMP_MAP_TO_PSET
:
14158 case GOMP_MAP_FORCE_DEALLOC
:
14159 case GOMP_MAP_RELEASE
:
14160 case GOMP_MAP_ALWAYS_TO
:
14161 case GOMP_MAP_ALWAYS_FROM
:
14162 case GOMP_MAP_ALWAYS_TOFROM
:
14163 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
14164 case GOMP_MAP_STRUCT
:
14166 case GOMP_MAP_FORCE_ALLOC
:
14167 case GOMP_MAP_FORCE_TO
:
14168 case GOMP_MAP_FORCE_FROM
:
14169 case GOMP_MAP_FORCE_TOFROM
:
14170 case GOMP_MAP_FORCE_PRESENT
:
14171 case GOMP_MAP_FORCE_DEVICEPTR
:
14172 gcc_assert (is_gimple_omp_oacc (stmt
));
14175 gcc_unreachable ();
14179 case OMP_CLAUSE_TO
:
14180 case OMP_CLAUSE_FROM
:
14181 var
= OMP_CLAUSE_DECL (c
);
14184 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
14185 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
14186 && (OMP_CLAUSE_MAP_KIND (c
)
14187 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
14192 if (DECL_SIZE (var
)
14193 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
14195 tree var2
= DECL_VALUE_EXPR (var
);
14196 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
14197 var2
= TREE_OPERAND (var2
, 0);
14198 gcc_assert (DECL_P (var2
));
14203 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
14205 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
14207 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
14208 && varpool_node::get_create (var
)->offloadable
)
14211 tree type
= build_pointer_type (TREE_TYPE (var
));
14212 tree new_var
= lookup_decl (var
, ctx
);
14213 x
= create_tmp_var_raw (type
, get_name (new_var
));
14214 gimple_add_tmp_var (x
);
14215 x
= build_simple_mem_ref (x
);
14216 SET_DECL_VALUE_EXPR (new_var
, x
);
14217 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14222 if (offloaded
&& OMP_CLAUSE_MAP_PRIVATE (c
))
14228 if (!maybe_lookup_field (var
, ctx
))
14233 x
= build_receiver_ref (var
, true, ctx
);
14234 tree new_var
= lookup_decl (var
, ctx
);
14236 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
14237 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
14238 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
14239 x
= build_simple_mem_ref (x
);
14240 SET_DECL_VALUE_EXPR (new_var
, x
);
14241 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14246 case OMP_CLAUSE_FIRSTPRIVATE
:
14248 var
= OMP_CLAUSE_DECL (c
);
14249 if (!is_reference (var
)
14250 && !is_gimple_reg_type (TREE_TYPE (var
)))
14252 tree new_var
= lookup_decl (var
, ctx
);
14253 if (is_variable_sized (var
))
14255 tree pvar
= DECL_VALUE_EXPR (var
);
14256 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14257 pvar
= TREE_OPERAND (pvar
, 0);
14258 gcc_assert (DECL_P (pvar
));
14259 tree new_pvar
= lookup_decl (pvar
, ctx
);
14260 x
= build_fold_indirect_ref (new_pvar
);
14261 TREE_THIS_NOTRAP (x
) = 1;
14264 x
= build_receiver_ref (var
, true, ctx
);
14265 SET_DECL_VALUE_EXPR (new_var
, x
);
14266 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14270 case OMP_CLAUSE_PRIVATE
:
14271 var
= OMP_CLAUSE_DECL (c
);
14272 if (is_variable_sized (var
))
14274 tree new_var
= lookup_decl (var
, ctx
);
14275 tree pvar
= DECL_VALUE_EXPR (var
);
14276 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14277 pvar
= TREE_OPERAND (pvar
, 0);
14278 gcc_assert (DECL_P (pvar
));
14279 tree new_pvar
= lookup_decl (pvar
, ctx
);
14280 x
= build_fold_indirect_ref (new_pvar
);
14281 TREE_THIS_NOTRAP (x
) = 1;
14282 SET_DECL_VALUE_EXPR (new_var
, x
);
14283 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14287 case OMP_CLAUSE_USE_DEVICE_PTR
:
14288 case OMP_CLAUSE_IS_DEVICE_PTR
:
14289 var
= OMP_CLAUSE_DECL (c
);
14291 if (is_variable_sized (var
))
14293 tree new_var
= lookup_decl (var
, ctx
);
14294 tree pvar
= DECL_VALUE_EXPR (var
);
14295 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14296 pvar
= TREE_OPERAND (pvar
, 0);
14297 gcc_assert (DECL_P (pvar
));
14298 tree new_pvar
= lookup_decl (pvar
, ctx
);
14299 x
= build_fold_indirect_ref (new_pvar
);
14300 TREE_THIS_NOTRAP (x
) = 1;
14301 SET_DECL_VALUE_EXPR (new_var
, x
);
14302 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14304 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
14306 tree new_var
= lookup_decl (var
, ctx
);
14307 tree type
= build_pointer_type (TREE_TYPE (var
));
14308 x
= create_tmp_var_raw (type
, get_name (new_var
));
14309 gimple_add_tmp_var (x
);
14310 x
= build_simple_mem_ref (x
);
14311 SET_DECL_VALUE_EXPR (new_var
, x
);
14312 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14319 target_nesting_level
++;
14320 lower_omp (&tgt_body
, ctx
);
14321 target_nesting_level
--;
14323 else if (data_region
)
14324 lower_omp (&tgt_body
, ctx
);
14328 /* Declare all the variables created by mapping and the variables
14329 declared in the scope of the target body. */
14330 record_vars_into (ctx
->block_vars
, child_fn
);
14331 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
14336 if (ctx
->record_type
)
14339 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
14340 DECL_NAMELESS (ctx
->sender_decl
) = 1;
14341 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
14342 t
= make_tree_vec (3);
14343 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
14344 TREE_VEC_ELT (t
, 1)
14345 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
14346 ".omp_data_sizes");
14347 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
14348 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
14349 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
14350 tree tkind_type
= short_unsigned_type_node
;
14351 int talign_shift
= 8;
14352 TREE_VEC_ELT (t
, 2)
14353 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
14354 ".omp_data_kinds");
14355 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
14356 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
14357 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
14358 gimple_omp_target_set_data_arg (stmt
, t
);
14360 vec
<constructor_elt
, va_gc
> *vsize
;
14361 vec
<constructor_elt
, va_gc
> *vkind
;
14362 vec_alloc (vsize
, map_cnt
);
14363 vec_alloc (vkind
, map_cnt
);
14364 unsigned int map_idx
= 0;
14366 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14367 switch (OMP_CLAUSE_CODE (c
))
14369 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
14370 unsigned int talign
;
14374 case OMP_CLAUSE_MAP
:
14375 case OMP_CLAUSE_TO
:
14376 case OMP_CLAUSE_FROM
:
14378 ovar
= OMP_CLAUSE_DECL (c
);
14379 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
14380 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
14382 if (!DECL_P (ovar
))
14384 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
14385 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
14387 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
14388 == get_base_address (ovar
));
14389 nc
= OMP_CLAUSE_CHAIN (c
);
14390 ovar
= OMP_CLAUSE_DECL (nc
);
14394 tree x
= build_sender_ref (ovar
, ctx
);
14396 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
14397 gimplify_assign (x
, v
, &ilist
);
14403 if (DECL_SIZE (ovar
)
14404 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
14406 tree ovar2
= DECL_VALUE_EXPR (ovar
);
14407 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
14408 ovar2
= TREE_OPERAND (ovar2
, 0);
14409 gcc_assert (DECL_P (ovar2
));
14412 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
14413 && OMP_CLAUSE_MAP_PRIVATE (c
))
14415 if (!maybe_lookup_field ((splay_tree_key
) &DECL_UID (ovar
),
14419 else if (!maybe_lookup_field (ovar
, ctx
))
14423 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
14424 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
14425 talign
= DECL_ALIGN_UNIT (ovar
);
14428 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
14429 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
14430 && OMP_CLAUSE_MAP_PRIVATE (c
))
14431 x
= build_sender_ref ((splay_tree_key
) &DECL_UID (ovar
),
14434 x
= build_sender_ref (ovar
, ctx
);
14435 if (maybe_lookup_oacc_reduction (var
, ctx
))
14437 gcc_checking_assert (offloaded
14438 && is_gimple_omp_oacc (stmt
));
14439 gimplify_assign (x
, var
, &ilist
);
14441 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
14442 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
14443 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
14444 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
14446 gcc_assert (offloaded
);
14448 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
14449 mark_addressable (avar
);
14450 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
14451 talign
= DECL_ALIGN_UNIT (avar
);
14452 avar
= build_fold_addr_expr (avar
);
14453 gimplify_assign (x
, avar
, &ilist
);
14455 else if (is_gimple_reg (var
))
14457 gcc_assert (offloaded
);
14458 tree avar
= create_tmp_var (TREE_TYPE (var
));
14459 mark_addressable (avar
);
14460 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
14461 if (GOMP_MAP_COPY_TO_P (map_kind
)
14462 || map_kind
== GOMP_MAP_POINTER
14463 || map_kind
== GOMP_MAP_TO_PSET
14464 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
14465 gimplify_assign (avar
, var
, &ilist
);
14466 avar
= build_fold_addr_expr (avar
);
14467 gimplify_assign (x
, avar
, &ilist
);
14468 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
14469 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
14470 && !TYPE_READONLY (TREE_TYPE (var
)))
14472 x
= unshare_expr (x
);
14473 x
= build_simple_mem_ref (x
);
14474 gimplify_assign (var
, x
, &olist
);
14479 var
= build_fold_addr_expr (var
);
14480 gimplify_assign (x
, var
, &ilist
);
14483 s
= OMP_CLAUSE_SIZE (c
);
14484 if (s
== NULL_TREE
)
14485 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
14486 s
= fold_convert (size_type_node
, s
);
14487 purpose
= size_int (map_idx
++);
14488 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
14489 if (TREE_CODE (s
) != INTEGER_CST
)
14490 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
14492 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
14493 switch (OMP_CLAUSE_CODE (c
))
14495 case OMP_CLAUSE_MAP
:
14496 tkind
= OMP_CLAUSE_MAP_KIND (c
);
14497 tkind_zero
= tkind
;
14498 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
14501 case GOMP_MAP_ALLOC
:
14503 case GOMP_MAP_FROM
:
14504 case GOMP_MAP_TOFROM
:
14505 case GOMP_MAP_ALWAYS_TO
:
14506 case GOMP_MAP_ALWAYS_FROM
:
14507 case GOMP_MAP_ALWAYS_TOFROM
:
14508 case GOMP_MAP_RELEASE
:
14509 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
14511 case GOMP_MAP_DELETE
:
14512 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
14516 if (tkind_zero
!= tkind
)
14518 if (integer_zerop (s
))
14519 tkind
= tkind_zero
;
14520 else if (integer_nonzerop (s
))
14521 tkind_zero
= tkind
;
14524 case OMP_CLAUSE_TO
:
14525 tkind
= GOMP_MAP_TO
;
14526 tkind_zero
= tkind
;
14528 case OMP_CLAUSE_FROM
:
14529 tkind
= GOMP_MAP_FROM
;
14530 tkind_zero
= tkind
;
14533 gcc_unreachable ();
14535 gcc_checking_assert (tkind
14536 < (HOST_WIDE_INT_C (1U) << talign_shift
));
14537 gcc_checking_assert (tkind_zero
14538 < (HOST_WIDE_INT_C (1U) << talign_shift
));
14539 talign
= ceil_log2 (talign
);
14540 tkind
|= talign
<< talign_shift
;
14541 tkind_zero
|= talign
<< talign_shift
;
14542 gcc_checking_assert (tkind
14543 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
14544 gcc_checking_assert (tkind_zero
14545 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
14546 if (tkind
== tkind_zero
)
14547 x
= build_int_cstu (tkind_type
, tkind
);
14550 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
14551 x
= build3 (COND_EXPR
, tkind_type
,
14552 fold_build2 (EQ_EXPR
, boolean_type_node
,
14553 unshare_expr (s
), size_zero_node
),
14554 build_int_cstu (tkind_type
, tkind_zero
),
14555 build_int_cstu (tkind_type
, tkind
));
14557 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
14562 case OMP_CLAUSE_FIRSTPRIVATE
:
14563 ovar
= OMP_CLAUSE_DECL (c
);
14564 if (is_reference (ovar
))
14565 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
14567 talign
= DECL_ALIGN_UNIT (ovar
);
14568 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
14569 x
= build_sender_ref (ovar
, ctx
);
14570 tkind
= GOMP_MAP_FIRSTPRIVATE
;
14571 type
= TREE_TYPE (ovar
);
14572 if (is_reference (ovar
))
14573 type
= TREE_TYPE (type
);
14574 bool use_firstprivate_int
, force_addr
;
14575 use_firstprivate_int
= false;
14576 force_addr
= false;
14577 if ((INTEGRAL_TYPE_P (type
)
14578 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
14579 || TREE_CODE (type
) == POINTER_TYPE
)
14580 use_firstprivate_int
= true;
14583 if (is_reference (var
))
14584 use_firstprivate_int
= false;
14585 else if (is_gimple_reg (var
))
14587 if (DECL_HAS_VALUE_EXPR_P (var
))
14589 tree v
= get_base_address (var
);
14590 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
14592 use_firstprivate_int
= false;
14596 switch (TREE_CODE (v
))
14600 use_firstprivate_int
= false;
14609 use_firstprivate_int
= false;
14611 if (use_firstprivate_int
)
14613 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
14615 if (is_reference (var
))
14616 t
= build_simple_mem_ref (var
);
14617 if (TREE_CODE (type
) != POINTER_TYPE
)
14618 t
= fold_convert (pointer_sized_int_node
, t
);
14619 t
= fold_convert (TREE_TYPE (x
), t
);
14620 gimplify_assign (x
, t
, &ilist
);
14622 else if (is_reference (var
))
14623 gimplify_assign (x
, var
, &ilist
);
14624 else if (!force_addr
&& is_gimple_reg (var
))
14626 tree avar
= create_tmp_var (TREE_TYPE (var
));
14627 mark_addressable (avar
);
14628 gimplify_assign (avar
, var
, &ilist
);
14629 avar
= build_fold_addr_expr (avar
);
14630 gimplify_assign (x
, avar
, &ilist
);
14634 var
= build_fold_addr_expr (var
);
14635 gimplify_assign (x
, var
, &ilist
);
14637 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
14639 else if (is_reference (var
))
14640 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
14642 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
14643 s
= fold_convert (size_type_node
, s
);
14644 purpose
= size_int (map_idx
++);
14645 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
14646 if (TREE_CODE (s
) != INTEGER_CST
)
14647 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
14649 gcc_checking_assert (tkind
14650 < (HOST_WIDE_INT_C (1U) << talign_shift
));
14651 talign
= ceil_log2 (talign
);
14652 tkind
|= talign
<< talign_shift
;
14653 gcc_checking_assert (tkind
14654 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
14655 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
14656 build_int_cstu (tkind_type
, tkind
));
14659 case OMP_CLAUSE_USE_DEVICE_PTR
:
14660 case OMP_CLAUSE_IS_DEVICE_PTR
:
14661 ovar
= OMP_CLAUSE_DECL (c
);
14662 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
14663 x
= build_sender_ref (ovar
, ctx
);
14664 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
14665 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
14667 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
14668 type
= TREE_TYPE (ovar
);
14669 if (TREE_CODE (type
) == ARRAY_TYPE
)
14670 var
= build_fold_addr_expr (var
);
14673 if (is_reference (ovar
))
14675 type
= TREE_TYPE (type
);
14676 if (TREE_CODE (type
) != ARRAY_TYPE
)
14677 var
= build_simple_mem_ref (var
);
14678 var
= fold_convert (TREE_TYPE (x
), var
);
14681 gimplify_assign (x
, var
, &ilist
);
14683 purpose
= size_int (map_idx
++);
14684 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
14685 gcc_checking_assert (tkind
14686 < (HOST_WIDE_INT_C (1U) << talign_shift
));
14687 gcc_checking_assert (tkind
14688 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
14689 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
14690 build_int_cstu (tkind_type
, tkind
));
14694 gcc_assert (map_idx
== map_cnt
);
14696 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
14697 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
14698 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
14699 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
14700 for (int i
= 1; i
<= 2; i
++)
14701 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
14703 gimple_seq initlist
= NULL
;
14704 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
14705 TREE_VEC_ELT (t
, i
)),
14706 &initlist
, true, NULL_TREE
);
14707 gimple_seq_add_seq (&ilist
, initlist
);
14709 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
14711 TREE_THIS_VOLATILE (clobber
) = 1;
14712 gimple_seq_add_stmt (&olist
,
14713 gimple_build_assign (TREE_VEC_ELT (t
, i
),
14717 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14718 TREE_THIS_VOLATILE (clobber
) = 1;
14719 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
14723 /* Once all the expansions are done, sequence all the different
14724 fragments inside gimple_omp_body. */
14729 && ctx
->record_type
)
14731 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14732 /* fixup_child_record_type might have changed receiver_decl's type. */
14733 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
14734 gimple_seq_add_stmt (&new_body
,
14735 gimple_build_assign (ctx
->receiver_decl
, t
));
14738 if (offloaded
|| data_region
)
14740 tree prev
= NULL_TREE
;
14741 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14742 switch (OMP_CLAUSE_CODE (c
))
14747 case OMP_CLAUSE_FIRSTPRIVATE
:
14748 var
= OMP_CLAUSE_DECL (c
);
14749 if (is_reference (var
)
14750 || is_gimple_reg_type (TREE_TYPE (var
)))
14752 tree new_var
= lookup_decl (var
, ctx
);
14754 type
= TREE_TYPE (var
);
14755 if (is_reference (var
))
14756 type
= TREE_TYPE (type
);
14757 bool use_firstprivate_int
;
14758 use_firstprivate_int
= false;
14759 if ((INTEGRAL_TYPE_P (type
)
14760 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
14761 || TREE_CODE (type
) == POINTER_TYPE
)
14762 use_firstprivate_int
= true;
14765 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
14766 if (is_reference (v
))
14767 use_firstprivate_int
= false;
14768 else if (is_gimple_reg (v
))
14770 if (DECL_HAS_VALUE_EXPR_P (v
))
14772 v
= get_base_address (v
);
14773 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
14774 use_firstprivate_int
= false;
14776 switch (TREE_CODE (v
))
14780 use_firstprivate_int
= false;
14788 use_firstprivate_int
= false;
14790 if (use_firstprivate_int
)
14792 x
= build_receiver_ref (var
, false, ctx
);
14793 if (TREE_CODE (type
) != POINTER_TYPE
)
14794 x
= fold_convert (pointer_sized_int_node
, x
);
14795 x
= fold_convert (type
, x
);
14796 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
14798 if (is_reference (var
))
14800 tree v
= create_tmp_var_raw (type
, get_name (var
));
14801 gimple_add_tmp_var (v
);
14802 TREE_ADDRESSABLE (v
) = 1;
14803 gimple_seq_add_stmt (&new_body
,
14804 gimple_build_assign (v
, x
));
14805 x
= build_fold_addr_expr (v
);
14807 gimple_seq_add_stmt (&new_body
,
14808 gimple_build_assign (new_var
, x
));
14812 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
14813 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
14815 gimple_seq_add_stmt (&new_body
,
14816 gimple_build_assign (new_var
, x
));
14819 else if (is_variable_sized (var
))
14821 tree pvar
= DECL_VALUE_EXPR (var
);
14822 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14823 pvar
= TREE_OPERAND (pvar
, 0);
14824 gcc_assert (DECL_P (pvar
));
14825 tree new_var
= lookup_decl (pvar
, ctx
);
14826 x
= build_receiver_ref (var
, false, ctx
);
14827 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
14828 gimple_seq_add_stmt (&new_body
,
14829 gimple_build_assign (new_var
, x
));
14832 case OMP_CLAUSE_PRIVATE
:
14833 var
= OMP_CLAUSE_DECL (c
);
14834 if (is_reference (var
))
14836 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
14837 tree new_var
= lookup_decl (var
, ctx
);
14838 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
14839 if (TREE_CONSTANT (x
))
14841 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
14843 gimple_add_tmp_var (x
);
14844 TREE_ADDRESSABLE (x
) = 1;
14845 x
= build_fold_addr_expr_loc (clause_loc
, x
);
14850 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
14851 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
14852 tree al
= size_int (TYPE_ALIGN (rtype
));
14853 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
14856 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
14857 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
14858 gimple_seq_add_stmt (&new_body
,
14859 gimple_build_assign (new_var
, x
));
14862 case OMP_CLAUSE_USE_DEVICE_PTR
:
14863 case OMP_CLAUSE_IS_DEVICE_PTR
:
14864 var
= OMP_CLAUSE_DECL (c
);
14865 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
14866 x
= build_sender_ref (var
, ctx
);
14868 x
= build_receiver_ref (var
, false, ctx
);
14869 if (is_variable_sized (var
))
14871 tree pvar
= DECL_VALUE_EXPR (var
);
14872 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14873 pvar
= TREE_OPERAND (pvar
, 0);
14874 gcc_assert (DECL_P (pvar
));
14875 tree new_var
= lookup_decl (pvar
, ctx
);
14876 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
14877 gimple_seq_add_stmt (&new_body
,
14878 gimple_build_assign (new_var
, x
));
14880 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
14882 tree new_var
= lookup_decl (var
, ctx
);
14883 new_var
= DECL_VALUE_EXPR (new_var
);
14884 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
14885 new_var
= TREE_OPERAND (new_var
, 0);
14886 gcc_assert (DECL_P (new_var
));
14887 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
14888 gimple_seq_add_stmt (&new_body
,
14889 gimple_build_assign (new_var
, x
));
14893 tree type
= TREE_TYPE (var
);
14894 tree new_var
= lookup_decl (var
, ctx
);
14895 if (is_reference (var
))
14897 type
= TREE_TYPE (type
);
14898 if (TREE_CODE (type
) != ARRAY_TYPE
)
14900 tree v
= create_tmp_var_raw (type
, get_name (var
));
14901 gimple_add_tmp_var (v
);
14902 TREE_ADDRESSABLE (v
) = 1;
14903 x
= fold_convert (type
, x
);
14904 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
14906 gimple_seq_add_stmt (&new_body
,
14907 gimple_build_assign (v
, x
));
14908 x
= build_fold_addr_expr (v
);
14911 x
= fold_convert (TREE_TYPE (new_var
), x
);
14912 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
14913 gimple_seq_add_stmt (&new_body
,
14914 gimple_build_assign (new_var
, x
));
14918 /* Handle GOMP_MAP_FIRSTPRIVATE_POINTER in second pass,
14919 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
14920 are already handled. */
14921 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14922 switch (OMP_CLAUSE_CODE (c
))
14927 case OMP_CLAUSE_MAP
:
14928 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
14930 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
14931 HOST_WIDE_INT offset
= 0;
14933 var
= OMP_CLAUSE_DECL (c
);
14935 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
14936 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
14938 && varpool_node::get_create (var
)->offloadable
)
14940 if (TREE_CODE (var
) == INDIRECT_REF
14941 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
14942 var
= TREE_OPERAND (var
, 0);
14943 if (TREE_CODE (var
) == COMPONENT_REF
)
14945 var
= get_addr_base_and_unit_offset (var
, &offset
);
14946 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
14948 else if (DECL_SIZE (var
)
14949 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
14951 tree var2
= DECL_VALUE_EXPR (var
);
14952 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
14953 var2
= TREE_OPERAND (var2
, 0);
14954 gcc_assert (DECL_P (var2
));
14957 tree new_var
= lookup_decl (var
, ctx
), x
;
14958 tree type
= TREE_TYPE (new_var
);
14960 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
14961 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
14964 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
14966 new_var
= build2 (MEM_REF
, type
,
14967 build_fold_addr_expr (new_var
),
14968 build_int_cst (build_pointer_type (type
),
14971 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
14973 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
14974 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
14975 new_var
= build2 (MEM_REF
, type
,
14976 build_fold_addr_expr (new_var
),
14977 build_int_cst (build_pointer_type (type
),
14981 is_ref
= is_reference (var
);
14982 bool ref_to_array
= false;
14985 type
= TREE_TYPE (type
);
14986 if (TREE_CODE (type
) == ARRAY_TYPE
)
14988 type
= build_pointer_type (type
);
14989 ref_to_array
= true;
14992 else if (TREE_CODE (type
) == ARRAY_TYPE
)
14994 tree decl2
= DECL_VALUE_EXPR (new_var
);
14995 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
14996 decl2
= TREE_OPERAND (decl2
, 0);
14997 gcc_assert (DECL_P (decl2
));
14999 type
= TREE_TYPE (new_var
);
15001 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
15002 x
= fold_convert_loc (clause_loc
, type
, x
);
15003 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
15005 tree bias
= OMP_CLAUSE_SIZE (c
);
15007 bias
= lookup_decl (bias
, ctx
);
15008 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
15009 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
15011 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
15012 TREE_TYPE (x
), x
, bias
);
15015 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15016 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15017 if (is_ref
&& !ref_to_array
)
15019 tree t
= create_tmp_var_raw (type
, get_name (var
));
15020 gimple_add_tmp_var (t
);
15021 TREE_ADDRESSABLE (t
) = 1;
15022 gimple_seq_add_stmt (&new_body
,
15023 gimple_build_assign (t
, x
));
15024 x
= build_fold_addr_expr_loc (clause_loc
, t
);
15026 gimple_seq_add_stmt (&new_body
,
15027 gimple_build_assign (new_var
, x
));
15030 else if (OMP_CLAUSE_CHAIN (c
)
15031 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
15033 && OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15034 == GOMP_MAP_FIRSTPRIVATE_POINTER
)
15037 case OMP_CLAUSE_PRIVATE
:
15038 var
= OMP_CLAUSE_DECL (c
);
15039 if (is_variable_sized (var
))
15041 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15042 tree new_var
= lookup_decl (var
, ctx
);
15043 tree pvar
= DECL_VALUE_EXPR (var
);
15044 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15045 pvar
= TREE_OPERAND (pvar
, 0);
15046 gcc_assert (DECL_P (pvar
));
15047 tree new_pvar
= lookup_decl (pvar
, ctx
);
15048 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15049 tree al
= size_int (DECL_ALIGN (var
));
15050 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
15051 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15052 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
15053 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15054 gimple_seq_add_stmt (&new_body
,
15055 gimple_build_assign (new_pvar
, x
));
15059 gimple_seq_add_seq (&new_body
, tgt_body
);
15061 new_body
= maybe_catch_exception (new_body
);
15063 else if (data_region
)
15064 new_body
= tgt_body
;
15065 if (offloaded
|| data_region
)
15067 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15068 gimple_omp_set_body (stmt
, new_body
);
15071 bind
= gimple_build_bind (NULL
, NULL
,
15072 tgt_bind
? gimple_bind_block (tgt_bind
)
15074 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15075 gimple_bind_add_seq (bind
, irlist
);
15076 gimple_bind_add_seq (bind
, ilist
);
15077 gimple_bind_add_stmt (bind
, stmt
);
15078 gimple_bind_add_seq (bind
, olist
);
15079 gimple_bind_add_seq (bind
, orlist
);
15081 pop_gimplify_context (NULL
);
15085 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15086 gimple_bind_add_stmt (dep_bind
, bind
);
15087 gimple_bind_add_seq (dep_bind
, dep_olist
);
15088 pop_gimplify_context (dep_bind
);
15092 /* Expand code for an OpenMP teams directive. */
15095 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15097 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
15098 push_gimplify_context ();
15100 tree block
= make_node (BLOCK
);
15101 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
15102 gsi_replace (gsi_p
, bind
, true);
15103 gimple_seq bind_body
= NULL
;
15104 gimple_seq dlist
= NULL
;
15105 gimple_seq olist
= NULL
;
15107 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15108 OMP_CLAUSE_NUM_TEAMS
);
15109 if (num_teams
== NULL_TREE
)
15110 num_teams
= build_int_cst (unsigned_type_node
, 0);
15113 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
15114 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
15115 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
15117 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15118 OMP_CLAUSE_THREAD_LIMIT
);
15119 if (thread_limit
== NULL_TREE
)
15120 thread_limit
= build_int_cst (unsigned_type_node
, 0);
15123 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
15124 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
15125 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
15129 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
15130 &bind_body
, &dlist
, ctx
, NULL
);
15131 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
15132 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
15133 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
15135 location_t loc
= gimple_location (teams_stmt
);
15136 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
15137 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
15138 gimple_set_location (call
, loc
);
15139 gimple_seq_add_stmt (&bind_body
, call
);
15141 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
15142 gimple_omp_set_body (teams_stmt
, NULL
);
15143 gimple_seq_add_seq (&bind_body
, olist
);
15144 gimple_seq_add_seq (&bind_body
, dlist
);
15145 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
15146 gimple_bind_set_body (bind
, bind_body
);
15148 pop_gimplify_context (bind
);
15150 gimple_bind_append_vars (bind
, ctx
->block_vars
);
15151 BLOCK_VARS (block
) = ctx
->block_vars
;
15152 if (BLOCK_VARS (block
))
15153 TREE_USED (block
) = 1;
15157 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
15158 regimplified. If DATA is non-NULL, lower_omp_1 is outside
15159 of OMP context, but with task_shared_vars set. */
15162 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
15167 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
15168 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
15171 if (task_shared_vars
15173 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
15176 /* If a global variable has been privatized, TREE_CONSTANT on
15177 ADDR_EXPR might be wrong. */
15178 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
15179 recompute_tree_invariant_for_addr_expr (t
);
15181 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
15185 /* Data to be communicated between lower_omp_regimplify_operands and
15186 lower_omp_regimplify_operands_p. */
15188 struct lower_omp_regimplify_operands_data
15194 /* Helper function for lower_omp_regimplify_operands. Find
15195 omp_member_access_dummy_var vars and adjust temporarily their
15196 DECL_VALUE_EXPRs if needed. */
15199 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
15202 tree t
= omp_member_access_dummy_var (*tp
);
15205 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
15206 lower_omp_regimplify_operands_data
*ldata
15207 = (lower_omp_regimplify_operands_data
*) wi
->info
;
15208 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
15211 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
15212 ldata
->decls
->safe_push (*tp
);
15213 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
15214 SET_DECL_VALUE_EXPR (*tp
, v
);
15217 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
15221 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
15222 of omp_member_access_dummy_var vars during regimplification. */
15225 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
15226 gimple_stmt_iterator
*gsi_p
)
15228 auto_vec
<tree
, 10> decls
;
15231 struct walk_stmt_info wi
;
15232 memset (&wi
, '\0', sizeof (wi
));
15233 struct lower_omp_regimplify_operands_data data
;
15235 data
.decls
= &decls
;
15237 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
15239 gimple_regimplify_operands (stmt
, gsi_p
);
15240 while (!decls
.is_empty ())
15242 tree t
= decls
.pop ();
15243 tree v
= decls
.pop ();
15244 SET_DECL_VALUE_EXPR (t
, v
);
15249 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15251 gimple
*stmt
= gsi_stmt (*gsi_p
);
15252 struct walk_stmt_info wi
;
15255 if (gimple_has_location (stmt
))
15256 input_location
= gimple_location (stmt
);
15258 if (task_shared_vars
)
15259 memset (&wi
, '\0', sizeof (wi
));
15261 /* If we have issued syntax errors, avoid doing any heavy lifting.
15262 Just replace the OMP directives with a NOP to avoid
15263 confusing RTL expansion. */
15264 if (seen_error () && is_gimple_omp (stmt
))
15266 gsi_replace (gsi_p
, gimple_build_nop (), true);
15270 switch (gimple_code (stmt
))
15274 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
15275 if ((ctx
|| task_shared_vars
)
15276 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
15277 lower_omp_regimplify_p
,
15278 ctx
? NULL
: &wi
, NULL
)
15279 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
15280 lower_omp_regimplify_p
,
15281 ctx
? NULL
: &wi
, NULL
)))
15282 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
15286 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
15288 case GIMPLE_EH_FILTER
:
15289 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
15292 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
15293 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
15295 case GIMPLE_TRANSACTION
:
15296 lower_omp (gimple_transaction_body_ptr (
15297 as_a
<gtransaction
*> (stmt
)),
15301 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
15303 case GIMPLE_OMP_PARALLEL
:
15304 case GIMPLE_OMP_TASK
:
15305 ctx
= maybe_lookup_ctx (stmt
);
15307 if (ctx
->cancellable
)
15308 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
15309 lower_omp_taskreg (gsi_p
, ctx
);
15311 case GIMPLE_OMP_FOR
:
15312 ctx
= maybe_lookup_ctx (stmt
);
15314 if (ctx
->cancellable
)
15315 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
15316 lower_omp_for (gsi_p
, ctx
);
15318 case GIMPLE_OMP_SECTIONS
:
15319 ctx
= maybe_lookup_ctx (stmt
);
15321 if (ctx
->cancellable
)
15322 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
15323 lower_omp_sections (gsi_p
, ctx
);
15325 case GIMPLE_OMP_SINGLE
:
15326 ctx
= maybe_lookup_ctx (stmt
);
15328 lower_omp_single (gsi_p
, ctx
);
15330 case GIMPLE_OMP_MASTER
:
15331 ctx
= maybe_lookup_ctx (stmt
);
15333 lower_omp_master (gsi_p
, ctx
);
15335 case GIMPLE_OMP_TASKGROUP
:
15336 ctx
= maybe_lookup_ctx (stmt
);
15338 lower_omp_taskgroup (gsi_p
, ctx
);
15340 case GIMPLE_OMP_ORDERED
:
15341 ctx
= maybe_lookup_ctx (stmt
);
15343 lower_omp_ordered (gsi_p
, ctx
);
15345 case GIMPLE_OMP_CRITICAL
:
15346 ctx
= maybe_lookup_ctx (stmt
);
15348 lower_omp_critical (gsi_p
, ctx
);
15350 case GIMPLE_OMP_ATOMIC_LOAD
:
15351 if ((ctx
|| task_shared_vars
)
15352 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
15353 as_a
<gomp_atomic_load
*> (stmt
)),
15354 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
15355 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
15357 case GIMPLE_OMP_TARGET
:
15358 ctx
= maybe_lookup_ctx (stmt
);
15360 lower_omp_target (gsi_p
, ctx
);
15362 case GIMPLE_OMP_TEAMS
:
15363 ctx
= maybe_lookup_ctx (stmt
);
15365 lower_omp_teams (gsi_p
, ctx
);
15369 call_stmt
= as_a
<gcall
*> (stmt
);
15370 fndecl
= gimple_call_fndecl (call_stmt
);
15372 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
15373 switch (DECL_FUNCTION_CODE (fndecl
))
15375 case BUILT_IN_GOMP_BARRIER
:
15379 case BUILT_IN_GOMP_CANCEL
:
15380 case BUILT_IN_GOMP_CANCELLATION_POINT
:
15383 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
15384 cctx
= cctx
->outer
;
15385 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
15386 if (!cctx
->cancellable
)
15388 if (DECL_FUNCTION_CODE (fndecl
)
15389 == BUILT_IN_GOMP_CANCELLATION_POINT
)
15391 stmt
= gimple_build_nop ();
15392 gsi_replace (gsi_p
, stmt
, false);
15396 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
15398 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
15399 gimple_call_set_fndecl (call_stmt
, fndecl
);
15400 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
15403 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
15404 gimple_call_set_lhs (call_stmt
, lhs
);
15405 tree fallthru_label
;
15406 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
15408 g
= gimple_build_label (fallthru_label
);
15409 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
15410 g
= gimple_build_cond (NE_EXPR
, lhs
,
15411 fold_convert (TREE_TYPE (lhs
),
15412 boolean_false_node
),
15413 cctx
->cancel_label
, fallthru_label
);
15414 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
15421 if ((ctx
|| task_shared_vars
)
15422 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
15425 /* Just remove clobbers, this should happen only if we have
15426 "privatized" local addressable variables in SIMD regions,
15427 the clobber isn't needed in that case and gimplifying address
15428 of the ARRAY_REF into a pointer and creating MEM_REF based
15429 clobber would create worse code than we get with the clobber
15431 if (gimple_clobber_p (stmt
))
15433 gsi_replace (gsi_p
, gimple_build_nop (), true);
15436 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
15443 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
15445 location_t saved_location
= input_location
;
15446 gimple_stmt_iterator gsi
;
15447 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
15448 lower_omp_1 (&gsi
, ctx
);
15449 /* During gimplification, we haven't folded statments inside offloading
15450 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
15451 if (target_nesting_level
|| taskreg_nesting_level
)
15452 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
15454 input_location
= saved_location
;
15457 /* Main entry point. */
15459 static unsigned int
15460 execute_lower_omp (void)
15466 /* This pass always runs, to provide PROP_gimple_lomp.
15467 But often, there is nothing to do. */
15468 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
15469 && flag_openmp_simd
== 0)
15472 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
15473 delete_omp_context
);
15475 body
= gimple_body (current_function_decl
);
15476 scan_omp (&body
, NULL
);
15477 gcc_assert (taskreg_nesting_level
== 0);
15478 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
15479 finish_taskreg_scan (ctx
);
15480 taskreg_contexts
.release ();
15482 if (all_contexts
->root
)
15484 if (task_shared_vars
)
15485 push_gimplify_context ();
15486 lower_omp (&body
, NULL
);
15487 if (task_shared_vars
)
15488 pop_gimplify_context (NULL
);
15493 splay_tree_delete (all_contexts
);
15494 all_contexts
= NULL
;
15496 BITMAP_FREE (task_shared_vars
);
15502 const pass_data pass_data_lower_omp
=
15504 GIMPLE_PASS
, /* type */
15505 "omplower", /* name */
15506 OPTGROUP_NONE
, /* optinfo_flags */
15507 TV_NONE
, /* tv_id */
15508 PROP_gimple_any
, /* properties_required */
15509 PROP_gimple_lomp
, /* properties_provided */
15510 0, /* properties_destroyed */
15511 0, /* todo_flags_start */
15512 0, /* todo_flags_finish */
15515 class pass_lower_omp
: public gimple_opt_pass
15518 pass_lower_omp (gcc::context
*ctxt
)
15519 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
15522 /* opt_pass methods: */
15523 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
15525 }; // class pass_lower_omp
15527 } // anon namespace
15530 make_pass_lower_omp (gcc::context
*ctxt
)
15532 return new pass_lower_omp (ctxt
);
15535 /* The following is a utility to diagnose structured block violations.
15536 It is not part of the "omplower" pass, as that's invoked too late. It
15537 should be invoked by the respective front ends after gimplification. */
15539 static splay_tree all_labels
;
15541 /* Check for mismatched contexts and generate an error if needed. Return
15542 true if an error is detected. */
15545 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
15546 gimple
*branch_ctx
, gimple
*label_ctx
)
15548 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
15549 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
15551 if (label_ctx
== branch_ctx
)
15554 const char* kind
= NULL
;
15559 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
15560 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
15562 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
15563 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
15564 kind
= "Cilk Plus";
15568 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
15569 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
15571 gcc_checking_assert (kind
== NULL
);
15577 gcc_checking_assert (flag_openmp
);
15582 Previously we kept track of the label's entire context in diagnose_sb_[12]
15583 so we could traverse it and issue a correct "exit" or "enter" error
15584 message upon a structured block violation.
15586 We built the context by building a list with tree_cons'ing, but there is
15587 no easy counterpart in gimple tuples. It seems like far too much work
15588 for issuing exit/enter error messages. If someone really misses the
15589 distinct error message... patches welcome.
15593 /* Try to avoid confusing the user by producing and error message
15594 with correct "exit" or "enter" verbiage. We prefer "exit"
15595 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
15596 if (branch_ctx
== NULL
)
15602 if (TREE_VALUE (label_ctx
) == branch_ctx
)
15607 label_ctx
= TREE_CHAIN (label_ctx
);
15612 error ("invalid exit from %s structured block", kind
);
15614 error ("invalid entry to %s structured block", kind
);
15617 /* If it's obvious we have an invalid entry, be specific about the error. */
15618 if (branch_ctx
== NULL
)
15619 error ("invalid entry to %s structured block", kind
);
15622 /* Otherwise, be vague and lazy, but efficient. */
15623 error ("invalid branch to/from %s structured block", kind
);
15626 gsi_replace (gsi_p
, gimple_build_nop (), false);
15630 /* Pass 1: Create a minimal tree of structured blocks, and record
15631 where each label is found. */
15634 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
15635 struct walk_stmt_info
*wi
)
15637 gimple
*context
= (gimple
*) wi
->info
;
15638 gimple
*inner_context
;
15639 gimple
*stmt
= gsi_stmt (*gsi_p
);
15641 *handled_ops_p
= true;
15643 switch (gimple_code (stmt
))
15647 case GIMPLE_OMP_PARALLEL
:
15648 case GIMPLE_OMP_TASK
:
15649 case GIMPLE_OMP_SECTIONS
:
15650 case GIMPLE_OMP_SINGLE
:
15651 case GIMPLE_OMP_SECTION
:
15652 case GIMPLE_OMP_MASTER
:
15653 case GIMPLE_OMP_ORDERED
:
15654 case GIMPLE_OMP_CRITICAL
:
15655 case GIMPLE_OMP_TARGET
:
15656 case GIMPLE_OMP_TEAMS
:
15657 case GIMPLE_OMP_TASKGROUP
:
15658 /* The minimal context here is just the current OMP construct. */
15659 inner_context
= stmt
;
15660 wi
->info
= inner_context
;
15661 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
15662 wi
->info
= context
;
15665 case GIMPLE_OMP_FOR
:
15666 inner_context
= stmt
;
15667 wi
->info
= inner_context
;
15668 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15670 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
15671 diagnose_sb_1
, NULL
, wi
);
15672 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
15673 wi
->info
= context
;
15677 splay_tree_insert (all_labels
,
15678 (splay_tree_key
) gimple_label_label (
15679 as_a
<glabel
*> (stmt
)),
15680 (splay_tree_value
) context
);
15690 /* Pass 2: Check each branch and see if its context differs from that of
15691 the destination label's context. */
15694 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
15695 struct walk_stmt_info
*wi
)
15697 gimple
*context
= (gimple
*) wi
->info
;
15699 gimple
*stmt
= gsi_stmt (*gsi_p
);
15701 *handled_ops_p
= true;
15703 switch (gimple_code (stmt
))
15707 case GIMPLE_OMP_PARALLEL
:
15708 case GIMPLE_OMP_TASK
:
15709 case GIMPLE_OMP_SECTIONS
:
15710 case GIMPLE_OMP_SINGLE
:
15711 case GIMPLE_OMP_SECTION
:
15712 case GIMPLE_OMP_MASTER
:
15713 case GIMPLE_OMP_ORDERED
:
15714 case GIMPLE_OMP_CRITICAL
:
15715 case GIMPLE_OMP_TARGET
:
15716 case GIMPLE_OMP_TEAMS
:
15717 case GIMPLE_OMP_TASKGROUP
:
15719 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
15720 wi
->info
= context
;
15723 case GIMPLE_OMP_FOR
:
15725 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
15727 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
15728 diagnose_sb_2
, NULL
, wi
);
15729 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
15730 wi
->info
= context
;
15735 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
15736 tree lab
= gimple_cond_true_label (cond_stmt
);
15739 n
= splay_tree_lookup (all_labels
,
15740 (splay_tree_key
) lab
);
15741 diagnose_sb_0 (gsi_p
, context
,
15742 n
? (gimple
*) n
->value
: NULL
);
15744 lab
= gimple_cond_false_label (cond_stmt
);
15747 n
= splay_tree_lookup (all_labels
,
15748 (splay_tree_key
) lab
);
15749 diagnose_sb_0 (gsi_p
, context
,
15750 n
? (gimple
*) n
->value
: NULL
);
15757 tree lab
= gimple_goto_dest (stmt
);
15758 if (TREE_CODE (lab
) != LABEL_DECL
)
15761 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
15762 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
15766 case GIMPLE_SWITCH
:
15768 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
15770 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
15772 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
15773 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
15774 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
15780 case GIMPLE_RETURN
:
15781 diagnose_sb_0 (gsi_p
, context
, NULL
);
15791 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
15794 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
15797 gimple
*last
= last_stmt (bb
);
15798 enum gimple_code code
= gimple_code (last
);
15799 struct omp_region
*cur_region
= *region
;
15800 bool fallthru
= false;
15804 case GIMPLE_OMP_PARALLEL
:
15805 case GIMPLE_OMP_TASK
:
15806 case GIMPLE_OMP_FOR
:
15807 case GIMPLE_OMP_SINGLE
:
15808 case GIMPLE_OMP_TEAMS
:
15809 case GIMPLE_OMP_MASTER
:
15810 case GIMPLE_OMP_TASKGROUP
:
15811 case GIMPLE_OMP_CRITICAL
:
15812 case GIMPLE_OMP_SECTION
:
15813 cur_region
= new_omp_region (bb
, code
, cur_region
);
15817 case GIMPLE_OMP_ORDERED
:
15818 cur_region
= new_omp_region (bb
, code
, cur_region
);
15820 if (find_omp_clause (gimple_omp_ordered_clauses
15821 (as_a
<gomp_ordered
*> (last
)),
15822 OMP_CLAUSE_DEPEND
))
15823 cur_region
= cur_region
->outer
;
15826 case GIMPLE_OMP_TARGET
:
15827 cur_region
= new_omp_region (bb
, code
, cur_region
);
15829 switch (gimple_omp_target_kind (last
))
15831 case GF_OMP_TARGET_KIND_REGION
:
15832 case GF_OMP_TARGET_KIND_DATA
:
15833 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15834 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15835 case GF_OMP_TARGET_KIND_OACC_DATA
:
15837 case GF_OMP_TARGET_KIND_UPDATE
:
15838 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15839 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15840 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15841 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15842 cur_region
= cur_region
->outer
;
15845 gcc_unreachable ();
15849 case GIMPLE_OMP_SECTIONS
:
15850 cur_region
= new_omp_region (bb
, code
, cur_region
);
15854 case GIMPLE_OMP_SECTIONS_SWITCH
:
15858 case GIMPLE_OMP_ATOMIC_LOAD
:
15859 case GIMPLE_OMP_ATOMIC_STORE
:
15863 case GIMPLE_OMP_RETURN
:
15864 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
15865 somewhere other than the next block. This will be
15867 cur_region
->exit
= bb
;
15868 if (cur_region
->type
== GIMPLE_OMP_TASK
)
15869 /* Add an edge corresponding to not scheduling the task
15871 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
15872 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
15873 cur_region
= cur_region
->outer
;
15876 case GIMPLE_OMP_CONTINUE
:
15877 cur_region
->cont
= bb
;
15878 switch (cur_region
->type
)
15880 case GIMPLE_OMP_FOR
:
15881 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
15882 succs edges as abnormal to prevent splitting
15884 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
15885 /* Make the loopback edge. */
15886 make_edge (bb
, single_succ (cur_region
->entry
),
15889 /* Create an edge from GIMPLE_OMP_FOR to exit, which
15890 corresponds to the case that the body of the loop
15891 is not executed at all. */
15892 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
15893 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
15897 case GIMPLE_OMP_SECTIONS
:
15898 /* Wire up the edges into and out of the nested sections. */
15900 basic_block switch_bb
= single_succ (cur_region
->entry
);
15902 struct omp_region
*i
;
15903 for (i
= cur_region
->inner
; i
; i
= i
->next
)
15905 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
15906 make_edge (switch_bb
, i
->entry
, 0);
15907 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
15910 /* Make the loopback edge to the block with
15911 GIMPLE_OMP_SECTIONS_SWITCH. */
15912 make_edge (bb
, switch_bb
, 0);
15914 /* Make the edge from the switch to exit. */
15915 make_edge (switch_bb
, bb
->next_bb
, 0);
15920 case GIMPLE_OMP_TASK
:
15925 gcc_unreachable ();
15930 gcc_unreachable ();
15933 if (*region
!= cur_region
)
15935 *region
= cur_region
;
15937 *region_idx
= cur_region
->entry
->index
;
15945 static unsigned int
15946 diagnose_omp_structured_block_errors (void)
15948 struct walk_stmt_info wi
;
15949 gimple_seq body
= gimple_body (current_function_decl
);
15951 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
15953 memset (&wi
, 0, sizeof (wi
));
15954 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
15956 memset (&wi
, 0, sizeof (wi
));
15957 wi
.want_locations
= true;
15958 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
15960 gimple_set_body (current_function_decl
, body
);
15962 splay_tree_delete (all_labels
);
15970 const pass_data pass_data_diagnose_omp_blocks
=
15972 GIMPLE_PASS
, /* type */
15973 "*diagnose_omp_blocks", /* name */
15974 OPTGROUP_NONE
, /* optinfo_flags */
15975 TV_NONE
, /* tv_id */
15976 PROP_gimple_any
, /* properties_required */
15977 0, /* properties_provided */
15978 0, /* properties_destroyed */
15979 0, /* todo_flags_start */
15980 0, /* todo_flags_finish */
15983 class pass_diagnose_omp_blocks
: public gimple_opt_pass
15986 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
15987 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
15990 /* opt_pass methods: */
15991 virtual bool gate (function
*)
15993 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
15995 virtual unsigned int execute (function
*)
15997 return diagnose_omp_structured_block_errors ();
16000 }; // class pass_diagnose_omp_blocks
16002 } // anon namespace
16005 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16007 return new pass_diagnose_omp_blocks (ctxt
);
16010 /* SIMD clone supporting code. */
16012 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
16013 of arguments to reserve space for. */
16015 static struct cgraph_simd_clone
*
16016 simd_clone_struct_alloc (int nargs
)
16018 struct cgraph_simd_clone
*clone_info
;
16019 size_t len
= (sizeof (struct cgraph_simd_clone
)
16020 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
16021 clone_info
= (struct cgraph_simd_clone
*)
16022 ggc_internal_cleared_alloc (len
);
16026 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
16029 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
16030 struct cgraph_simd_clone
*from
)
16032 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
16033 + ((from
->nargs
- from
->inbranch
)
16034 * sizeof (struct cgraph_simd_clone_arg
))));
16037 /* Return vector of parameter types of function FNDECL. This uses
16038 TYPE_ARG_TYPES if available, otherwise falls back to types of
16039 DECL_ARGUMENTS types. */
16042 simd_clone_vector_of_formal_parm_types (tree fndecl
)
16044 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
16045 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
16046 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
16049 FOR_EACH_VEC_ELT (args
, i
, arg
)
16050 args
[i
] = TREE_TYPE (args
[i
]);
16054 /* Given a simd function in NODE, extract the simd specific
16055 information from the OMP clauses passed in CLAUSES, and return
16056 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
16057 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
16058 otherwise set to FALSE. */
16060 static struct cgraph_simd_clone
*
16061 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
16062 bool *inbranch_specified
)
16064 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
16067 *inbranch_specified
= false;
16069 n
= args
.length ();
16070 if (n
> 0 && args
.last () == void_type_node
)
16073 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
16074 be cloned have a distinctive artificial label in addition to "omp
16078 && lookup_attribute ("cilk simd function",
16079 DECL_ATTRIBUTES (node
->decl
)));
16081 /* Allocate one more than needed just in case this is an in-branch
16082 clone which will require a mask argument. */
16083 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
16084 clone_info
->nargs
= n
;
16085 clone_info
->cilk_elemental
= cilk_clone
;
16092 clauses
= TREE_VALUE (clauses
);
16093 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
16096 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
16098 switch (OMP_CLAUSE_CODE (t
))
16100 case OMP_CLAUSE_INBRANCH
:
16101 clone_info
->inbranch
= 1;
16102 *inbranch_specified
= true;
16104 case OMP_CLAUSE_NOTINBRANCH
:
16105 clone_info
->inbranch
= 0;
16106 *inbranch_specified
= true;
16108 case OMP_CLAUSE_SIMDLEN
:
16109 clone_info
->simdlen
16110 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
16112 case OMP_CLAUSE_LINEAR
:
16114 tree decl
= OMP_CLAUSE_DECL (t
);
16115 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
16116 int argno
= TREE_INT_CST_LOW (decl
);
16117 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
16119 clone_info
->args
[argno
].arg_type
16120 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
16121 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
16122 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
16123 && clone_info
->args
[argno
].linear_step
< n
);
16127 if (POINTER_TYPE_P (args
[argno
]))
16128 step
= fold_convert (ssizetype
, step
);
16129 if (!tree_fits_shwi_p (step
))
16131 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
16132 "ignoring large linear step");
16136 else if (integer_zerop (step
))
16138 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
16139 "ignoring zero linear step");
16145 enum cgraph_simd_clone_arg_type arg_type
;
16146 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
16147 switch (OMP_CLAUSE_LINEAR_KIND (t
))
16149 case OMP_CLAUSE_LINEAR_REF
:
16151 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
16153 case OMP_CLAUSE_LINEAR_UVAL
:
16155 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
16157 case OMP_CLAUSE_LINEAR_VAL
:
16158 case OMP_CLAUSE_LINEAR_DEFAULT
:
16160 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
16163 gcc_unreachable ();
16166 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
16167 clone_info
->args
[argno
].arg_type
= arg_type
;
16168 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
16173 case OMP_CLAUSE_UNIFORM
:
16175 tree decl
= OMP_CLAUSE_DECL (t
);
16176 int argno
= tree_to_uhwi (decl
);
16177 clone_info
->args
[argno
].arg_type
16178 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
16181 case OMP_CLAUSE_ALIGNED
:
16183 tree decl
= OMP_CLAUSE_DECL (t
);
16184 int argno
= tree_to_uhwi (decl
);
16185 clone_info
->args
[argno
].alignment
16186 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
16197 /* Given a SIMD clone in NODE, calculate the characteristic data
16198 type and return the coresponding type. The characteristic data
16199 type is computed as described in the Intel Vector ABI. */
16202 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
16203 struct cgraph_simd_clone
*clone_info
)
16205 tree type
= integer_type_node
;
16206 tree fndecl
= node
->decl
;
16208 /* a) For non-void function, the characteristic data type is the
16210 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
16211 type
= TREE_TYPE (TREE_TYPE (fndecl
));
16213 /* b) If the function has any non-uniform, non-linear parameters,
16214 then the characteristic data type is the type of the first
16218 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
16219 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
16220 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
16228 /* c) If the characteristic data type determined by a) or b) above
16229 is struct, union, or class type which is pass-by-value (except
16230 for the type that maps to the built-in complex data type), the
16231 characteristic data type is int. */
16232 if (RECORD_OR_UNION_TYPE_P (type
)
16233 && !aggregate_value_p (type
, NULL
)
16234 && TREE_CODE (type
) != COMPLEX_TYPE
)
16235 return integer_type_node
;
16237 /* d) If none of the above three classes is applicable, the
16238 characteristic data type is int. */
16242 /* e) For Intel Xeon Phi native and offload compilation, if the
16243 resulting characteristic data type is 8-bit or 16-bit integer
16244 data type, the characteristic data type is int. */
16245 /* Well, we don't handle Xeon Phi yet. */
16249 simd_clone_mangle (struct cgraph_node
*node
,
16250 struct cgraph_simd_clone
*clone_info
)
16252 char vecsize_mangle
= clone_info
->vecsize_mangle
;
16253 char mask
= clone_info
->inbranch
? 'M' : 'N';
16254 unsigned int simdlen
= clone_info
->simdlen
;
16258 gcc_assert (vecsize_mangle
&& simdlen
);
16260 pp_string (&pp
, "_ZGV");
16261 pp_character (&pp
, vecsize_mangle
);
16262 pp_character (&pp
, mask
);
16263 pp_decimal_int (&pp
, simdlen
);
16265 for (n
= 0; n
< clone_info
->nargs
; ++n
)
16267 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
16269 switch (arg
.arg_type
)
16271 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
16272 pp_character (&pp
, 'u');
16274 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
16275 pp_character (&pp
, 'l');
16276 goto mangle_linear
;
16277 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
16278 pp_character (&pp
, 'R');
16279 goto mangle_linear
;
16280 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
16281 pp_character (&pp
, 'L');
16282 goto mangle_linear
;
16283 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
16284 pp_character (&pp
, 'U');
16285 goto mangle_linear
;
16287 gcc_assert (arg
.linear_step
!= 0);
16288 if (arg
.linear_step
> 1)
16289 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
16290 else if (arg
.linear_step
< 0)
16292 pp_character (&pp
, 'n');
16293 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
16297 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
16298 pp_character (&pp
, 's');
16299 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
16302 pp_character (&pp
, 'v');
16306 pp_character (&pp
, 'a');
16307 pp_decimal_int (&pp
, arg
.alignment
);
16311 pp_underscore (&pp
);
16312 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
16315 pp_string (&pp
, str
);
16316 str
= pp_formatted_text (&pp
);
16318 /* If there already is a SIMD clone with the same mangled name, don't
16319 add another one. This can happen e.g. for
16320 #pragma omp declare simd
16321 #pragma omp declare simd simdlen(8)
16322 int foo (int, int);
16323 if the simdlen is assumed to be 8 for the first one, etc. */
16324 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
16325 clone
= clone
->simdclone
->next_clone
)
16326 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
16330 return get_identifier (str
);
16333 /* Create a simd clone of OLD_NODE and return it. */
16335 static struct cgraph_node
*
16336 simd_clone_create (struct cgraph_node
*old_node
)
16338 struct cgraph_node
*new_node
;
16339 if (old_node
->definition
)
16341 if (!old_node
->has_gimple_body_p ())
16343 old_node
->get_body ();
16344 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
16350 tree old_decl
= old_node
->decl
;
16351 tree new_decl
= copy_node (old_node
->decl
);
16352 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
16353 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
16354 SET_DECL_RTL (new_decl
, NULL
);
16355 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
16356 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
16357 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
16358 if (old_node
->in_other_partition
)
16359 new_node
->in_other_partition
= 1;
16360 symtab
->call_cgraph_insertion_hooks (new_node
);
16362 if (new_node
== NULL
)
16365 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
16367 /* The function cgraph_function_versioning () will force the new
16368 symbol local. Undo this, and inherit external visability from
16370 new_node
->local
.local
= old_node
->local
.local
;
16371 new_node
->externally_visible
= old_node
->externally_visible
;
16376 /* Adjust the return type of the given function to its appropriate
16377 vector counterpart. Returns a simd array to be used throughout the
16378 function as a return value. */
16381 simd_clone_adjust_return_type (struct cgraph_node
*node
)
16383 tree fndecl
= node
->decl
;
16384 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
16385 unsigned int veclen
;
16388 /* Adjust the function return type. */
16389 if (orig_rettype
== void_type_node
)
16391 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
16392 t
= TREE_TYPE (TREE_TYPE (fndecl
));
16393 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
16394 veclen
= node
->simdclone
->vecsize_int
;
16396 veclen
= node
->simdclone
->vecsize_float
;
16397 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
16398 if (veclen
> node
->simdclone
->simdlen
)
16399 veclen
= node
->simdclone
->simdlen
;
16400 if (POINTER_TYPE_P (t
))
16401 t
= pointer_sized_int_node
;
16402 if (veclen
== node
->simdclone
->simdlen
)
16403 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
16406 t
= build_vector_type (t
, veclen
);
16407 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
16409 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
16410 if (!node
->definition
)
16413 t
= DECL_RESULT (fndecl
);
16414 /* Adjust the DECL_RESULT. */
16415 gcc_assert (TREE_TYPE (t
) != void_type_node
);
16416 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
16419 tree atype
= build_array_type_nelts (orig_rettype
,
16420 node
->simdclone
->simdlen
);
16421 if (veclen
!= node
->simdclone
->simdlen
)
16422 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
16424 /* Set up a SIMD array to use as the return value. */
16425 tree retval
= create_tmp_var_raw (atype
, "retval");
16426 gimple_add_tmp_var (retval
);
16430 /* Each vector argument has a corresponding array to be used locally
16431 as part of the eventual loop. Create such temporary array and
16434 PREFIX is the prefix to be used for the temporary.
16436 TYPE is the inner element type.
16438 SIMDLEN is the number of elements. */
16441 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
16443 tree atype
= build_array_type_nelts (type
, simdlen
);
16444 tree avar
= create_tmp_var_raw (atype
, prefix
);
16445 gimple_add_tmp_var (avar
);
16449 /* Modify the function argument types to their corresponding vector
16450 counterparts if appropriate. Also, create one array for each simd
16451 argument to be used locally when using the function arguments as
16454 NODE is the function whose arguments are to be adjusted.
16456 Returns an adjustment vector that will be filled describing how the
16457 argument types will be adjusted. */
16459 static ipa_parm_adjustment_vec
16460 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
16463 ipa_parm_adjustment_vec adjustments
;
16465 if (node
->definition
)
16466 args
= ipa_get_vector_of_formal_parms (node
->decl
);
16468 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
16469 adjustments
.create (args
.length ());
16470 unsigned i
, j
, veclen
;
16471 struct ipa_parm_adjustment adj
;
16472 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
16474 memset (&adj
, 0, sizeof (adj
));
16475 tree parm
= args
[i
];
16476 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
16477 adj
.base_index
= i
;
16480 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
16481 node
->simdclone
->args
[i
].orig_type
= parm_type
;
16483 switch (node
->simdclone
->args
[i
].arg_type
)
16486 /* No adjustment necessary for scalar arguments. */
16487 adj
.op
= IPA_PARM_OP_COPY
;
16489 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
16490 if (node
->definition
)
16491 node
->simdclone
->args
[i
].simd_array
16492 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
16493 TREE_TYPE (parm_type
),
16494 node
->simdclone
->simdlen
);
16495 adj
.op
= IPA_PARM_OP_COPY
;
16497 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
16498 case SIMD_CLONE_ARG_TYPE_VECTOR
:
16499 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
16500 veclen
= node
->simdclone
->vecsize_int
;
16502 veclen
= node
->simdclone
->vecsize_float
;
16503 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
16504 if (veclen
> node
->simdclone
->simdlen
)
16505 veclen
= node
->simdclone
->simdlen
;
16506 adj
.arg_prefix
= "simd";
16507 if (POINTER_TYPE_P (parm_type
))
16508 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
16510 adj
.type
= build_vector_type (parm_type
, veclen
);
16511 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
16512 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
16514 adjustments
.safe_push (adj
);
16517 memset (&adj
, 0, sizeof (adj
));
16518 adj
.op
= IPA_PARM_OP_NEW
;
16519 adj
.arg_prefix
= "simd";
16520 adj
.base_index
= i
;
16521 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
16525 if (node
->definition
)
16526 node
->simdclone
->args
[i
].simd_array
16527 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
16528 parm_type
, node
->simdclone
->simdlen
);
16530 adjustments
.safe_push (adj
);
16533 if (node
->simdclone
->inbranch
)
16536 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
16539 memset (&adj
, 0, sizeof (adj
));
16540 adj
.op
= IPA_PARM_OP_NEW
;
16541 adj
.arg_prefix
= "mask";
16543 adj
.base_index
= i
;
16544 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
16545 veclen
= node
->simdclone
->vecsize_int
;
16547 veclen
= node
->simdclone
->vecsize_float
;
16548 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
16549 if (veclen
> node
->simdclone
->simdlen
)
16550 veclen
= node
->simdclone
->simdlen
;
16551 if (POINTER_TYPE_P (base_type
))
16552 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
16554 adj
.type
= build_vector_type (base_type
, veclen
);
16555 adjustments
.safe_push (adj
);
16557 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
16558 adjustments
.safe_push (adj
);
16560 /* We have previously allocated one extra entry for the mask. Use
16562 struct cgraph_simd_clone
*sc
= node
->simdclone
;
16564 if (node
->definition
)
16566 sc
->args
[i
].orig_arg
16567 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
16568 sc
->args
[i
].simd_array
16569 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
16571 sc
->args
[i
].orig_type
= base_type
;
16572 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
16575 if (node
->definition
)
16576 ipa_modify_formal_parameters (node
->decl
, adjustments
);
16579 tree new_arg_types
= NULL_TREE
, new_reversed
;
16580 bool last_parm_void
= false;
16581 if (args
.length () > 0 && args
.last () == void_type_node
)
16582 last_parm_void
= true;
16584 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
16585 j
= adjustments
.length ();
16586 for (i
= 0; i
< j
; i
++)
16588 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
16590 if (adj
->op
== IPA_PARM_OP_COPY
)
16591 ptype
= args
[adj
->base_index
];
16594 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
16596 new_reversed
= nreverse (new_arg_types
);
16597 if (last_parm_void
)
16600 TREE_CHAIN (new_arg_types
) = void_list_node
;
16602 new_reversed
= void_list_node
;
16605 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
16606 TYPE_ARG_TYPES (new_type
) = new_reversed
;
16607 TREE_TYPE (node
->decl
) = new_type
;
16609 adjustments
.release ();
16612 return adjustments
;
16615 /* Initialize and copy the function arguments in NODE to their
16616 corresponding local simd arrays. Returns a fresh gimple_seq with
16617 the instruction sequence generated. */
16620 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
16621 ipa_parm_adjustment_vec adjustments
)
16623 gimple_seq seq
= NULL
;
16624 unsigned i
= 0, j
= 0, k
;
16626 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
16628 arg
= DECL_CHAIN (arg
), i
++, j
++)
16630 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
16631 || POINTER_TYPE_P (TREE_TYPE (arg
)))
16634 node
->simdclone
->args
[i
].vector_arg
= arg
;
16636 tree array
= node
->simdclone
->args
[i
].simd_array
;
16637 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
16639 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
16640 tree ptr
= build_fold_addr_expr (array
);
16641 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
16642 build_int_cst (ptype
, 0));
16643 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
16644 gimplify_and_add (t
, &seq
);
16648 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
16649 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
16650 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
16652 tree ptr
= build_fold_addr_expr (array
);
16656 arg
= DECL_CHAIN (arg
);
16660 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
16661 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
16662 build_int_cst (ptype
, k
* elemsize
));
16663 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
16664 gimplify_and_add (t
, &seq
);
16671 /* Callback info for ipa_simd_modify_stmt_ops below. */
16673 struct modify_stmt_info
{
16674 ipa_parm_adjustment_vec adjustments
;
16676 /* True if the parent statement was modified by
16677 ipa_simd_modify_stmt_ops. */
16681 /* Callback for walk_gimple_op.
16683 Adjust operands from a given statement as specified in the
16684 adjustments vector in the callback data. */
16687 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
16689 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16690 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
16691 tree
*orig_tp
= tp
;
16692 if (TREE_CODE (*tp
) == ADDR_EXPR
)
16693 tp
= &TREE_OPERAND (*tp
, 0);
16694 struct ipa_parm_adjustment
*cand
= NULL
;
16695 if (TREE_CODE (*tp
) == PARM_DECL
)
16696 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
16700 *walk_subtrees
= 0;
16703 tree repl
= NULL_TREE
;
16705 repl
= unshare_expr (cand
->new_decl
);
16710 *walk_subtrees
= 0;
16711 bool modified
= info
->modified
;
16712 info
->modified
= false;
16713 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
16714 if (!info
->modified
)
16716 info
->modified
= modified
;
16719 info
->modified
= modified
;
16728 repl
= build_fold_addr_expr (repl
);
16730 if (is_gimple_debug (info
->stmt
))
16732 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
16733 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
16734 DECL_ARTIFICIAL (vexpr
) = 1;
16735 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
16736 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
16741 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
16742 repl
= gimple_assign_lhs (stmt
);
16744 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
16745 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
16748 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
16750 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
16756 info
->modified
= true;
16760 /* Traverse the function body and perform all modifications as
16761 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
16762 modified such that the replacement/reduction value will now be an
16763 offset into the corresponding simd_array.
16765 This function will replace all function argument uses with their
16766 corresponding simd array elements, and ajust the return values
16770 ipa_simd_modify_function_body (struct cgraph_node
*node
,
16771 ipa_parm_adjustment_vec adjustments
,
16772 tree retval_array
, tree iter
)
16775 unsigned int i
, j
, l
;
16777 /* Re-use the adjustments array, but this time use it to replace
16778 every function argument use to an offset into the corresponding
16780 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
16782 if (!node
->simdclone
->args
[i
].vector_arg
)
16785 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
16786 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
16787 adjustments
[j
].new_decl
16788 = build4 (ARRAY_REF
,
16790 node
->simdclone
->args
[i
].simd_array
,
16792 NULL_TREE
, NULL_TREE
);
16793 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
16794 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
16795 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
16798 l
= adjustments
.length ();
16799 for (i
= 1; i
< num_ssa_names
; i
++)
16801 tree name
= ssa_name (i
);
16803 && SSA_NAME_VAR (name
)
16804 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
16806 for (j
= 0; j
< l
; j
++)
16807 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
16808 && adjustments
[j
].new_decl
)
16811 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
16814 = copy_var_decl (adjustments
[j
].base
,
16815 DECL_NAME (adjustments
[j
].base
),
16816 TREE_TYPE (adjustments
[j
].base
));
16817 adjustments
[j
].new_ssa_base
= base_var
;
16820 base_var
= adjustments
[j
].new_ssa_base
;
16821 if (SSA_NAME_IS_DEFAULT_DEF (name
))
16823 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
16824 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
16825 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
16826 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
16827 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
16828 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
16829 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
16830 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
16833 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
16838 struct modify_stmt_info info
;
16839 info
.adjustments
= adjustments
;
16841 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
16843 gimple_stmt_iterator gsi
;
16845 gsi
= gsi_start_bb (bb
);
16846 while (!gsi_end_p (gsi
))
16848 gimple
*stmt
= gsi_stmt (gsi
);
16850 struct walk_stmt_info wi
;
16852 memset (&wi
, 0, sizeof (wi
));
16853 info
.modified
= false;
16855 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
16857 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
16859 tree retval
= gimple_return_retval (return_stmt
);
16862 gsi_remove (&gsi
, true);
16866 /* Replace `return foo' with `retval_array[iter] = foo'. */
16867 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
16868 retval_array
, iter
, NULL
, NULL
);
16869 stmt
= gimple_build_assign (ref
, retval
);
16870 gsi_replace (&gsi
, stmt
, true);
16871 info
.modified
= true;
16876 update_stmt (stmt
);
16877 if (maybe_clean_eh_stmt (stmt
))
16878 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
16885 /* Adjust the argument types in NODE to their appropriate vector
16889 simd_clone_adjust (struct cgraph_node
*node
)
16891 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
16893 targetm
.simd_clone
.adjust (node
);
16895 tree retval
= simd_clone_adjust_return_type (node
);
16896 ipa_parm_adjustment_vec adjustments
16897 = simd_clone_adjust_argument_types (node
);
16899 push_gimplify_context ();
16901 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
16903 /* Adjust all uses of vector arguments accordingly. Adjust all
16904 return values accordingly. */
16905 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
16906 tree iter1
= make_ssa_name (iter
);
16907 tree iter2
= make_ssa_name (iter
);
16908 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
16910 /* Initialize the iteration variable. */
16911 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
16912 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
16913 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
16914 /* Insert the SIMD array and iv initialization at function
16916 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
16918 pop_gimplify_context (NULL
);
16920 /* Create a new BB right before the original exit BB, to hold the
16921 iteration increment and the condition/branch. */
16922 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
16923 basic_block incr_bb
= create_empty_bb (orig_exit
);
16924 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
16925 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
16926 flag. Set it now to be a FALLTHRU_EDGE. */
16927 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
16928 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
16929 for (unsigned i
= 0;
16930 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
16932 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
16933 redirect_edge_succ (e
, incr_bb
);
16935 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
16936 e
->probability
= REG_BR_PROB_BASE
;
16937 gsi
= gsi_last_bb (incr_bb
);
16938 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
16939 build_int_cst (unsigned_type_node
, 1));
16940 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
16942 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
16943 struct loop
*loop
= alloc_loop ();
16944 cfun
->has_force_vectorize_loops
= true;
16945 loop
->safelen
= node
->simdclone
->simdlen
;
16946 loop
->force_vectorize
= true;
16947 loop
->header
= body_bb
;
16949 /* Branch around the body if the mask applies. */
16950 if (node
->simdclone
->inbranch
)
16952 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
16954 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
16955 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
16956 tree aref
= build4 (ARRAY_REF
,
16957 TREE_TYPE (TREE_TYPE (mask_array
)),
16960 g
= gimple_build_assign (mask
, aref
);
16961 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
16962 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
16963 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
16965 aref
= build1 (VIEW_CONVERT_EXPR
,
16966 build_nonstandard_integer_type (bitsize
, 0), mask
);
16967 mask
= make_ssa_name (TREE_TYPE (aref
));
16968 g
= gimple_build_assign (mask
, aref
);
16969 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
16972 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
16974 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
16975 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
16976 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
16979 /* Generate the condition. */
16980 g
= gimple_build_cond (LT_EXPR
,
16982 build_int_cst (unsigned_type_node
,
16983 node
->simdclone
->simdlen
),
16985 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
16986 e
= split_block (incr_bb
, gsi_stmt (gsi
));
16987 basic_block latch_bb
= e
->dest
;
16988 basic_block new_exit_bb
;
16989 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
16990 loop
->latch
= latch_bb
;
16992 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
16994 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
16995 /* The successor of incr_bb is already pointing to latch_bb; just
16997 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
16998 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
17000 gphi
*phi
= create_phi_node (iter1
, body_bb
);
17001 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
17002 edge latch_edge
= single_succ_edge (latch_bb
);
17003 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
17005 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17007 /* Generate the new return. */
17008 gsi
= gsi_last_bb (new_exit_bb
);
17010 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
17011 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
17012 retval
= TREE_OPERAND (retval
, 0);
17015 retval
= build1 (VIEW_CONVERT_EXPR
,
17016 TREE_TYPE (TREE_TYPE (node
->decl
)),
17018 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
17019 false, GSI_CONTINUE_LINKING
);
17021 g
= gimple_build_return (retval
);
17022 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17024 /* Handle aligned clauses by replacing default defs of the aligned
17025 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
17026 lhs. Handle linear by adding PHIs. */
17027 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
17028 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
17029 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
17030 || !is_gimple_reg_type
17031 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
17033 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17034 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
17035 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
17038 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
17039 gimple_add_tmp_var (iter1
);
17041 gsi
= gsi_after_labels (entry_bb
);
17042 g
= gimple_build_assign (iter1
, orig_arg
);
17043 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17044 gsi
= gsi_after_labels (body_bb
);
17045 g
= gimple_build_assign (orig_arg
, iter1
);
17046 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17048 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
17049 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
17050 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
17052 && TREE_ADDRESSABLE
17053 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
17055 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17056 tree def
= ssa_default_def (cfun
, orig_arg
);
17057 if (def
&& !has_zero_uses (def
))
17059 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
17060 gimple_add_tmp_var (iter1
);
17061 gsi
= gsi_after_labels (entry_bb
);
17062 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
17063 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17064 gsi
= gsi_after_labels (body_bb
);
17065 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
17066 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17069 else if (node
->simdclone
->args
[i
].alignment
17070 && node
->simdclone
->args
[i
].arg_type
17071 == SIMD_CLONE_ARG_TYPE_UNIFORM
17072 && (node
->simdclone
->args
[i
].alignment
17073 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
17074 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
17077 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
17078 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17079 tree def
= ssa_default_def (cfun
, orig_arg
);
17080 if (def
&& !has_zero_uses (def
))
17082 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
17083 gimple_seq seq
= NULL
;
17084 bool need_cvt
= false;
17086 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
17088 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
17091 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
17092 gimple_call_set_lhs (g
, t
);
17093 gimple_seq_add_stmt_without_update (&seq
, g
);
17096 t
= make_ssa_name (orig_arg
);
17097 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
17098 gimple_seq_add_stmt_without_update (&seq
, g
);
17100 gsi_insert_seq_on_edge_immediate
17101 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
17103 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17104 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
17106 node
->create_edge (cgraph_node::get_create (fn
),
17107 call
, entry_bb
->count
, freq
);
17109 imm_use_iterator iter
;
17110 use_operand_p use_p
;
17112 tree repl
= gimple_get_lhs (g
);
17113 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
17114 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
17117 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
17118 SET_USE (use_p
, repl
);
17121 else if ((node
->simdclone
->args
[i
].arg_type
17122 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
17123 || (node
->simdclone
->args
[i
].arg_type
17124 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
))
17126 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17127 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
17128 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
17129 tree def
= NULL_TREE
;
17130 if (TREE_ADDRESSABLE (orig_arg
))
17132 def
= make_ssa_name (TREE_TYPE (orig_arg
));
17133 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
17134 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
17135 gsi
= gsi_after_labels (entry_bb
);
17136 g
= gimple_build_assign (def
, orig_arg
);
17137 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17141 def
= ssa_default_def (cfun
, orig_arg
);
17142 if (!def
|| has_zero_uses (def
))
17146 iter1
= make_ssa_name (orig_arg
);
17147 iter2
= make_ssa_name (orig_arg
);
17152 phi
= create_phi_node (iter1
, body_bb
);
17153 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
17154 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17155 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
17156 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
17157 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
17158 ? TREE_TYPE (orig_arg
) : sizetype
;
17160 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
17161 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
17162 gsi
= gsi_last_bb (incr_bb
);
17163 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17165 imm_use_iterator iter
;
17166 use_operand_p use_p
;
17168 if (TREE_ADDRESSABLE (orig_arg
))
17170 gsi
= gsi_after_labels (body_bb
);
17171 g
= gimple_build_assign (orig_arg
, iter1
);
17172 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17175 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
17176 if (use_stmt
== phi
)
17179 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
17180 SET_USE (use_p
, iter1
);
17183 else if (node
->simdclone
->args
[i
].arg_type
17184 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
)
17186 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17187 tree def
= ssa_default_def (cfun
, orig_arg
);
17188 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
17189 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
17190 if (def
&& !has_zero_uses (def
))
17192 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
17193 iter1
= make_ssa_name (orig_arg
);
17194 iter2
= make_ssa_name (orig_arg
);
17195 tree iter3
= make_ssa_name (rtype
);
17196 tree iter4
= make_ssa_name (rtype
);
17197 tree iter5
= make_ssa_name (rtype
);
17198 gsi
= gsi_after_labels (entry_bb
);
17200 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
17201 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
17203 tree array
= node
->simdclone
->args
[i
].simd_array
;
17204 TREE_ADDRESSABLE (array
) = 1;
17205 tree ptr
= build_fold_addr_expr (array
);
17206 phi
= create_phi_node (iter1
, body_bb
);
17207 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
17208 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17209 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
17210 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
17211 gsi
= gsi_last_bb (incr_bb
);
17212 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17214 phi
= create_phi_node (iter4
, body_bb
);
17215 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
17216 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
17217 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
17218 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
17219 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
17220 ? TREE_TYPE (iter3
) : sizetype
;
17222 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
17223 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
17224 gsi
= gsi_last_bb (incr_bb
);
17225 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17227 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
17228 gsi
= gsi_after_labels (body_bb
);
17229 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17231 imm_use_iterator iter
;
17232 use_operand_p use_p
;
17234 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
17235 if (use_stmt
== load
)
17238 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
17239 SET_USE (use_p
, iter1
);
17241 if (!TYPE_READONLY (rtype
))
17243 tree v
= make_ssa_name (rtype
);
17244 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
17245 size_zero_node
, NULL_TREE
,
17247 gsi
= gsi_after_labels (new_exit_bb
);
17248 g
= gimple_build_assign (v
, aref
);
17249 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17250 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
17251 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17256 calculate_dominance_info (CDI_DOMINATORS
);
17257 add_loop (loop
, loop
->header
->loop_father
);
17258 update_ssa (TODO_update_ssa
);
17263 /* If the function in NODE is tagged as an elemental SIMD function,
17264 create the appropriate SIMD clones. */
17267 expand_simd_clones (struct cgraph_node
*node
)
17269 tree attr
= lookup_attribute ("omp declare simd",
17270 DECL_ATTRIBUTES (node
->decl
));
17271 if (attr
== NULL_TREE
17272 || node
->global
.inlined_to
17273 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
17277 #pragma omp declare simd
17279 in C, there we don't know the argument types at all. */
17280 if (!node
->definition
17281 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
17286 /* Start with parsing the "omp declare simd" attribute(s). */
17287 bool inbranch_clause_specified
;
17288 struct cgraph_simd_clone
*clone_info
17289 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
17290 &inbranch_clause_specified
);
17291 if (clone_info
== NULL
)
17294 int orig_simdlen
= clone_info
->simdlen
;
17295 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
17296 /* The target can return 0 (no simd clones should be created),
17297 1 (just one ISA of simd clones should be created) or higher
17298 count of ISA variants. In that case, clone_info is initialized
17299 for the first ISA variant. */
17301 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
17306 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
17307 also create one inbranch and one !inbranch clone of it. */
17308 for (int i
= 0; i
< count
* 2; i
++)
17310 struct cgraph_simd_clone
*clone
= clone_info
;
17311 if (inbranch_clause_specified
&& (i
& 1) != 0)
17316 clone
= simd_clone_struct_alloc (clone_info
->nargs
17318 simd_clone_struct_copy (clone
, clone_info
);
17319 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
17320 and simd_clone_adjust_argument_types did to the first
17322 clone
->nargs
-= clone_info
->inbranch
;
17323 clone
->simdlen
= orig_simdlen
;
17324 /* And call the target hook again to get the right ISA. */
17325 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
17329 clone
->inbranch
= 1;
17332 /* simd_clone_mangle might fail if such a clone has been created
17334 tree id
= simd_clone_mangle (node
, clone
);
17335 if (id
== NULL_TREE
)
17338 /* Only when we are sure we want to create the clone actually
17339 clone the function (or definitions) or create another
17340 extern FUNCTION_DECL (for prototypes without definitions). */
17341 struct cgraph_node
*n
= simd_clone_create (node
);
17345 n
->simdclone
= clone
;
17346 clone
->origin
= node
;
17347 clone
->next_clone
= NULL
;
17348 if (node
->simd_clones
== NULL
)
17350 clone
->prev_clone
= n
;
17351 node
->simd_clones
= n
;
17355 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
17356 clone
->prev_clone
->simdclone
->next_clone
= n
;
17357 node
->simd_clones
->simdclone
->prev_clone
= n
;
17359 symtab
->change_decl_assembler_name (n
->decl
, id
);
17360 /* And finally adjust the return type, parameters and for
17361 definitions also function body. */
17362 if (node
->definition
)
17363 simd_clone_adjust (n
);
17366 simd_clone_adjust_return_type (n
);
17367 simd_clone_adjust_argument_types (n
);
17371 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
17374 /* Entry point for IPA simd clone creation pass. */
17376 static unsigned int
17377 ipa_omp_simd_clone (void)
17379 struct cgraph_node
*node
;
17380 FOR_EACH_FUNCTION (node
)
17381 expand_simd_clones (node
);
17387 const pass_data pass_data_omp_simd_clone
=
17389 SIMPLE_IPA_PASS
, /* type */
17390 "simdclone", /* name */
17391 OPTGROUP_NONE
, /* optinfo_flags */
17392 TV_NONE
, /* tv_id */
17393 ( PROP_ssa
| PROP_cfg
), /* properties_required */
17394 0, /* properties_provided */
17395 0, /* properties_destroyed */
17396 0, /* todo_flags_start */
17397 0, /* todo_flags_finish */
17400 class pass_omp_simd_clone
: public simple_ipa_opt_pass
17403 pass_omp_simd_clone(gcc::context
*ctxt
)
17404 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
17407 /* opt_pass methods: */
17408 virtual bool gate (function
*);
17409 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
17413 pass_omp_simd_clone::gate (function
*)
17415 return ((flag_openmp
|| flag_openmp_simd
17417 || (in_lto_p
&& !flag_wpa
))
17418 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
17421 } // anon namespace
17423 simple_ipa_opt_pass
*
17424 make_pass_omp_simd_clone (gcc::context
*ctxt
)
17426 return new pass_omp_simd_clone (ctxt
);
17429 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
17430 adds their addresses and sizes to constructor-vector V_CTOR. */
17432 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
17433 vec
<constructor_elt
, va_gc
> *v_ctor
)
17435 unsigned len
= vec_safe_length (v_decls
);
17436 for (unsigned i
= 0; i
< len
; i
++)
17438 tree it
= (*v_decls
)[i
];
17439 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
17441 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
17443 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
17444 fold_convert (const_ptr_type_node
,
17445 DECL_SIZE_UNIT (it
)));
17449 /* Create new symbols containing (address, size) pairs for global variables,
17450 marked with "omp declare target" attribute, as well as addresses for the
17451 functions, which are outlined offloading regions. */
17453 omp_finish_file (void)
17455 unsigned num_funcs
= vec_safe_length (offload_funcs
);
17456 unsigned num_vars
= vec_safe_length (offload_vars
);
17458 if (num_funcs
== 0 && num_vars
== 0)
17461 if (targetm_common
.have_named_sections
)
17463 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
17464 vec_alloc (v_f
, num_funcs
);
17465 vec_alloc (v_v
, num_vars
* 2);
17467 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
17468 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
17470 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
17472 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
17474 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
17475 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
17476 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
17477 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
17478 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
17479 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
17480 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
17481 get_identifier (".offload_func_table"),
17483 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
17484 get_identifier (".offload_var_table"),
17486 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
17487 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
17488 otherwise a joint table in a binary will contain padding between
17489 tables from multiple object files. */
17490 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
17491 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
17492 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
17493 DECL_INITIAL (funcs_decl
) = ctor_f
;
17494 DECL_INITIAL (vars_decl
) = ctor_v
;
17495 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
17496 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
17498 varpool_node::finalize_decl (vars_decl
);
17499 varpool_node::finalize_decl (funcs_decl
);
17503 for (unsigned i
= 0; i
< num_funcs
; i
++)
17505 tree it
= (*offload_funcs
)[i
];
17506 targetm
.record_offload_symbol (it
);
17508 for (unsigned i
= 0; i
< num_vars
; i
++)
17510 tree it
= (*offload_vars
)[i
];
17511 targetm
.record_offload_symbol (it
);
17516 /* Validate and update the dimensions for offloaded FN. ATTRS is the
17517 raw attribute. DIMS is an array of dimensions, which is returned.
17518 Returns the function level dimensionality -- the level at which an
17519 offload routine wishes to partition a loop. */
17522 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
17524 tree purpose
[GOMP_DIM_MAX
];
17526 tree pos
= TREE_VALUE (attrs
);
17529 /* Make sure the attribute creator attached the dimension
17533 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
17535 purpose
[ix
] = TREE_PURPOSE (pos
);
17539 if (integer_zerop (purpose
[ix
]))
17541 else if (fn_level
< 0)
17545 tree val
= TREE_VALUE (pos
);
17546 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
17547 pos
= TREE_CHAIN (pos
);
17550 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
17552 /* Default anything left to 1. */
17553 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
17562 /* Replace the attribute with new values. */
17564 for (ix
= GOMP_DIM_MAX
; ix
--;)
17565 pos
= tree_cons (purpose
[ix
],
17566 build_int_cst (integer_type_node
, dims
[ix
]),
17568 replace_oacc_fn_attrib (fn
, pos
);
17574 /* Main entry point for oacc transformations which run on the device
17575 compiler after LTO, so we know what the target device is at this
17576 point (including the host fallback). */
17578 static unsigned int
17579 execute_oacc_device_lower ()
17581 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
17582 int dims
[GOMP_DIM_MAX
];
17585 /* Not an offloaded function. */
17588 oacc_validate_dims (current_function_decl
, attrs
, dims
);
17593 /* Default launch dimension validator. Force everything to 1. A
17594 backend that wants to provide larger dimensions must override this
17598 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
17599 int ARG_UNUSED (fn_level
))
17601 bool changed
= false;
17603 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
17617 const pass_data pass_data_oacc_device_lower
=
17619 GIMPLE_PASS
, /* type */
17620 "oaccdevlow", /* name */
17621 OPTGROUP_NONE
, /* optinfo_flags */
17622 TV_NONE
, /* tv_id */
17623 PROP_cfg
, /* properties_required */
17624 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
17625 0, /* properties_destroyed */
17626 0, /* todo_flags_start */
17627 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
17630 class pass_oacc_device_lower
: public gimple_opt_pass
17633 pass_oacc_device_lower (gcc::context
*ctxt
)
17634 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
17637 /* opt_pass methods: */
17638 virtual unsigned int execute (function
*)
17640 bool gate
= (flag_openacc
!= 0 && !seen_error ());
17645 return execute_oacc_device_lower ();
17648 }; // class pass_oacc_transform
17650 } // anon namespace
17653 make_pass_oacc_device_lower (gcc::context
*ctxt
)
17655 return new pass_oacc_device_lower (ctxt
);
17658 #include "gt-omp-low.h"