1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
37 #include "hard-reg-set.h"
40 #include "dominance.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
60 #include "plugin-api.h"
64 #include "tree-phinodes.h"
65 #include "ssa-iterators.h"
66 #include "tree-ssanames.h"
67 #include "tree-into-ssa.h"
73 #include "tree-pass.h"
75 #include "splay-tree.h"
76 #include "insn-codes.h"
80 #include "common/common-target.h"
82 #include "gimple-low.h"
83 #include "tree-cfgcleanup.h"
84 #include "pretty-print.h"
85 #include "alloc-pool.h"
87 #include "tree-nested.h"
91 #include "lto-section-names.h"
94 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
95 phases. The first phase scans the function looking for OMP statements
96 and then for variables that must be replaced to satisfy data sharing
97 clauses. The second phase expands code for the constructs, as well as
98 re-gimplifying things when variables have been replaced with complex
101 Final code generation is done by pass_expand_omp. The flowgraph is
102 scanned for parallel regions which are then moved to a new
103 function, to be invoked by the thread library. */
105 /* Parallel region information. Every parallel and workshare
106 directive is enclosed between two markers, the OMP_* directive
107 and a corresponding OMP_RETURN statement. */
111 /* The enclosing region. */
112 struct omp_region
*outer
;
114 /* First child region. */
115 struct omp_region
*inner
;
117 /* Next peer region. */
118 struct omp_region
*next
;
120 /* Block containing the omp directive as its last stmt. */
123 /* Block containing the OMP_RETURN as its last stmt. */
126 /* Block containing the OMP_CONTINUE as its last stmt. */
129 /* If this is a combined parallel+workshare region, this is a list
130 of additional arguments needed by the combined parallel+workshare
132 vec
<tree
, va_gc
> *ws_args
;
134 /* The code for the omp directive of this region. */
135 enum gimple_code type
;
137 /* Schedule kind, only used for OMP_FOR type regions. */
138 enum omp_clause_schedule_kind sched_kind
;
140 /* True if this is a combined parallel+workshare region. */
141 bool is_combined_parallel
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
147 typedef struct omp_context
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* A map of reduction pointer variables. For accelerators, each
179 reduction variable is replaced with an array. Each thread, in turn,
180 is assigned to a slot on that array. */
181 splay_tree reduction_map
;
183 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
184 barriers should jump to during omplower pass. */
187 /* What to do with variables with implicitly determined sharing
189 enum omp_clause_default_kind default_kind
;
191 /* Nesting depth of this context. Used to beautify error messages re
192 invalid gotos. The outermost ctx is depth 1, with depth 0 being
193 reserved for the main body of the function. */
196 /* True if this parallel directive is nested within another. */
199 /* True if this construct can be cancelled. */
203 /* A structure holding the elements of:
204 for (V = N1; V cond N2; V += STEP) [...] */
206 struct omp_for_data_loop
208 tree v
, n1
, n2
, step
;
209 enum tree_code cond_code
;
212 /* A structure describing the main elements of a parallel loop. */
216 struct omp_for_data_loop loop
;
221 bool have_nowait
, have_ordered
;
222 enum omp_clause_schedule_kind sched_kind
;
223 struct omp_for_data_loop
*loops
;
227 static splay_tree all_contexts
;
228 static int taskreg_nesting_level
;
229 static int target_nesting_level
;
230 static struct omp_region
*root_omp_region
;
231 static bitmap task_shared_vars
;
232 static vec
<omp_context
*> taskreg_contexts
;
234 static void scan_omp (gimple_seq
*, omp_context
*);
235 static tree
scan_omp_1_op (tree
*, int *, void *);
237 #define WALK_SUBSTMTS \
241 case GIMPLE_EH_FILTER: \
242 case GIMPLE_TRANSACTION: \
243 /* The sub-statements for these should be walked. */ \
244 *handled_ops_p = false; \
247 /* Helper function to get the name of the array containing the partial
248 reductions for OpenACC reductions. */
250 oacc_get_reduction_array_id (tree node
)
252 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
253 int len
= strlen ("OACC") + strlen (id
);
254 char *temp_name
= XALLOCAVEC (char, len
+ 1);
255 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
256 return IDENTIFIER_POINTER (get_identifier (temp_name
));
259 /* Determine the number of threads OpenACC threads used to determine the
260 size of the array of partial reductions. Currently, this is num_gangs
261 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
262 because it is independed of the device used. */
265 oacc_max_threads (omp_context
*ctx
)
267 tree nthreads
, vector_length
, gangs
, clauses
;
269 gangs
= fold_convert (sizetype
, integer_one_node
);
270 vector_length
= gangs
;
272 /* The reduction clause may be nested inside a loop directive.
273 Scan for the innermost vector_length clause. */
274 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
276 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
277 || (gimple_omp_target_kind (oc
->stmt
)
278 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
281 clauses
= gimple_omp_target_clauses (oc
->stmt
);
283 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
285 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
287 OMP_CLAUSE_VECTOR_LENGTH_EXPR
290 vector_length
= fold_convert (sizetype
, integer_one_node
);
292 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
294 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
295 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
297 gangs
= fold_convert (sizetype
, integer_one_node
);
302 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
307 /* Holds offload tables with decls. */
308 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
310 /* Holds a decl for __OFFLOAD_TABLE__. */
311 static GTY(()) tree offload_symbol_decl
;
313 /* Get the __OFFLOAD_TABLE__ symbol. */
315 get_offload_symbol_decl (void)
317 if (!offload_symbol_decl
)
319 tree decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
320 get_identifier ("__OFFLOAD_TABLE__"),
322 TREE_ADDRESSABLE (decl
) = 1;
323 TREE_PUBLIC (decl
) = 1;
324 DECL_EXTERNAL (decl
) = 1;
325 DECL_WEAK (decl
) = 1;
326 DECL_ATTRIBUTES (decl
)
327 = tree_cons (get_identifier ("weak"),
328 NULL_TREE
, DECL_ATTRIBUTES (decl
));
329 offload_symbol_decl
= decl
;
331 return offload_symbol_decl
;
334 /* Convenience function for calling scan_omp_1_op on tree operands. */
337 scan_omp_op (tree
*tp
, omp_context
*ctx
)
339 struct walk_stmt_info wi
;
341 memset (&wi
, 0, sizeof (wi
));
343 wi
.want_locations
= true;
345 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
348 static void lower_omp (gimple_seq
*, omp_context
*);
349 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
350 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
352 /* Find an OpenMP clause of type KIND within CLAUSES. */
355 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
357 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
358 if (OMP_CLAUSE_CODE (clauses
) == kind
)
364 /* Return true if CTX is for an omp parallel. */
367 is_parallel_ctx (omp_context
*ctx
)
369 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
373 /* Return true if CTX is for an omp task. */
376 is_task_ctx (omp_context
*ctx
)
378 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
382 /* Return true if CTX is for an omp parallel or omp task. */
385 is_taskreg_ctx (omp_context
*ctx
)
387 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
388 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
392 /* Return true if REGION is a combined parallel+workshare region. */
395 is_combined_parallel (struct omp_region
*region
)
397 return region
->is_combined_parallel
;
401 /* Extract the header elements of parallel loop FOR_STMT and store
405 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
406 struct omp_for_data_loop
*loops
)
408 tree t
, var
, *collapse_iter
, *collapse_count
;
409 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
410 struct omp_for_data_loop
*loop
;
412 struct omp_for_data_loop dummy_loop
;
413 location_t loc
= gimple_location (for_stmt
);
414 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
415 bool distribute
= gimple_omp_for_kind (for_stmt
)
416 == GF_OMP_FOR_KIND_DISTRIBUTE
;
418 fd
->for_stmt
= for_stmt
;
420 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
421 if (fd
->collapse
> 1)
424 fd
->loops
= &fd
->loop
;
426 fd
->have_nowait
= (gimple_omp_for_kind (for_stmt
) != GF_OMP_FOR_KIND_FOR
);
427 fd
->have_ordered
= false;
428 fd
->sched_kind
= /* TODO: OACC_LOOP */ OMP_CLAUSE_SCHEDULE_STATIC
;
429 fd
->chunk_size
= NULL_TREE
;
430 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
431 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
432 collapse_iter
= NULL
;
433 collapse_count
= NULL
;
435 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
436 switch (OMP_CLAUSE_CODE (t
))
438 case OMP_CLAUSE_NOWAIT
:
439 fd
->have_nowait
= true;
441 case OMP_CLAUSE_ORDERED
:
442 fd
->have_ordered
= true;
444 case OMP_CLAUSE_SCHEDULE
:
445 gcc_assert (!distribute
);
446 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
447 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
449 case OMP_CLAUSE_DIST_SCHEDULE
:
450 gcc_assert (distribute
);
451 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
453 case OMP_CLAUSE_COLLAPSE
:
454 if (fd
->collapse
> 1)
456 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
457 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
464 /* FIXME: for now map schedule(auto) to schedule(static).
465 There should be analysis to determine whether all iterations
466 are approximately the same amount of work (then schedule(static)
467 is best) or if it varies (then schedule(dynamic,N) is better). */
468 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
470 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
471 gcc_assert (fd
->chunk_size
== NULL
);
473 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
474 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
475 gcc_assert (fd
->chunk_size
== NULL
);
476 else if (fd
->chunk_size
== NULL
)
478 /* We only need to compute a default chunk size for ordered
479 static loops and dynamic loops. */
480 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
482 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
483 ? integer_zero_node
: integer_one_node
;
486 for (i
= 0; i
< fd
->collapse
; i
++)
488 if (fd
->collapse
== 1)
490 else if (loops
!= NULL
)
495 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
496 gcc_assert (SSA_VAR_P (loop
->v
));
497 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
498 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
499 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
500 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
502 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
503 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
504 switch (loop
->cond_code
)
510 gcc_assert (gimple_omp_for_kind (for_stmt
)
511 == GF_OMP_FOR_KIND_CILKSIMD
512 || (gimple_omp_for_kind (for_stmt
)
513 == GF_OMP_FOR_KIND_CILKFOR
));
516 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
517 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
519 loop
->n2
= fold_build2_loc (loc
,
520 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
521 build_int_cst (TREE_TYPE (loop
->n2
), 1));
522 loop
->cond_code
= LT_EXPR
;
525 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
526 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
528 loop
->n2
= fold_build2_loc (loc
,
529 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
530 build_int_cst (TREE_TYPE (loop
->n2
), 1));
531 loop
->cond_code
= GT_EXPR
;
537 t
= gimple_omp_for_incr (for_stmt
, i
);
538 gcc_assert (TREE_OPERAND (t
, 0) == var
);
539 switch (TREE_CODE (t
))
542 loop
->step
= TREE_OPERAND (t
, 1);
544 case POINTER_PLUS_EXPR
:
545 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
548 loop
->step
= TREE_OPERAND (t
, 1);
549 loop
->step
= fold_build1_loc (loc
,
550 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
558 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
559 && !fd
->have_ordered
))
561 if (fd
->collapse
== 1)
562 iter_type
= TREE_TYPE (loop
->v
);
564 || TYPE_PRECISION (iter_type
)
565 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
567 = build_nonstandard_integer_type
568 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
570 else if (iter_type
!= long_long_unsigned_type_node
)
572 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
573 iter_type
= long_long_unsigned_type_node
;
574 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
575 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
576 >= TYPE_PRECISION (iter_type
))
580 if (loop
->cond_code
== LT_EXPR
)
581 n
= fold_build2_loc (loc
,
582 PLUS_EXPR
, TREE_TYPE (loop
->v
),
583 loop
->n2
, loop
->step
);
586 if (TREE_CODE (n
) != INTEGER_CST
587 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
588 iter_type
= long_long_unsigned_type_node
;
590 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
591 > TYPE_PRECISION (iter_type
))
595 if (loop
->cond_code
== LT_EXPR
)
598 n2
= fold_build2_loc (loc
,
599 PLUS_EXPR
, TREE_TYPE (loop
->v
),
600 loop
->n2
, loop
->step
);
604 n1
= fold_build2_loc (loc
,
605 MINUS_EXPR
, TREE_TYPE (loop
->v
),
606 loop
->n2
, loop
->step
);
609 if (TREE_CODE (n1
) != INTEGER_CST
610 || TREE_CODE (n2
) != INTEGER_CST
611 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
612 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
613 iter_type
= long_long_unsigned_type_node
;
617 if (collapse_count
&& *collapse_count
== NULL
)
619 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
620 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
621 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
622 if (t
&& integer_zerop (t
))
623 count
= build_zero_cst (long_long_unsigned_type_node
);
624 else if ((i
== 0 || count
!= NULL_TREE
)
625 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
626 && TREE_CONSTANT (loop
->n1
)
627 && TREE_CONSTANT (loop
->n2
)
628 && TREE_CODE (loop
->step
) == INTEGER_CST
)
630 tree itype
= TREE_TYPE (loop
->v
);
632 if (POINTER_TYPE_P (itype
))
633 itype
= signed_type_for (itype
);
634 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
635 t
= fold_build2_loc (loc
,
637 fold_convert_loc (loc
, itype
, loop
->step
), t
);
638 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
639 fold_convert_loc (loc
, itype
, loop
->n2
));
640 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
641 fold_convert_loc (loc
, itype
, loop
->n1
));
642 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
643 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
644 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
645 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
646 fold_convert_loc (loc
, itype
,
649 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
650 fold_convert_loc (loc
, itype
, loop
->step
));
651 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
652 if (count
!= NULL_TREE
)
653 count
= fold_build2_loc (loc
,
654 MULT_EXPR
, long_long_unsigned_type_node
,
658 if (TREE_CODE (count
) != INTEGER_CST
)
661 else if (count
&& !integer_zerop (count
))
668 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
669 || fd
->have_ordered
))
671 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
672 iter_type
= long_long_unsigned_type_node
;
674 iter_type
= long_integer_type_node
;
676 else if (collapse_iter
&& *collapse_iter
!= NULL
)
677 iter_type
= TREE_TYPE (*collapse_iter
);
678 fd
->iter_type
= iter_type
;
679 if (collapse_iter
&& *collapse_iter
== NULL
)
680 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
681 if (collapse_count
&& *collapse_count
== NULL
)
684 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
686 *collapse_count
= create_tmp_var (iter_type
, ".count");
689 if (fd
->collapse
> 1)
691 fd
->loop
.v
= *collapse_iter
;
692 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
693 fd
->loop
.n2
= *collapse_count
;
694 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
695 fd
->loop
.cond_code
= LT_EXPR
;
699 /* For OpenACC loops, force a chunk size of one, as this avoids the default
700 scheduling where several subsequent iterations are being executed by the
702 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
704 gcc_assert (fd
->chunk_size
== NULL_TREE
);
705 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
710 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
711 is the immediate dominator of PAR_ENTRY_BB, return true if there
712 are no data dependencies that would prevent expanding the parallel
713 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
715 When expanding a combined parallel+workshare region, the call to
716 the child function may need additional arguments in the case of
717 GIMPLE_OMP_FOR regions. In some cases, these arguments are
718 computed out of variables passed in from the parent to the child
719 via 'struct .omp_data_s'. For instance:
721 #pragma omp parallel for schedule (guided, i * 4)
726 # BLOCK 2 (PAR_ENTRY_BB)
728 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
730 # BLOCK 3 (WS_ENTRY_BB)
731 .omp_data_i = &.omp_data_o;
732 D.1667 = .omp_data_i->i;
734 #pragma omp for schedule (guided, D.1598)
736 When we outline the parallel region, the call to the child function
737 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
738 that value is computed *after* the call site. So, in principle we
739 cannot do the transformation.
741 To see whether the code in WS_ENTRY_BB blocks the combined
742 parallel+workshare call, we collect all the variables used in the
743 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
744 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
747 FIXME. If we had the SSA form built at this point, we could merely
748 hoist the code in block 3 into block 2 and be done with it. But at
749 this point we don't have dataflow information and though we could
750 hack something up here, it is really not worth the aggravation. */
753 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
755 struct omp_for_data fd
;
756 gimple ws_stmt
= last_stmt (ws_entry_bb
);
758 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
761 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
763 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
765 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
767 if (fd
.iter_type
!= long_integer_type_node
)
770 /* FIXME. We give up too easily here. If any of these arguments
771 are not constants, they will likely involve variables that have
772 been mapped into fields of .omp_data_s for sharing with the child
773 function. With appropriate data flow, it would be possible to
775 if (!is_gimple_min_invariant (fd
.loop
.n1
)
776 || !is_gimple_min_invariant (fd
.loop
.n2
)
777 || !is_gimple_min_invariant (fd
.loop
.step
)
778 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
785 /* Collect additional arguments needed to emit a combined
786 parallel+workshare call. WS_STMT is the workshare directive being
789 static vec
<tree
, va_gc
> *
790 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
793 location_t loc
= gimple_location (ws_stmt
);
794 vec
<tree
, va_gc
> *ws_args
;
796 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
798 struct omp_for_data fd
;
801 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
805 if (gimple_omp_for_combined_into_p (ws_stmt
))
808 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
809 OMP_CLAUSE__LOOPTEMP_
);
811 n1
= OMP_CLAUSE_DECL (innerc
);
812 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
813 OMP_CLAUSE__LOOPTEMP_
);
815 n2
= OMP_CLAUSE_DECL (innerc
);
818 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
820 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
821 ws_args
->quick_push (t
);
823 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
824 ws_args
->quick_push (t
);
826 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
827 ws_args
->quick_push (t
);
831 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
832 ws_args
->quick_push (t
);
837 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
839 /* Number of sections is equal to the number of edges from the
840 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
841 the exit of the sections region. */
842 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
843 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
844 vec_alloc (ws_args
, 1);
845 ws_args
->quick_push (t
);
853 /* Discover whether REGION is a combined parallel+workshare region. */
856 determine_parallel_type (struct omp_region
*region
)
858 basic_block par_entry_bb
, par_exit_bb
;
859 basic_block ws_entry_bb
, ws_exit_bb
;
861 if (region
== NULL
|| region
->inner
== NULL
862 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
863 || region
->inner
->cont
== NULL
)
866 /* We only support parallel+for and parallel+sections. */
867 if (region
->type
!= GIMPLE_OMP_PARALLEL
868 || (region
->inner
->type
!= GIMPLE_OMP_FOR
869 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
872 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
873 WS_EXIT_BB -> PAR_EXIT_BB. */
874 par_entry_bb
= region
->entry
;
875 par_exit_bb
= region
->exit
;
876 ws_entry_bb
= region
->inner
->entry
;
877 ws_exit_bb
= region
->inner
->exit
;
879 if (single_succ (par_entry_bb
) == ws_entry_bb
880 && single_succ (ws_exit_bb
) == par_exit_bb
881 && workshare_safe_to_combine_p (ws_entry_bb
)
882 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
883 || (last_and_only_stmt (ws_entry_bb
)
884 && last_and_only_stmt (par_exit_bb
))))
886 gimple par_stmt
= last_stmt (par_entry_bb
);
887 gimple ws_stmt
= last_stmt (ws_entry_bb
);
889 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
891 /* If this is a combined parallel loop, we need to determine
892 whether or not to use the combined library calls. There
893 are two cases where we do not apply the transformation:
894 static loops and any kind of ordered loop. In the first
895 case, we already open code the loop so there is no need
896 to do anything else. In the latter case, the combined
897 parallel loop call would still need extra synchronization
898 to implement ordered semantics, so there would not be any
899 gain in using the combined call. */
900 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
901 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
903 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
904 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
906 region
->is_combined_parallel
= false;
907 region
->inner
->is_combined_parallel
= false;
912 region
->is_combined_parallel
= true;
913 region
->inner
->is_combined_parallel
= true;
914 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
919 /* Return true if EXPR is variable sized. */
922 is_variable_sized (const_tree expr
)
924 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
927 /* Return true if DECL is a reference type. */
930 is_reference (tree decl
)
932 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
935 /* Return the type of a decl. If the decl is reference type,
936 return its base type. */
938 get_base_type (tree decl
)
940 tree type
= TREE_TYPE (decl
);
941 if (is_reference (decl
))
942 type
= TREE_TYPE (type
);
946 /* Lookup variables. The "maybe" form
947 allows for the variable form to not have been entered, otherwise we
948 assert that the variable must have been entered. */
951 lookup_decl (tree var
, omp_context
*ctx
)
953 tree
*n
= ctx
->cb
.decl_map
->get (var
);
958 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
960 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
961 return n
? *n
: NULL_TREE
;
965 lookup_field (tree var
, omp_context
*ctx
)
968 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
969 return (tree
) n
->value
;
973 lookup_sfield (tree var
, omp_context
*ctx
)
976 n
= splay_tree_lookup (ctx
->sfield_map
977 ? ctx
->sfield_map
: ctx
->field_map
,
978 (splay_tree_key
) var
);
979 return (tree
) n
->value
;
983 maybe_lookup_field (tree var
, omp_context
*ctx
)
986 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
987 return n
? (tree
) n
->value
: NULL_TREE
;
991 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
993 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
996 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
997 return (tree
) n
->value
;
1001 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1003 splay_tree_node n
= NULL
;
1004 if (ctx
->reduction_map
)
1005 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1006 return n
? (tree
) n
->value
: NULL_TREE
;
1009 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1010 the parallel context if DECL is to be shared. */
1013 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1015 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1018 /* We can only use copy-in/copy-out semantics for shared variables
1019 when we know the value is not accessible from an outer scope. */
1022 gcc_assert (!is_gimple_omp_oacc_specifically (shared_ctx
->stmt
));
1024 /* ??? Trivially accessible from anywhere. But why would we even
1025 be passing an address in this case? Should we simply assert
1026 this to be false, or should we have a cleanup pass that removes
1027 these from the list of mappings? */
1028 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1031 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1032 without analyzing the expression whether or not its location
1033 is accessible to anyone else. In the case of nested parallel
1034 regions it certainly may be. */
1035 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1038 /* Do not use copy-in/copy-out for variables that have their
1040 if (TREE_ADDRESSABLE (decl
))
1043 /* lower_send_shared_vars only uses copy-in, but not copy-out
1045 if (TREE_READONLY (decl
)
1046 || ((TREE_CODE (decl
) == RESULT_DECL
1047 || TREE_CODE (decl
) == PARM_DECL
)
1048 && DECL_BY_REFERENCE (decl
)))
1051 /* Disallow copy-in/out in nested parallel if
1052 decl is shared in outer parallel, otherwise
1053 each thread could store the shared variable
1054 in its own copy-in location, making the
1055 variable no longer really shared. */
1056 if (shared_ctx
->is_nested
)
1060 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1061 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1068 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1069 c
; c
= OMP_CLAUSE_CHAIN (c
))
1070 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1071 && OMP_CLAUSE_DECL (c
) == decl
)
1075 goto maybe_mark_addressable_and_ret
;
1079 /* For tasks avoid using copy-in/out. As tasks can be
1080 deferred or executed in different thread, when GOMP_task
1081 returns, the task hasn't necessarily terminated. */
1082 if (is_task_ctx (shared_ctx
))
1085 maybe_mark_addressable_and_ret
:
1086 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1087 if (is_gimple_reg (outer
))
1089 /* Taking address of OUTER in lower_send_shared_vars
1090 might need regimplification of everything that uses the
1092 if (!task_shared_vars
)
1093 task_shared_vars
= BITMAP_ALLOC (NULL
);
1094 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1095 TREE_ADDRESSABLE (outer
) = 1;
1104 /* Construct a new automatic decl similar to VAR. */
1107 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1109 tree copy
= copy_var_decl (var
, name
, type
);
1111 DECL_CONTEXT (copy
) = current_function_decl
;
1112 DECL_CHAIN (copy
) = ctx
->block_vars
;
1113 ctx
->block_vars
= copy
;
1119 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1121 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1124 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1127 omp_build_component_ref (tree obj
, tree field
)
1129 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1130 if (TREE_THIS_VOLATILE (field
))
1131 TREE_THIS_VOLATILE (ret
) |= 1;
1132 if (TREE_READONLY (field
))
1133 TREE_READONLY (ret
) |= 1;
1137 /* Build tree nodes to access the field for VAR on the receiver side. */
1140 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1142 tree x
, field
= lookup_field (var
, ctx
);
1144 /* If the receiver record type was remapped in the child function,
1145 remap the field into the new record type. */
1146 x
= maybe_lookup_field (field
, ctx
);
1150 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1151 x
= omp_build_component_ref (x
, field
);
1153 x
= build_simple_mem_ref (x
);
1158 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1159 of a parallel, this is a component reference; for workshare constructs
1160 this is some variable. */
1163 build_outer_var_ref (tree var
, omp_context
*ctx
)
1167 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1169 else if (is_variable_sized (var
))
1171 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1172 x
= build_outer_var_ref (x
, ctx
);
1173 x
= build_simple_mem_ref (x
);
1175 else if (is_taskreg_ctx (ctx
))
1177 bool by_ref
= use_pointer_for_field (var
, NULL
);
1178 x
= build_receiver_ref (var
, by_ref
, ctx
);
1180 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1181 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1183 /* #pragma omp simd isn't a worksharing construct, and can reference even
1184 private vars in its linear etc. clauses. */
1186 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1187 x
= lookup_decl (var
, ctx
->outer
);
1188 else if (ctx
->outer
)
1189 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1193 else if (ctx
->outer
)
1194 x
= lookup_decl (var
, ctx
->outer
);
1195 else if (is_reference (var
))
1196 /* This can happen with orphaned constructs. If var is reference, it is
1197 possible it is shared and as such valid. */
1202 if (is_reference (var
))
1203 x
= build_simple_mem_ref (x
);
1208 /* Build tree nodes to access the field for VAR on the sender side. */
1211 build_sender_ref (tree var
, omp_context
*ctx
)
1213 tree field
= lookup_sfield (var
, ctx
);
1214 return omp_build_component_ref (ctx
->sender_decl
, field
);
1217 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1220 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1222 tree field
, type
, sfield
= NULL_TREE
;
1224 gcc_assert ((mask
& 1) == 0
1225 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1226 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1227 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1228 gcc_assert ((mask
& 3) == 3
1229 || !is_gimple_omp_oacc_specifically (ctx
->stmt
));
1231 type
= TREE_TYPE (var
);
1234 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1235 type
= build_pointer_type (build_pointer_type (type
));
1238 type
= build_pointer_type (type
);
1239 else if ((mask
& 3) == 1 && is_reference (var
))
1240 type
= TREE_TYPE (type
);
1242 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1243 FIELD_DECL
, DECL_NAME (var
), type
);
1245 /* Remember what variable this field was created for. This does have a
1246 side effect of making dwarf2out ignore this member, so for helpful
1247 debugging we clear it later in delete_omp_context. */
1248 DECL_ABSTRACT_ORIGIN (field
) = var
;
1249 if (type
== TREE_TYPE (var
))
1251 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1252 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1253 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1256 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1258 if ((mask
& 3) == 3)
1260 insert_field_into_struct (ctx
->record_type
, field
);
1261 if (ctx
->srecord_type
)
1263 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1264 FIELD_DECL
, DECL_NAME (var
), type
);
1265 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1266 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1267 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1268 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1269 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1274 if (ctx
->srecord_type
== NULL_TREE
)
1278 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1279 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1280 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1282 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1283 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1284 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1285 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1286 splay_tree_insert (ctx
->sfield_map
,
1287 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1288 (splay_tree_value
) sfield
);
1292 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1293 : ctx
->srecord_type
, field
);
1297 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1298 (splay_tree_value
) field
);
1299 if ((mask
& 2) && ctx
->sfield_map
)
1300 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1301 (splay_tree_value
) sfield
);
1305 install_var_local (tree var
, omp_context
*ctx
)
1307 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1308 insert_decl_map (&ctx
->cb
, var
, new_var
);
1312 /* Adjust the replacement for DECL in CTX for the new context. This means
1313 copying the DECL_VALUE_EXPR, and fixing up the type. */
1316 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1318 tree new_decl
, size
;
1320 new_decl
= lookup_decl (decl
, ctx
);
1322 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1324 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1325 && DECL_HAS_VALUE_EXPR_P (decl
))
1327 tree ve
= DECL_VALUE_EXPR (decl
);
1328 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1329 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1330 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1333 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1335 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1336 if (size
== error_mark_node
)
1337 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1338 DECL_SIZE (new_decl
) = size
;
1340 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1341 if (size
== error_mark_node
)
1342 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1343 DECL_SIZE_UNIT (new_decl
) = size
;
1347 /* The callback for remap_decl. Search all containing contexts for a
1348 mapping of the variable; this avoids having to duplicate the splay
1349 tree ahead of time. We know a mapping doesn't already exist in the
1350 given context. Create new mappings to implement default semantics. */
1353 omp_copy_decl (tree var
, copy_body_data
*cb
)
1355 omp_context
*ctx
= (omp_context
*) cb
;
1358 if (TREE_CODE (var
) == LABEL_DECL
)
1360 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1361 DECL_CONTEXT (new_var
) = current_function_decl
;
1362 insert_decl_map (&ctx
->cb
, var
, new_var
);
1366 while (!is_taskreg_ctx (ctx
))
1371 new_var
= maybe_lookup_decl (var
, ctx
);
1376 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1379 return error_mark_node
;
1383 /* Debugging dumps for parallel regions. */
1384 void dump_omp_region (FILE *, struct omp_region
*, int);
1385 void debug_omp_region (struct omp_region
*);
1386 void debug_all_omp_regions (void);
1388 /* Dump the parallel region tree rooted at REGION. */
1391 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1393 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1394 gimple_code_name
[region
->type
]);
1397 dump_omp_region (file
, region
->inner
, indent
+ 4);
1401 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1402 region
->cont
->index
);
1406 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1407 region
->exit
->index
);
1409 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1412 dump_omp_region (file
, region
->next
, indent
);
1416 debug_omp_region (struct omp_region
*region
)
1418 dump_omp_region (stderr
, region
, 0);
1422 debug_all_omp_regions (void)
1424 dump_omp_region (stderr
, root_omp_region
, 0);
1428 /* Create a new parallel region starting at STMT inside region PARENT. */
1430 static struct omp_region
*
1431 new_omp_region (basic_block bb
, enum gimple_code type
,
1432 struct omp_region
*parent
)
1434 struct omp_region
*region
= XCNEW (struct omp_region
);
1436 region
->outer
= parent
;
1438 region
->type
= type
;
1442 /* This is a nested region. Add it to the list of inner
1443 regions in PARENT. */
1444 region
->next
= parent
->inner
;
1445 parent
->inner
= region
;
1449 /* This is a toplevel region. Add it to the list of toplevel
1450 regions in ROOT_OMP_REGION. */
1451 region
->next
= root_omp_region
;
1452 root_omp_region
= region
;
1458 /* Release the memory associated with the region tree rooted at REGION. */
1461 free_omp_region_1 (struct omp_region
*region
)
1463 struct omp_region
*i
, *n
;
1465 for (i
= region
->inner
; i
; i
= n
)
1468 free_omp_region_1 (i
);
1474 /* Release the memory for the entire omp region tree. */
1477 free_omp_regions (void)
1479 struct omp_region
*r
, *n
;
1480 for (r
= root_omp_region
; r
; r
= n
)
1483 free_omp_region_1 (r
);
1485 root_omp_region
= NULL
;
1489 /* Create a new context, with OUTER_CTX being the surrounding context. */
1491 static omp_context
*
1492 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1494 omp_context
*ctx
= XCNEW (omp_context
);
1496 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1497 (splay_tree_value
) ctx
);
1502 ctx
->outer
= outer_ctx
;
1503 ctx
->cb
= outer_ctx
->cb
;
1504 ctx
->cb
.block
= NULL
;
1505 ctx
->depth
= outer_ctx
->depth
+ 1;
1506 /* FIXME: handle reductions recursively. */
1507 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1511 ctx
->cb
.src_fn
= current_function_decl
;
1512 ctx
->cb
.dst_fn
= current_function_decl
;
1513 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1514 gcc_checking_assert (ctx
->cb
.src_node
);
1515 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1516 ctx
->cb
.src_cfun
= cfun
;
1517 ctx
->cb
.copy_decl
= omp_copy_decl
;
1518 ctx
->cb
.eh_lp_nr
= 0;
1519 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1521 //TODO ctx->reduction_map = TODO;
1524 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1529 static gimple_seq
maybe_catch_exception (gimple_seq
);
1531 /* Finalize task copyfn. */
1534 finalize_task_copyfn (gimple task_stmt
)
1536 struct function
*child_cfun
;
1538 gimple_seq seq
= NULL
, new_seq
;
1541 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1542 if (child_fn
== NULL_TREE
)
1545 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1546 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1548 push_cfun (child_cfun
);
1549 bind
= gimplify_body (child_fn
, false);
1550 gimple_seq_add_stmt (&seq
, bind
);
1551 new_seq
= maybe_catch_exception (seq
);
1554 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1556 gimple_seq_add_stmt (&seq
, bind
);
1558 gimple_set_body (child_fn
, seq
);
1561 /* Inform the callgraph about the new function. */
1562 cgraph_node::add_new_function (child_fn
, false);
1565 /* Destroy a omp_context data structures. Called through the splay tree
1566 value delete callback. */
1569 delete_omp_context (splay_tree_value value
)
1571 omp_context
*ctx
= (omp_context
*) value
;
1573 delete ctx
->cb
.decl_map
;
1576 splay_tree_delete (ctx
->field_map
);
1577 if (ctx
->sfield_map
)
1578 splay_tree_delete (ctx
->sfield_map
);
1579 if (ctx
->reduction_map
1580 /* Shared over several omp_contexts. */
1581 && (ctx
->outer
== NULL
1582 || ctx
->reduction_map
!= ctx
->outer
->reduction_map
))
1583 splay_tree_delete (ctx
->reduction_map
);
1585 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1586 it produces corrupt debug information. */
1587 if (ctx
->record_type
)
1590 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1591 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1593 if (ctx
->srecord_type
)
1596 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1597 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1600 if (is_task_ctx (ctx
))
1601 finalize_task_copyfn (ctx
->stmt
);
1606 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1610 fixup_child_record_type (omp_context
*ctx
)
1612 tree f
, type
= ctx
->record_type
;
1614 /* ??? It isn't sufficient to just call remap_type here, because
1615 variably_modified_type_p doesn't work the way we expect for
1616 record types. Testing each field for whether it needs remapping
1617 and creating a new record by hand works, however. */
1618 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1619 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1623 tree name
, new_fields
= NULL
;
1625 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1626 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1627 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1628 TYPE_DECL
, name
, type
);
1629 TYPE_NAME (type
) = name
;
1631 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1633 tree new_f
= copy_node (f
);
1634 DECL_CONTEXT (new_f
) = type
;
1635 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1636 DECL_CHAIN (new_f
) = new_fields
;
1637 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1638 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1640 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1644 /* Arrange to be able to look up the receiver field
1645 given the sender field. */
1646 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1647 (splay_tree_value
) new_f
);
1649 TYPE_FIELDS (type
) = nreverse (new_fields
);
1653 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1656 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1657 specified by CLAUSES. */
1660 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1663 bool scan_array_reductions
= false;
1665 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1669 switch (OMP_CLAUSE_CODE (c
))
1671 case OMP_CLAUSE_PRIVATE
:
1672 decl
= OMP_CLAUSE_DECL (c
);
1673 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1675 else if (!is_variable_sized (decl
))
1676 install_var_local (decl
, ctx
);
1679 case OMP_CLAUSE_SHARED
:
1680 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1681 decl
= OMP_CLAUSE_DECL (c
);
1682 /* Ignore shared directives in teams construct. */
1683 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1685 /* Global variables don't need to be copied,
1686 the receiver side will use them directly. */
1687 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1688 if (is_global_var (odecl
))
1690 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1693 gcc_assert (is_taskreg_ctx (ctx
));
1694 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1695 || !is_variable_sized (decl
));
1696 /* Global variables don't need to be copied,
1697 the receiver side will use them directly. */
1698 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1700 by_ref
= use_pointer_for_field (decl
, ctx
);
1701 if (! TREE_READONLY (decl
)
1702 || TREE_ADDRESSABLE (decl
)
1704 || is_reference (decl
))
1706 install_var_field (decl
, by_ref
, 3, ctx
);
1707 install_var_local (decl
, ctx
);
1710 /* We don't need to copy const scalar vars back. */
1711 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1714 case OMP_CLAUSE_LASTPRIVATE
:
1715 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1716 /* Let the corresponding firstprivate clause create
1718 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1722 case OMP_CLAUSE_FIRSTPRIVATE
:
1723 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
1725 sorry ("clause not supported yet");
1729 case OMP_CLAUSE_LINEAR
:
1730 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1732 case OMP_CLAUSE_REDUCTION
:
1733 decl
= OMP_CLAUSE_DECL (c
);
1735 if (is_variable_sized (decl
))
1737 if (is_task_ctx (ctx
))
1738 install_var_field (decl
, false, 1, ctx
);
1741 else if (is_taskreg_ctx (ctx
))
1744 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1745 by_ref
= use_pointer_for_field (decl
, NULL
);
1747 if (is_task_ctx (ctx
)
1748 && (global
|| by_ref
|| is_reference (decl
)))
1750 install_var_field (decl
, false, 1, ctx
);
1752 install_var_field (decl
, by_ref
, 2, ctx
);
1755 install_var_field (decl
, by_ref
, 3, ctx
);
1757 install_var_local (decl
, ctx
);
1759 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
1761 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1763 /* Create a decl for the reduction array. */
1764 tree var
= OMP_CLAUSE_DECL (c
);
1765 tree type
= get_base_type (var
);
1766 tree ptype
= build_pointer_type (type
);
1767 tree array
= create_tmp_var (ptype
,
1768 oacc_get_reduction_array_id (var
));
1769 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1770 install_var_field (array
, true, 3, c
);
1771 install_var_local (array
, c
);
1773 /* Insert it into the current context. */
1774 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1775 oacc_get_reduction_array_id (var
),
1776 (splay_tree_value
) array
);
1777 splay_tree_insert (ctx
->reduction_map
,
1778 (splay_tree_key
) array
,
1779 (splay_tree_value
) array
);
1784 case OMP_CLAUSE__LOOPTEMP_
:
1785 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1786 gcc_assert (is_parallel_ctx (ctx
));
1787 decl
= OMP_CLAUSE_DECL (c
);
1788 install_var_field (decl
, false, 3, ctx
);
1789 install_var_local (decl
, ctx
);
1792 case OMP_CLAUSE_COPYPRIVATE
:
1793 case OMP_CLAUSE_COPYIN
:
1794 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1795 decl
= OMP_CLAUSE_DECL (c
);
1796 by_ref
= use_pointer_for_field (decl
, NULL
);
1797 install_var_field (decl
, by_ref
, 3, ctx
);
1800 case OMP_CLAUSE_DEFAULT
:
1801 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1802 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1805 case OMP_CLAUSE_FINAL
:
1806 case OMP_CLAUSE_NUM_THREADS
:
1807 case OMP_CLAUSE_NUM_TEAMS
:
1808 case OMP_CLAUSE_THREAD_LIMIT
:
1809 case OMP_CLAUSE_DEVICE
:
1810 case OMP_CLAUSE_SCHEDULE
:
1811 case OMP_CLAUSE_DIST_SCHEDULE
:
1812 case OMP_CLAUSE_DEPEND
:
1813 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1814 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1817 case OMP_CLAUSE_NUM_GANGS
:
1818 case OMP_CLAUSE_NUM_WORKERS
:
1819 case OMP_CLAUSE_VECTOR_LENGTH
:
1821 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1825 case OMP_CLAUSE_FROM
:
1826 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1828 case OMP_CLAUSE_MAP
:
1829 switch (OMP_CLAUSE_CODE (c
))
1832 case OMP_CLAUSE_FROM
:
1833 /* The to and from clauses are only ever seen with OpenMP target
1834 update constructs. */
1835 gcc_assert (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1836 && (gimple_omp_target_kind (ctx
->stmt
)
1837 == GF_OMP_TARGET_KIND_UPDATE
));
1839 case OMP_CLAUSE_MAP
:
1840 /* The map clause is never seen with OpenMP target update
1842 gcc_assert (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
1843 || (gimple_omp_target_kind (ctx
->stmt
)
1844 != GF_OMP_TARGET_KIND_UPDATE
));
1850 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1851 decl
= OMP_CLAUSE_DECL (c
);
1852 /* Global variables with "omp declare target" attribute
1853 don't need to be copied, the receiver side will use them
1855 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1857 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1858 && varpool_node::get_create (decl
)->offloadable
)
1860 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1863 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1864 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1866 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1867 target regions that are not offloaded; there is nothing to map for
1869 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1870 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1874 /* In target regions that are not offloaded, libgomp won't pay
1875 attention to OMP_CLAUSE_MAP_FORCE_DEVICEPTR -- but I think we need
1876 to handle it here anyway, in order to create a visible copy of the
1878 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1879 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
1881 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1887 if (DECL_SIZE (decl
)
1888 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1890 tree decl2
= DECL_VALUE_EXPR (decl
);
1891 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1892 decl2
= TREE_OPERAND (decl2
, 0);
1893 gcc_assert (DECL_P (decl2
));
1894 install_var_field (decl2
, true, 3, ctx
);
1895 install_var_local (decl2
, ctx
);
1896 install_var_local (decl
, ctx
);
1900 gcc_assert (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
1901 || (OMP_CLAUSE_MAP_KIND (c
)
1902 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
1903 || TREE_CODE (TREE_TYPE (decl
)) != ARRAY_TYPE
);
1904 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1905 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1906 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1907 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1908 install_var_field (decl
, true, 7, ctx
);
1910 install_var_field (decl
, true, 3, ctx
);
1911 if (is_gimple_omp_offloaded (ctx
->stmt
))
1912 install_var_local (decl
, ctx
);
1917 tree base
= get_base_address (decl
);
1918 tree nc
= OMP_CLAUSE_CHAIN (c
);
1921 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1922 && OMP_CLAUSE_DECL (nc
) == base
1923 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1924 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1926 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1927 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1933 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1934 decl
= OMP_CLAUSE_DECL (c
);
1936 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1937 (splay_tree_key
) decl
));
1939 = build_decl (OMP_CLAUSE_LOCATION (c
),
1940 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1941 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1942 insert_field_into_struct (ctx
->record_type
, field
);
1943 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1944 (splay_tree_value
) field
);
1949 case OMP_CLAUSE_NOWAIT
:
1950 case OMP_CLAUSE_ORDERED
:
1951 case OMP_CLAUSE_UNTIED
:
1952 case OMP_CLAUSE_MERGEABLE
:
1953 case OMP_CLAUSE_PROC_BIND
:
1954 case OMP_CLAUSE_SAFELEN
:
1955 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
1957 sorry ("clause not supported yet");
1961 case OMP_CLAUSE_COLLAPSE
:
1962 case OMP_CLAUSE_ASYNC
:
1963 case OMP_CLAUSE_WAIT
:
1966 case OMP_CLAUSE_ALIGNED
:
1967 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1968 decl
= OMP_CLAUSE_DECL (c
);
1969 if (is_global_var (decl
)
1970 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1971 install_var_local (decl
, ctx
);
1974 case OMP_CLAUSE_DEVICE_RESIDENT
:
1975 case OMP_CLAUSE_USE_DEVICE
:
1976 case OMP_CLAUSE_GANG
:
1977 case OMP_CLAUSE__CACHE_
:
1978 case OMP_CLAUSE_INDEPENDENT
:
1979 case OMP_CLAUSE_WORKER
:
1980 case OMP_CLAUSE_VECTOR
:
1981 sorry ("Clause not supported yet");
1989 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1991 switch (OMP_CLAUSE_CODE (c
))
1993 case OMP_CLAUSE_LASTPRIVATE
:
1994 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
1995 /* Let the corresponding firstprivate clause create
1997 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1998 scan_array_reductions
= true;
1999 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2003 case OMP_CLAUSE_FIRSTPRIVATE
:
2004 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
2006 sorry ("clause not supported yet");
2010 case OMP_CLAUSE_LINEAR
:
2011 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2013 case OMP_CLAUSE_REDUCTION
:
2014 case OMP_CLAUSE_PRIVATE
:
2015 decl
= OMP_CLAUSE_DECL (c
);
2016 if (is_variable_sized (decl
))
2017 install_var_local (decl
, ctx
);
2018 fixup_remapped_decl (decl
, ctx
,
2019 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2020 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2021 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2022 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2023 scan_array_reductions
= true;
2024 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2025 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2026 scan_array_reductions
= true;
2029 case OMP_CLAUSE_SHARED
:
2030 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2031 /* Ignore shared directives in teams construct. */
2032 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2034 decl
= OMP_CLAUSE_DECL (c
);
2035 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2036 fixup_remapped_decl (decl
, ctx
, false);
2039 case OMP_CLAUSE_MAP
:
2040 /* The map clause is never seen with OpenMP target update
2042 gcc_assert (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2043 || (gimple_omp_target_kind (ctx
->stmt
)
2044 != GF_OMP_TARGET_KIND_UPDATE
));
2045 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2047 decl
= OMP_CLAUSE_DECL (c
);
2049 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2050 && varpool_node::get_create (decl
)->offloadable
)
2052 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2057 gcc_assert ((OMP_CLAUSE_MAP_KIND (c
)
2058 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
2059 || TREE_CODE (TREE_TYPE (decl
)) != ARRAY_TYPE
);
2060 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
2061 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2062 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2064 tree new_decl
= lookup_decl (decl
, ctx
);
2065 TREE_TYPE (new_decl
)
2066 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2068 else if (DECL_SIZE (decl
)
2069 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2071 gcc_assert (OMP_CLAUSE_MAP_KIND (c
)
2072 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
);
2074 tree decl2
= DECL_VALUE_EXPR (decl
);
2075 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2076 decl2
= TREE_OPERAND (decl2
, 0);
2077 gcc_assert (DECL_P (decl2
));
2078 fixup_remapped_decl (decl2
, ctx
, false);
2079 fixup_remapped_decl (decl
, ctx
, true);
2082 fixup_remapped_decl (decl
, ctx
, false);
2086 case OMP_CLAUSE_COPYPRIVATE
:
2087 case OMP_CLAUSE_COPYIN
:
2088 case OMP_CLAUSE_DEFAULT
:
2089 case OMP_CLAUSE_NUM_THREADS
:
2090 case OMP_CLAUSE_NUM_TEAMS
:
2091 case OMP_CLAUSE_THREAD_LIMIT
:
2092 case OMP_CLAUSE_DEVICE
:
2093 case OMP_CLAUSE_SCHEDULE
:
2094 case OMP_CLAUSE_DIST_SCHEDULE
:
2095 case OMP_CLAUSE_NOWAIT
:
2096 case OMP_CLAUSE_ORDERED
:
2097 case OMP_CLAUSE_UNTIED
:
2098 case OMP_CLAUSE_FINAL
:
2099 case OMP_CLAUSE_MERGEABLE
:
2100 case OMP_CLAUSE_PROC_BIND
:
2101 case OMP_CLAUSE_SAFELEN
:
2102 case OMP_CLAUSE_ALIGNED
:
2103 case OMP_CLAUSE_DEPEND
:
2104 case OMP_CLAUSE__LOOPTEMP_
:
2106 case OMP_CLAUSE_FROM
:
2107 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2108 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2110 case OMP_CLAUSE_ASYNC
:
2111 case OMP_CLAUSE_COLLAPSE
:
2113 case OMP_CLAUSE_NUM_GANGS
:
2114 case OMP_CLAUSE_NUM_WORKERS
:
2115 case OMP_CLAUSE_VECTOR_LENGTH
:
2116 case OMP_CLAUSE_WAIT
:
2119 case OMP_CLAUSE_DEVICE_RESIDENT
:
2120 case OMP_CLAUSE_USE_DEVICE
:
2121 case OMP_CLAUSE_GANG
:
2122 case OMP_CLAUSE__CACHE_
:
2123 case OMP_CLAUSE_INDEPENDENT
:
2124 case OMP_CLAUSE_WORKER
:
2125 case OMP_CLAUSE_VECTOR
:
2126 sorry ("Clause not supported yet");
2134 if (scan_array_reductions
)
2136 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
2137 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2138 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2139 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2141 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2142 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2144 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2145 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2146 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2147 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2148 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2149 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2153 /* Create a new name for omp child function. Returns an identifier. If
2154 IS_CILK_FOR is true then the suffix for the child function is
2158 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2161 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2162 return clone_function_name (current_function_decl
,
2163 task_copy
? "_omp_cpyfn" : "_omp_fn");
2166 /* Returns the type of the induction variable for the child function for
2167 _Cilk_for and the types for _high and _low variables based on TYPE. */
2170 cilk_for_check_loop_diff_type (tree type
)
2172 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2174 if (TYPE_UNSIGNED (type
))
2175 return uint32_type_node
;
2177 return integer_type_node
;
2181 if (TYPE_UNSIGNED (type
))
2182 return uint64_type_node
;
2184 return long_long_integer_type_node
;
2188 /* Build a decl for the omp child function. It'll not contain a body
2189 yet, just the bare decl. */
2192 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2194 tree decl
, type
, name
, t
;
2197 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2198 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2199 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2200 tree cilk_var_type
= NULL_TREE
;
2202 name
= create_omp_child_function_name (task_copy
,
2203 cilk_for_count
!= NULL_TREE
);
2205 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2206 ptr_type_node
, NULL_TREE
);
2207 else if (cilk_for_count
)
2209 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2210 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2211 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2212 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2215 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2217 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2219 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
)
2222 ctx
->cb
.dst_fn
= decl
;
2224 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2226 TREE_STATIC (decl
) = 1;
2227 TREE_USED (decl
) = 1;
2228 DECL_ARTIFICIAL (decl
) = 1;
2229 DECL_IGNORED_P (decl
) = 0;
2230 TREE_PUBLIC (decl
) = 0;
2231 DECL_UNINLINABLE (decl
) = 1;
2232 DECL_EXTERNAL (decl
) = 0;
2233 DECL_CONTEXT (decl
) = NULL_TREE
;
2234 DECL_INITIAL (decl
) = make_node (BLOCK
);
2235 if (cgraph_node::get (current_function_decl
)->offloadable
)
2236 cgraph_node::get_create (decl
)->offloadable
= 1;
2240 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2241 if (is_gimple_omp_offloaded (octx
->stmt
))
2243 cgraph_node::get_create (decl
)->offloadable
= 1;
2244 g
->have_offload
= true;
2249 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2250 RESULT_DECL
, NULL_TREE
, void_type_node
);
2251 DECL_ARTIFICIAL (t
) = 1;
2252 DECL_IGNORED_P (t
) = 1;
2253 DECL_CONTEXT (t
) = decl
;
2254 DECL_RESULT (decl
) = t
;
2256 /* _Cilk_for's child function requires two extra parameters called
2257 __low and __high that are set the by Cilk runtime when it calls this
2261 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2262 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2263 DECL_ARTIFICIAL (t
) = 1;
2264 DECL_NAMELESS (t
) = 1;
2265 DECL_ARG_TYPE (t
) = ptr_type_node
;
2266 DECL_CONTEXT (t
) = current_function_decl
;
2268 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2269 DECL_ARGUMENTS (decl
) = t
;
2271 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2272 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2273 DECL_ARTIFICIAL (t
) = 1;
2274 DECL_NAMELESS (t
) = 1;
2275 DECL_ARG_TYPE (t
) = ptr_type_node
;
2276 DECL_CONTEXT (t
) = current_function_decl
;
2278 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2279 DECL_ARGUMENTS (decl
) = t
;
2282 tree data_name
= get_identifier (".omp_data_i");
2283 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2285 DECL_ARTIFICIAL (t
) = 1;
2286 DECL_NAMELESS (t
) = 1;
2287 DECL_ARG_TYPE (t
) = ptr_type_node
;
2288 DECL_CONTEXT (t
) = current_function_decl
;
2291 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2292 DECL_ARGUMENTS (decl
) = t
;
2294 ctx
->receiver_decl
= t
;
2297 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2298 PARM_DECL
, get_identifier (".omp_data_o"),
2300 DECL_ARTIFICIAL (t
) = 1;
2301 DECL_NAMELESS (t
) = 1;
2302 DECL_ARG_TYPE (t
) = ptr_type_node
;
2303 DECL_CONTEXT (t
) = current_function_decl
;
2305 TREE_ADDRESSABLE (t
) = 1;
2306 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2307 DECL_ARGUMENTS (decl
) = t
;
2310 /* Allocate memory for the function structure. The call to
2311 allocate_struct_function clobbers CFUN, so we need to restore
2313 push_struct_function (decl
);
2314 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2318 /* Callback for walk_gimple_seq. Check if combined parallel
2319 contains gimple_omp_for_combined_into_p OMP_FOR. */
2322 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2323 bool *handled_ops_p
,
2324 struct walk_stmt_info
*wi
)
2326 gimple stmt
= gsi_stmt (*gsi_p
);
2328 *handled_ops_p
= true;
2329 switch (gimple_code (stmt
))
2333 case GIMPLE_OMP_FOR
:
2334 if (gimple_omp_for_combined_into_p (stmt
)
2335 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2338 return integer_zero_node
;
2347 /* Scan an OpenMP parallel directive. */
2350 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2354 gimple stmt
= gsi_stmt (*gsi
);
2356 /* Ignore parallel directives with empty bodies, unless there
2357 are copyin clauses. */
2359 && empty_body_p (gimple_omp_body (stmt
))
2360 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2361 OMP_CLAUSE_COPYIN
) == NULL
)
2363 gsi_replace (gsi
, gimple_build_nop (), false);
2367 if (gimple_omp_parallel_combined_p (stmt
))
2370 struct walk_stmt_info wi
;
2372 memset (&wi
, 0, sizeof (wi
));
2374 walk_gimple_seq (gimple_omp_body (stmt
),
2375 find_combined_for
, NULL
, &wi
);
2376 for_stmt
= (gimple
) wi
.info
;
2379 struct omp_for_data fd
;
2380 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2381 /* We need two temporaries with fd.loop.v type (istart/iend)
2382 and then (fd.collapse - 1) temporaries with the same
2383 type for count2 ... countN-1 vars if not constant. */
2384 size_t count
= 2, i
;
2385 tree type
= fd
.iter_type
;
2387 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2388 count
+= fd
.collapse
- 1;
2389 for (i
= 0; i
< count
; i
++)
2391 tree temp
= create_tmp_var (type
, NULL
);
2392 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2393 OMP_CLAUSE__LOOPTEMP_
);
2394 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2395 OMP_CLAUSE_DECL (c
) = temp
;
2396 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2397 gimple_omp_parallel_set_clauses (stmt
, c
);
2402 ctx
= new_omp_context (stmt
, outer_ctx
);
2403 taskreg_contexts
.safe_push (ctx
);
2404 if (taskreg_nesting_level
> 1)
2405 ctx
->is_nested
= true;
2406 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2407 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2408 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2409 name
= create_tmp_var_name (".omp_data_s");
2410 name
= build_decl (gimple_location (stmt
),
2411 TYPE_DECL
, name
, ctx
->record_type
);
2412 DECL_ARTIFICIAL (name
) = 1;
2413 DECL_NAMELESS (name
) = 1;
2414 TYPE_NAME (ctx
->record_type
) = name
;
2415 create_omp_child_function (ctx
, false);
2416 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2418 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2419 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2421 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2422 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2425 /* Scan an OpenMP task directive. */
2428 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2432 gimple stmt
= gsi_stmt (*gsi
);
2434 /* Ignore task directives with empty bodies. */
2436 && empty_body_p (gimple_omp_body (stmt
)))
2438 gsi_replace (gsi
, gimple_build_nop (), false);
2442 ctx
= new_omp_context (stmt
, outer_ctx
);
2443 taskreg_contexts
.safe_push (ctx
);
2444 if (taskreg_nesting_level
> 1)
2445 ctx
->is_nested
= true;
2446 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2447 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2448 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2449 name
= create_tmp_var_name (".omp_data_s");
2450 name
= build_decl (gimple_location (stmt
),
2451 TYPE_DECL
, name
, ctx
->record_type
);
2452 DECL_ARTIFICIAL (name
) = 1;
2453 DECL_NAMELESS (name
) = 1;
2454 TYPE_NAME (ctx
->record_type
) = name
;
2455 create_omp_child_function (ctx
, false);
2456 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2458 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2460 if (ctx
->srecord_type
)
2462 name
= create_tmp_var_name (".omp_data_a");
2463 name
= build_decl (gimple_location (stmt
),
2464 TYPE_DECL
, name
, ctx
->srecord_type
);
2465 DECL_ARTIFICIAL (name
) = 1;
2466 DECL_NAMELESS (name
) = 1;
2467 TYPE_NAME (ctx
->srecord_type
) = name
;
2468 create_omp_child_function (ctx
, true);
2471 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2473 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2475 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2476 t
= build_int_cst (long_integer_type_node
, 0);
2477 gimple_omp_task_set_arg_size (stmt
, t
);
2478 t
= build_int_cst (long_integer_type_node
, 1);
2479 gimple_omp_task_set_arg_align (stmt
, t
);
2484 /* If any decls have been made addressable during scan_omp,
2485 adjust their fields if needed, and layout record types
2486 of parallel/task constructs. */
2489 finish_taskreg_scan (omp_context
*ctx
)
2491 if (ctx
->record_type
== NULL_TREE
)
2494 /* If any task_shared_vars were needed, verify all
2495 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2496 statements if use_pointer_for_field hasn't changed
2497 because of that. If it did, update field types now. */
2498 if (task_shared_vars
)
2502 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2503 c
; c
= OMP_CLAUSE_CHAIN (c
))
2504 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2506 tree decl
= OMP_CLAUSE_DECL (c
);
2508 /* Global variables don't need to be copied,
2509 the receiver side will use them directly. */
2510 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2512 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2513 || !use_pointer_for_field (decl
, ctx
))
2515 tree field
= lookup_field (decl
, ctx
);
2516 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2517 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2519 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2520 TREE_THIS_VOLATILE (field
) = 0;
2521 DECL_USER_ALIGN (field
) = 0;
2522 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2523 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2524 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2525 if (ctx
->srecord_type
)
2527 tree sfield
= lookup_sfield (decl
, ctx
);
2528 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2529 TREE_THIS_VOLATILE (sfield
) = 0;
2530 DECL_USER_ALIGN (sfield
) = 0;
2531 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2532 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2533 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2538 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2540 layout_type (ctx
->record_type
);
2541 fixup_child_record_type (ctx
);
2545 location_t loc
= gimple_location (ctx
->stmt
);
2546 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2547 /* Move VLA fields to the end. */
2548 p
= &TYPE_FIELDS (ctx
->record_type
);
2550 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2551 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2554 *p
= TREE_CHAIN (*p
);
2555 TREE_CHAIN (*q
) = NULL_TREE
;
2556 q
= &TREE_CHAIN (*q
);
2559 p
= &DECL_CHAIN (*p
);
2561 layout_type (ctx
->record_type
);
2562 fixup_child_record_type (ctx
);
2563 if (ctx
->srecord_type
)
2564 layout_type (ctx
->srecord_type
);
2565 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2566 TYPE_SIZE_UNIT (ctx
->record_type
));
2567 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2568 t
= build_int_cst (long_integer_type_node
,
2569 TYPE_ALIGN_UNIT (ctx
->record_type
));
2570 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2575 /* Scan a GIMPLE_OMP_FOR. */
2578 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2583 ctx
= new_omp_context (stmt
, outer_ctx
);
2585 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2587 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2588 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2590 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2591 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2592 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2593 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2595 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2598 /* Scan an OpenMP sections directive. */
2601 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2605 ctx
= new_omp_context (stmt
, outer_ctx
);
2606 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2607 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2610 /* Scan an OpenMP single directive. */
2613 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2618 ctx
= new_omp_context (stmt
, outer_ctx
);
2619 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2620 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2621 name
= create_tmp_var_name (".omp_copy_s");
2622 name
= build_decl (gimple_location (stmt
),
2623 TYPE_DECL
, name
, ctx
->record_type
);
2624 TYPE_NAME (ctx
->record_type
) = name
;
2626 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2627 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2629 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2630 ctx
->record_type
= NULL
;
2632 layout_type (ctx
->record_type
);
2635 /* Scan a GIMPLE_OMP_TARGET. */
2638 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2642 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2644 ctx
= new_omp_context (stmt
, outer_ctx
);
2645 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2646 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2647 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2648 name
= create_tmp_var_name (".omp_data_t");
2649 name
= build_decl (gimple_location (stmt
),
2650 TYPE_DECL
, name
, ctx
->record_type
);
2651 DECL_ARTIFICIAL (name
) = 1;
2652 DECL_NAMELESS (name
) = 1;
2653 TYPE_NAME (ctx
->record_type
) = name
;
2656 if (is_gimple_omp_oacc_specifically (stmt
))
2657 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2660 create_omp_child_function (ctx
, false);
2661 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2664 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2665 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2667 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2668 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2671 TYPE_FIELDS (ctx
->record_type
)
2672 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2673 #ifdef ENABLE_CHECKING
2675 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2676 for (field
= TYPE_FIELDS (ctx
->record_type
);
2678 field
= DECL_CHAIN (field
))
2679 gcc_assert (DECL_ALIGN (field
) == align
);
2681 layout_type (ctx
->record_type
);
2683 fixup_child_record_type (ctx
);
2687 /* Scan an OpenMP teams directive. */
2690 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2692 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2693 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2694 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2697 /* Check nesting restrictions. */
2699 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2701 /* TODO: Some OpenACC/OpenMP nesting should be allowed. */
2703 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2704 inside an OpenACC CTX. */
2705 if (!(is_gimple_omp (stmt
)
2706 && is_gimple_omp_oacc_specifically (stmt
)))
2708 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2709 if (is_gimple_omp (ctx_
->stmt
)
2710 && is_gimple_omp_oacc_specifically (ctx_
->stmt
))
2712 error_at (gimple_location (stmt
),
2713 "non-OpenACC construct inside of OpenACC region");
2720 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2721 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2723 error_at (gimple_location (stmt
),
2724 "OpenMP constructs may not be nested inside simd region");
2727 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2729 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2730 || (gimple_omp_for_kind (stmt
)
2731 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2732 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2734 error_at (gimple_location (stmt
),
2735 "only distribute or parallel constructs are allowed to "
2736 "be closely nested inside teams construct");
2741 switch (gimple_code (stmt
))
2743 case GIMPLE_OMP_FOR
:
2744 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2746 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2748 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2750 error_at (gimple_location (stmt
),
2751 "distribute construct must be closely nested inside "
2759 if (is_gimple_call (stmt
)
2760 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2761 == BUILT_IN_GOMP_CANCEL
2762 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2763 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2765 const char *bad
= NULL
;
2766 const char *kind
= NULL
;
2769 error_at (gimple_location (stmt
), "orphaned %qs construct",
2770 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2771 == BUILT_IN_GOMP_CANCEL
2772 ? "#pragma omp cancel"
2773 : "#pragma omp cancellation point");
2776 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2777 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2781 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2782 bad
= "#pragma omp parallel";
2783 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2784 == BUILT_IN_GOMP_CANCEL
2785 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2786 ctx
->cancellable
= true;
2790 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2791 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2792 bad
= "#pragma omp for";
2793 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2794 == BUILT_IN_GOMP_CANCEL
2795 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2797 ctx
->cancellable
= true;
2798 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2800 warning_at (gimple_location (stmt
), 0,
2801 "%<#pragma omp cancel for%> inside "
2802 "%<nowait%> for construct");
2803 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2804 OMP_CLAUSE_ORDERED
))
2805 warning_at (gimple_location (stmt
), 0,
2806 "%<#pragma omp cancel for%> inside "
2807 "%<ordered%> for construct");
2812 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2813 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2814 bad
= "#pragma omp sections";
2815 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2816 == BUILT_IN_GOMP_CANCEL
2817 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2819 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2821 ctx
->cancellable
= true;
2822 if (find_omp_clause (gimple_omp_sections_clauses
2825 warning_at (gimple_location (stmt
), 0,
2826 "%<#pragma omp cancel sections%> inside "
2827 "%<nowait%> sections construct");
2831 gcc_assert (ctx
->outer
2832 && gimple_code (ctx
->outer
->stmt
)
2833 == GIMPLE_OMP_SECTIONS
);
2834 ctx
->outer
->cancellable
= true;
2835 if (find_omp_clause (gimple_omp_sections_clauses
2838 warning_at (gimple_location (stmt
), 0,
2839 "%<#pragma omp cancel sections%> inside "
2840 "%<nowait%> sections construct");
2846 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2847 bad
= "#pragma omp task";
2849 ctx
->cancellable
= true;
2853 error_at (gimple_location (stmt
), "invalid arguments");
2858 error_at (gimple_location (stmt
),
2859 "%<%s %s%> construct not closely nested inside of %qs",
2860 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2861 == BUILT_IN_GOMP_CANCEL
2862 ? "#pragma omp cancel"
2863 : "#pragma omp cancellation point", kind
, bad
);
2868 case GIMPLE_OMP_SECTIONS
:
2869 case GIMPLE_OMP_SINGLE
:
2870 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2871 switch (gimple_code (ctx
->stmt
))
2873 case GIMPLE_OMP_FOR
:
2874 case GIMPLE_OMP_SECTIONS
:
2875 case GIMPLE_OMP_SINGLE
:
2876 case GIMPLE_OMP_ORDERED
:
2877 case GIMPLE_OMP_MASTER
:
2878 case GIMPLE_OMP_TASK
:
2879 case GIMPLE_OMP_CRITICAL
:
2880 if (is_gimple_call (stmt
))
2882 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2883 != BUILT_IN_GOMP_BARRIER
)
2885 error_at (gimple_location (stmt
),
2886 "barrier region may not be closely nested inside "
2887 "of work-sharing, critical, ordered, master or "
2888 "explicit task region");
2891 error_at (gimple_location (stmt
),
2892 "work-sharing region may not be closely nested inside "
2893 "of work-sharing, critical, ordered, master or explicit "
2896 case GIMPLE_OMP_PARALLEL
:
2902 case GIMPLE_OMP_MASTER
:
2903 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2904 switch (gimple_code (ctx
->stmt
))
2906 case GIMPLE_OMP_FOR
:
2907 case GIMPLE_OMP_SECTIONS
:
2908 case GIMPLE_OMP_SINGLE
:
2909 case GIMPLE_OMP_TASK
:
2910 error_at (gimple_location (stmt
),
2911 "master region may not be closely nested inside "
2912 "of work-sharing or explicit task region");
2914 case GIMPLE_OMP_PARALLEL
:
2920 case GIMPLE_OMP_ORDERED
:
2921 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2922 switch (gimple_code (ctx
->stmt
))
2924 case GIMPLE_OMP_CRITICAL
:
2925 case GIMPLE_OMP_TASK
:
2926 error_at (gimple_location (stmt
),
2927 "ordered region may not be closely nested inside "
2928 "of critical or explicit task region");
2930 case GIMPLE_OMP_FOR
:
2931 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2932 OMP_CLAUSE_ORDERED
) == NULL
)
2934 error_at (gimple_location (stmt
),
2935 "ordered region must be closely nested inside "
2936 "a loop region with an ordered clause");
2940 case GIMPLE_OMP_PARALLEL
:
2941 error_at (gimple_location (stmt
),
2942 "ordered region must be closely nested inside "
2943 "a loop region with an ordered clause");
2949 case GIMPLE_OMP_CRITICAL
:
2950 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2951 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2952 && (gimple_omp_critical_name (stmt
)
2953 == gimple_omp_critical_name (ctx
->stmt
)))
2955 error_at (gimple_location (stmt
),
2956 "critical region may not be nested inside a critical "
2957 "region with the same name");
2961 case GIMPLE_OMP_TEAMS
:
2963 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2964 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2966 error_at (gimple_location (stmt
),
2967 "teams construct not closely nested inside of target "
2972 case GIMPLE_OMP_TARGET
:
2973 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2975 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2977 if (is_gimple_omp (stmt
)
2978 && is_gimple_omp_oacc_specifically (stmt
)
2979 && is_gimple_omp (ctx
->stmt
))
2981 error_at (gimple_location (stmt
),
2982 "OpenACC construct inside of non-OpenACC region");
2988 const char *stmt_name
, *ctx_stmt_name
;
2989 switch (gimple_omp_target_kind (stmt
))
2991 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
2992 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
2993 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
2994 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
2995 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
2996 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
2997 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
2998 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
2999 default: gcc_unreachable ();
3001 switch (gimple_omp_target_kind (ctx
->stmt
))
3003 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3004 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3005 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3006 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3007 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3008 default: gcc_unreachable ();
3011 /* OpenACC/OpenMP mismatch? */
3012 if (is_gimple_omp_oacc_specifically (stmt
)
3013 != is_gimple_omp_oacc_specifically (ctx
->stmt
))
3015 error_at (gimple_location (stmt
),
3016 "%s %s construct inside of %s %s region",
3017 (is_gimple_omp_oacc_specifically (stmt
)
3018 ? "OpenACC" : "OpenMP"), stmt_name
,
3019 (is_gimple_omp_oacc_specifically (ctx
->stmt
)
3020 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3023 if (is_gimple_omp_offloaded (ctx
->stmt
))
3025 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3026 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
3028 error_at (gimple_location (stmt
),
3029 "%s construct inside of %s region",
3030 stmt_name
, ctx_stmt_name
);
3035 gcc_assert (!is_gimple_omp_oacc_specifically (stmt
));
3036 warning_at (gimple_location (stmt
), 0,
3037 "%s construct inside of %s region",
3038 stmt_name
, ctx_stmt_name
);
3050 /* Helper function scan_omp.
3052 Callback for walk_tree or operators in walk_gimple_stmt used to
3053 scan for OpenMP directives in TP. */
3056 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3058 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3059 omp_context
*ctx
= (omp_context
*) wi
->info
;
3062 switch (TREE_CODE (t
))
3069 *tp
= remap_decl (t
, &ctx
->cb
);
3073 if (ctx
&& TYPE_P (t
))
3074 *tp
= remap_type (t
, &ctx
->cb
);
3075 else if (!DECL_P (t
))
3080 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3081 if (tem
!= TREE_TYPE (t
))
3083 if (TREE_CODE (t
) == INTEGER_CST
)
3084 *tp
= wide_int_to_tree (tem
, t
);
3086 TREE_TYPE (t
) = tem
;
3096 /* Return true if FNDECL is a setjmp or a longjmp. */
3099 setjmp_or_longjmp_p (const_tree fndecl
)
3101 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3102 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3103 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3106 tree declname
= DECL_NAME (fndecl
);
3109 const char *name
= IDENTIFIER_POINTER (declname
);
3110 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3114 /* Helper function for scan_omp.
3116 Callback for walk_gimple_stmt used to scan for OpenMP directives in
3117 the current statement in GSI. */
3120 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3121 struct walk_stmt_info
*wi
)
3123 gimple stmt
= gsi_stmt (*gsi
);
3124 omp_context
*ctx
= (omp_context
*) wi
->info
;
3126 if (gimple_has_location (stmt
))
3127 input_location
= gimple_location (stmt
);
3129 /* Check the OpenMP nesting restrictions. */
3130 bool remove
= false;
3131 if (is_gimple_omp (stmt
))
3132 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3133 else if (is_gimple_call (stmt
))
3135 tree fndecl
= gimple_call_fndecl (stmt
);
3138 if (setjmp_or_longjmp_p (fndecl
)
3140 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3141 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3144 error_at (gimple_location (stmt
),
3145 "setjmp/longjmp inside simd construct");
3147 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3148 switch (DECL_FUNCTION_CODE (fndecl
))
3150 case BUILT_IN_GOMP_BARRIER
:
3151 case BUILT_IN_GOMP_CANCEL
:
3152 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3153 case BUILT_IN_GOMP_TASKYIELD
:
3154 case BUILT_IN_GOMP_TASKWAIT
:
3155 case BUILT_IN_GOMP_TASKGROUP_START
:
3156 case BUILT_IN_GOMP_TASKGROUP_END
:
3157 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3166 stmt
= gimple_build_nop ();
3167 gsi_replace (gsi
, stmt
, false);
3170 *handled_ops_p
= true;
3172 switch (gimple_code (stmt
))
3174 case GIMPLE_OMP_PARALLEL
:
3175 taskreg_nesting_level
++;
3176 scan_omp_parallel (gsi
, ctx
);
3177 taskreg_nesting_level
--;
3180 case GIMPLE_OMP_TASK
:
3181 taskreg_nesting_level
++;
3182 scan_omp_task (gsi
, ctx
);
3183 taskreg_nesting_level
--;
3186 case GIMPLE_OMP_FOR
:
3187 scan_omp_for (stmt
, ctx
);
3190 case GIMPLE_OMP_SECTIONS
:
3191 scan_omp_sections (stmt
, ctx
);
3194 case GIMPLE_OMP_SINGLE
:
3195 scan_omp_single (stmt
, ctx
);
3198 case GIMPLE_OMP_SECTION
:
3199 case GIMPLE_OMP_MASTER
:
3200 case GIMPLE_OMP_TASKGROUP
:
3201 case GIMPLE_OMP_ORDERED
:
3202 case GIMPLE_OMP_CRITICAL
:
3203 ctx
= new_omp_context (stmt
, ctx
);
3204 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3207 case GIMPLE_OMP_TARGET
:
3208 scan_omp_target (stmt
, ctx
);
3211 case GIMPLE_OMP_TEAMS
:
3212 scan_omp_teams (stmt
, ctx
);
3219 *handled_ops_p
= false;
3221 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
3222 insert_decl_map (&ctx
->cb
, var
, var
);
3226 *handled_ops_p
= false;
3234 /* Scan all the statements starting at the current statement. CTX
3235 contains context information about the OpenMP directives and
3236 clauses found during the scan. */
3239 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3241 location_t saved_location
;
3242 struct walk_stmt_info wi
;
3244 memset (&wi
, 0, sizeof (wi
));
3246 wi
.want_locations
= true;
3248 saved_location
= input_location
;
3249 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3250 input_location
= saved_location
;
3253 /* Re-gimplification and code generation routines. */
3255 /* Build a call to GOMP_barrier. */
3258 build_omp_barrier (tree lhs
)
3260 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3261 : BUILT_IN_GOMP_BARRIER
);
3262 gimple g
= gimple_build_call (fndecl
, 0);
3264 gimple_call_set_lhs (g
, lhs
);
3268 /* If a context was created for STMT when it was scanned, return it. */
3270 static omp_context
*
3271 maybe_lookup_ctx (gimple stmt
)
3274 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3275 return n
? (omp_context
*) n
->value
: NULL
;
3279 /* Find the mapping for DECL in CTX or the immediately enclosing
3280 context that has a mapping for DECL.
3282 If CTX is a nested parallel directive, we may have to use the decl
3283 mappings created in CTX's parent context. Suppose that we have the
3284 following parallel nesting (variable UIDs showed for clarity):
3287 #omp parallel shared(iD.1562) -> outer parallel
3288 iD.1562 = iD.1562 + 1;
3290 #omp parallel shared (iD.1562) -> inner parallel
3291 iD.1562 = iD.1562 - 1;
3293 Each parallel structure will create a distinct .omp_data_s structure
3294 for copying iD.1562 in/out of the directive:
3296 outer parallel .omp_data_s.1.i -> iD.1562
3297 inner parallel .omp_data_s.2.i -> iD.1562
3299 A shared variable mapping will produce a copy-out operation before
3300 the parallel directive and a copy-in operation after it. So, in
3301 this case we would have:
3304 .omp_data_o.1.i = iD.1562;
3305 #omp parallel shared(iD.1562) -> outer parallel
3306 .omp_data_i.1 = &.omp_data_o.1
3307 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3309 .omp_data_o.2.i = iD.1562; -> **
3310 #omp parallel shared(iD.1562) -> inner parallel
3311 .omp_data_i.2 = &.omp_data_o.2
3312 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3315 ** This is a problem. The symbol iD.1562 cannot be referenced
3316 inside the body of the outer parallel region. But since we are
3317 emitting this copy operation while expanding the inner parallel
3318 directive, we need to access the CTX structure of the outer
3319 parallel directive to get the correct mapping:
3321 .omp_data_o.2.i = .omp_data_i.1->i
3323 Since there may be other workshare or parallel directives enclosing
3324 the parallel directive, it may be necessary to walk up the context
3325 parent chain. This is not a problem in general because nested
3326 parallelism happens only rarely. */
3329 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3334 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3335 t
= maybe_lookup_decl (decl
, up
);
3337 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3339 return t
? t
: decl
;
3343 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3344 in outer contexts. */
3347 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3352 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3353 t
= maybe_lookup_decl (decl
, up
);
3355 return t
? t
: decl
;
3359 /* Construct the initialization value for reduction CLAUSE. */
3362 omp_reduction_init (tree clause
, tree type
)
3364 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3365 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3372 case TRUTH_ORIF_EXPR
:
3373 case TRUTH_XOR_EXPR
:
3375 return build_zero_cst (type
);
3378 case TRUTH_AND_EXPR
:
3379 case TRUTH_ANDIF_EXPR
:
3381 return fold_convert_loc (loc
, type
, integer_one_node
);
3384 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3387 if (SCALAR_FLOAT_TYPE_P (type
))
3389 REAL_VALUE_TYPE max
, min
;
3390 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3393 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3396 real_maxval (&min
, 1, TYPE_MODE (type
));
3397 return build_real (type
, min
);
3401 gcc_assert (INTEGRAL_TYPE_P (type
));
3402 return TYPE_MIN_VALUE (type
);
3406 if (SCALAR_FLOAT_TYPE_P (type
))
3408 REAL_VALUE_TYPE max
;
3409 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3412 real_maxval (&max
, 0, TYPE_MODE (type
));
3413 return build_real (type
, max
);
3417 gcc_assert (INTEGRAL_TYPE_P (type
));
3418 return TYPE_MAX_VALUE (type
);
3426 /* Return alignment to be assumed for var in CLAUSE, which should be
3427 OMP_CLAUSE_ALIGNED. */
3430 omp_clause_aligned_alignment (tree clause
)
3432 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3433 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3435 /* Otherwise return implementation defined alignment. */
3436 unsigned int al
= 1;
3437 machine_mode mode
, vmode
;
3438 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3440 vs
= 1 << floor_log2 (vs
);
3441 static enum mode_class classes
[]
3442 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3443 for (int i
= 0; i
< 4; i
+= 2)
3444 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3446 mode
= GET_MODE_WIDER_MODE (mode
))
3448 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3449 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3452 && GET_MODE_SIZE (vmode
) < vs
3453 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3454 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3456 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3457 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3459 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3460 / GET_MODE_SIZE (mode
));
3461 if (TYPE_MODE (type
) != vmode
)
3463 if (TYPE_ALIGN_UNIT (type
) > al
)
3464 al
= TYPE_ALIGN_UNIT (type
);
3466 return build_int_cst (integer_type_node
, al
);
3469 /* Return maximum possible vectorization factor for the target. */
3476 || !flag_tree_loop_optimize
3477 || (!flag_tree_loop_vectorize
3478 && (global_options_set
.x_flag_tree_loop_vectorize
3479 || global_options_set
.x_flag_tree_vectorize
)))
3482 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3485 vs
= 1 << floor_log2 (vs
);
3488 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3489 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3490 return GET_MODE_NUNITS (vqimode
);
3494 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3498 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3499 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3501 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
3505 max_vf
= omp_max_vf ();
3508 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3509 OMP_CLAUSE_SAFELEN
);
3510 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3512 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3514 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3518 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3519 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3525 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3526 tree avar
= create_tmp_var_raw (atype
, NULL
);
3527 if (TREE_ADDRESSABLE (new_var
))
3528 TREE_ADDRESSABLE (avar
) = 1;
3529 DECL_ATTRIBUTES (avar
)
3530 = tree_cons (get_identifier ("omp simd array"), NULL
,
3531 DECL_ATTRIBUTES (avar
));
3532 gimple_add_tmp_var (avar
);
3533 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3534 NULL_TREE
, NULL_TREE
);
3535 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3536 NULL_TREE
, NULL_TREE
);
3537 if (DECL_P (new_var
))
3539 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3540 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3545 /* Helper function of lower_rec_input_clauses. For a reference
3546 in simd reduction, add an underlying variable it will reference. */
3549 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3551 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3552 if (TREE_CONSTANT (z
))
3554 const char *name
= NULL
;
3555 if (DECL_NAME (new_vard
))
3556 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3558 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3559 gimple_add_tmp_var (z
);
3560 TREE_ADDRESSABLE (z
) = 1;
3561 z
= build_fold_addr_expr_loc (loc
, z
);
3562 gimplify_assign (new_vard
, z
, ilist
);
3566 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3567 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3568 private variables. Initialization statements go in ILIST, while calls
3569 to destructors go in DLIST. */
3572 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3573 omp_context
*ctx
, struct omp_for_data
*fd
)
3575 tree c
, dtor
, copyin_seq
, x
, ptr
;
3576 bool copyin_by_ref
= false;
3577 bool lastprivate_firstprivate
= false;
3578 bool reduction_omp_orig_ref
= false;
3580 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3581 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3583 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3584 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3585 gimple_seq llist
[2] = { NULL
, NULL
};
3589 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3590 with data sharing clauses referencing variable sized vars. That
3591 is unnecessarily hard to support and very unlikely to result in
3592 vectorized code anyway. */
3594 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3595 switch (OMP_CLAUSE_CODE (c
))
3597 case OMP_CLAUSE_LINEAR
:
3598 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3601 case OMP_CLAUSE_REDUCTION
:
3602 case OMP_CLAUSE_PRIVATE
:
3603 case OMP_CLAUSE_FIRSTPRIVATE
:
3604 case OMP_CLAUSE_LASTPRIVATE
:
3605 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3612 /* Do all the fixed sized types in the first pass, and the variable sized
3613 types in the second pass. This makes sure that the scalar arguments to
3614 the variable sized types are processed before we use them in the
3615 variable sized operations. */
3616 for (pass
= 0; pass
< 2; ++pass
)
3618 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3620 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3623 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3627 case OMP_CLAUSE_PRIVATE
:
3628 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3631 case OMP_CLAUSE_SHARED
:
3632 /* Ignore shared directives in teams construct. */
3633 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3635 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3637 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3640 case OMP_CLAUSE_FIRSTPRIVATE
:
3641 case OMP_CLAUSE_COPYIN
:
3642 case OMP_CLAUSE_LINEAR
:
3644 case OMP_CLAUSE_REDUCTION
:
3645 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3646 reduction_omp_orig_ref
= true;
3648 case OMP_CLAUSE__LOOPTEMP_
:
3649 /* Handle _looptemp_ clauses only on parallel. */
3653 case OMP_CLAUSE_LASTPRIVATE
:
3654 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3656 lastprivate_firstprivate
= true;
3660 /* Even without corresponding firstprivate, if
3661 decl is Fortran allocatable, it needs outer var
3664 && lang_hooks
.decls
.omp_private_outer_ref
3665 (OMP_CLAUSE_DECL (c
)))
3666 lastprivate_firstprivate
= true;
3668 case OMP_CLAUSE_ALIGNED
:
3671 var
= OMP_CLAUSE_DECL (c
);
3672 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3673 && !is_global_var (var
))
3675 new_var
= maybe_lookup_decl (var
, ctx
);
3676 if (new_var
== NULL_TREE
)
3677 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3678 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3679 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3680 omp_clause_aligned_alignment (c
));
3681 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3682 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3683 gimplify_and_add (x
, ilist
);
3685 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3686 && is_global_var (var
))
3688 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3689 new_var
= lookup_decl (var
, ctx
);
3690 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3691 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3692 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3693 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3694 omp_clause_aligned_alignment (c
));
3695 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3696 x
= create_tmp_var (ptype
, NULL
);
3697 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3698 gimplify_and_add (t
, ilist
);
3699 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3700 SET_DECL_VALUE_EXPR (new_var
, t
);
3701 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3708 new_var
= var
= OMP_CLAUSE_DECL (c
);
3709 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3710 new_var
= lookup_decl (var
, ctx
);
3712 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3717 else if (is_variable_sized (var
))
3719 /* For variable sized types, we need to allocate the
3720 actual storage here. Call alloca and store the
3721 result in the pointer decl that we created elsewhere. */
3725 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3730 ptr
= DECL_VALUE_EXPR (new_var
);
3731 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3732 ptr
= TREE_OPERAND (ptr
, 0);
3733 gcc_assert (DECL_P (ptr
));
3734 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3736 /* void *tmp = __builtin_alloca */
3737 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3738 stmt
= gimple_build_call (atmp
, 1, x
);
3739 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3740 gimple_add_tmp_var (tmp
);
3741 gimple_call_set_lhs (stmt
, tmp
);
3743 gimple_seq_add_stmt (ilist
, stmt
);
3745 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3746 gimplify_assign (ptr
, x
, ilist
);
3749 else if (is_reference (var
))
3751 /* For references that are being privatized for Fortran,
3752 allocate new backing storage for the new pointer
3753 variable. This allows us to avoid changing all the
3754 code that expects a pointer to something that expects
3755 a direct variable. */
3759 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3760 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3762 x
= build_receiver_ref (var
, false, ctx
);
3763 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3765 else if (TREE_CONSTANT (x
))
3767 /* For reduction in SIMD loop, defer adding the
3768 initialization of the reference, because if we decide
3769 to use SIMD array for it, the initilization could cause
3771 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3775 const char *name
= NULL
;
3776 if (DECL_NAME (var
))
3777 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3779 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3781 gimple_add_tmp_var (x
);
3782 TREE_ADDRESSABLE (x
) = 1;
3783 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3788 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3789 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3794 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3795 gimplify_assign (new_var
, x
, ilist
);
3798 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3800 else if (c_kind
== OMP_CLAUSE_REDUCTION
3801 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3809 switch (OMP_CLAUSE_CODE (c
))
3811 case OMP_CLAUSE_SHARED
:
3812 /* Ignore shared directives in teams construct. */
3813 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3815 /* Shared global vars are just accessed directly. */
3816 if (is_global_var (new_var
))
3818 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3819 needs to be delayed until after fixup_child_record_type so
3820 that we get the correct type during the dereference. */
3821 by_ref
= use_pointer_for_field (var
, ctx
);
3822 x
= build_receiver_ref (var
, by_ref
, ctx
);
3823 SET_DECL_VALUE_EXPR (new_var
, x
);
3824 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3826 /* ??? If VAR is not passed by reference, and the variable
3827 hasn't been initialized yet, then we'll get a warning for
3828 the store into the omp_data_s structure. Ideally, we'd be
3829 able to notice this and not store anything at all, but
3830 we're generating code too early. Suppress the warning. */
3832 TREE_NO_WARNING (var
) = 1;
3835 case OMP_CLAUSE_LASTPRIVATE
:
3836 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3840 case OMP_CLAUSE_PRIVATE
:
3841 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3842 x
= build_outer_var_ref (var
, ctx
);
3843 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3845 if (is_task_ctx (ctx
))
3846 x
= build_receiver_ref (var
, false, ctx
);
3848 x
= build_outer_var_ref (var
, ctx
);
3854 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3857 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3858 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3859 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3860 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3861 idx
, lane
, ivar
, lvar
))
3864 x
= lang_hooks
.decls
.omp_clause_default_ctor
3865 (c
, unshare_expr (ivar
), x
);
3867 gimplify_and_add (x
, &llist
[0]);
3870 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3873 gimple_seq tseq
= NULL
;
3876 gimplify_stmt (&dtor
, &tseq
);
3877 gimple_seq_add_seq (&llist
[1], tseq
);
3884 gimplify_and_add (nx
, ilist
);
3888 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3891 gimple_seq tseq
= NULL
;
3894 gimplify_stmt (&dtor
, &tseq
);
3895 gimple_seq_add_seq (dlist
, tseq
);
3899 case OMP_CLAUSE_LINEAR
:
3900 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3901 goto do_firstprivate
;
3902 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3905 x
= build_outer_var_ref (var
, ctx
);
3908 case OMP_CLAUSE_FIRSTPRIVATE
:
3909 if (is_task_ctx (ctx
))
3911 if (is_reference (var
) || is_variable_sized (var
))
3913 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3915 || use_pointer_for_field (var
, NULL
))
3917 x
= build_receiver_ref (var
, false, ctx
);
3918 SET_DECL_VALUE_EXPR (new_var
, x
);
3919 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3924 x
= build_outer_var_ref (var
, ctx
);
3927 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3928 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3930 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3931 tree stept
= TREE_TYPE (t
);
3932 tree ct
= find_omp_clause (clauses
,
3933 OMP_CLAUSE__LOOPTEMP_
);
3935 tree l
= OMP_CLAUSE_DECL (ct
);
3936 tree n1
= fd
->loop
.n1
;
3937 tree step
= fd
->loop
.step
;
3938 tree itype
= TREE_TYPE (l
);
3939 if (POINTER_TYPE_P (itype
))
3940 itype
= signed_type_for (itype
);
3941 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3942 if (TYPE_UNSIGNED (itype
)
3943 && fd
->loop
.cond_code
== GT_EXPR
)
3944 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3945 fold_build1 (NEGATE_EXPR
, itype
, l
),
3946 fold_build1 (NEGATE_EXPR
,
3949 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3950 t
= fold_build2 (MULT_EXPR
, stept
,
3951 fold_convert (stept
, l
), t
);
3953 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3955 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3957 gimplify_and_add (x
, ilist
);
3961 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3962 x
= fold_build2 (POINTER_PLUS_EXPR
,
3963 TREE_TYPE (x
), x
, t
);
3965 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3968 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3969 || TREE_ADDRESSABLE (new_var
))
3970 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3971 idx
, lane
, ivar
, lvar
))
3973 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3975 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3976 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3977 gimplify_and_add (x
, ilist
);
3978 gimple_stmt_iterator gsi
3979 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3981 = gimple_build_assign (unshare_expr (lvar
), iv
);
3982 gsi_insert_before_without_update (&gsi
, g
,
3984 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3985 enum tree_code code
= PLUS_EXPR
;
3986 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3987 code
= POINTER_PLUS_EXPR
;
3988 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3989 gsi_insert_before_without_update (&gsi
, g
,
3993 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3994 (c
, unshare_expr (ivar
), x
);
3995 gimplify_and_add (x
, &llist
[0]);
3996 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3999 gimple_seq tseq
= NULL
;
4002 gimplify_stmt (&dtor
, &tseq
);
4003 gimple_seq_add_seq (&llist
[1], tseq
);
4008 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4009 gimplify_and_add (x
, ilist
);
4012 case OMP_CLAUSE__LOOPTEMP_
:
4013 gcc_assert (is_parallel_ctx (ctx
));
4014 x
= build_outer_var_ref (var
, ctx
);
4015 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4016 gimplify_and_add (x
, ilist
);
4019 case OMP_CLAUSE_COPYIN
:
4020 by_ref
= use_pointer_for_field (var
, NULL
);
4021 x
= build_receiver_ref (var
, by_ref
, ctx
);
4022 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4023 append_to_statement_list (x
, ©in_seq
);
4024 copyin_by_ref
|= by_ref
;
4027 case OMP_CLAUSE_REDUCTION
:
4028 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4030 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4032 x
= build_outer_var_ref (var
, ctx
);
4034 if (is_reference (var
)
4035 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4037 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4038 SET_DECL_VALUE_EXPR (placeholder
, x
);
4039 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4040 tree new_vard
= new_var
;
4041 if (is_reference (var
))
4043 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4044 new_vard
= TREE_OPERAND (new_var
, 0);
4045 gcc_assert (DECL_P (new_vard
));
4048 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4049 idx
, lane
, ivar
, lvar
))
4051 if (new_vard
== new_var
)
4053 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4054 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4058 SET_DECL_VALUE_EXPR (new_vard
,
4059 build_fold_addr_expr (ivar
));
4060 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4062 x
= lang_hooks
.decls
.omp_clause_default_ctor
4063 (c
, unshare_expr (ivar
),
4064 build_outer_var_ref (var
, ctx
));
4066 gimplify_and_add (x
, &llist
[0]);
4067 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4069 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4070 lower_omp (&tseq
, ctx
);
4071 gimple_seq_add_seq (&llist
[0], tseq
);
4073 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4074 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4075 lower_omp (&tseq
, ctx
);
4076 gimple_seq_add_seq (&llist
[1], tseq
);
4077 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4078 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4079 if (new_vard
== new_var
)
4080 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4082 SET_DECL_VALUE_EXPR (new_vard
,
4083 build_fold_addr_expr (lvar
));
4084 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4089 gimplify_stmt (&dtor
, &tseq
);
4090 gimple_seq_add_seq (&llist
[1], tseq
);
4094 /* If this is a reference to constant size reduction var
4095 with placeholder, we haven't emitted the initializer
4096 for it because it is undesirable if SIMD arrays are used.
4097 But if they aren't used, we need to emit the deferred
4098 initialization now. */
4099 else if (is_reference (var
) && is_simd
)
4100 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4101 x
= lang_hooks
.decls
.omp_clause_default_ctor
4102 (c
, unshare_expr (new_var
),
4103 build_outer_var_ref (var
, ctx
));
4105 gimplify_and_add (x
, ilist
);
4106 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4108 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4109 lower_omp (&tseq
, ctx
);
4110 gimple_seq_add_seq (ilist
, tseq
);
4112 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4115 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4116 lower_omp (&tseq
, ctx
);
4117 gimple_seq_add_seq (dlist
, tseq
);
4118 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4120 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4125 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4126 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4127 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4129 /* reduction(-:var) sums up the partial results, so it
4130 acts identically to reduction(+:var). */
4131 if (code
== MINUS_EXPR
)
4134 tree new_vard
= new_var
;
4135 if (is_simd
&& is_reference (var
))
4137 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4138 new_vard
= TREE_OPERAND (new_var
, 0);
4139 gcc_assert (DECL_P (new_vard
));
4142 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4143 idx
, lane
, ivar
, lvar
))
4145 tree ref
= build_outer_var_ref (var
, ctx
);
4147 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4149 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4150 ref
= build_outer_var_ref (var
, ctx
);
4151 gimplify_assign (ref
, x
, &llist
[1]);
4153 if (new_vard
!= new_var
)
4155 SET_DECL_VALUE_EXPR (new_vard
,
4156 build_fold_addr_expr (lvar
));
4157 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4162 if (is_reference (var
) && is_simd
)
4163 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4164 gimplify_assign (new_var
, x
, ilist
);
4167 tree ref
= build_outer_var_ref (var
, ctx
);
4169 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4170 ref
= build_outer_var_ref (var
, ctx
);
4171 gimplify_assign (ref
, x
, dlist
);
4185 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4186 /* Don't want uninit warnings on simduid, it is always uninitialized,
4187 but we use it not for the value, but for the DECL_UID only. */
4188 TREE_NO_WARNING (uid
) = 1;
4190 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4191 gimple_call_set_lhs (g
, lane
);
4192 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4193 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4194 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4195 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4196 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4197 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4198 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
4199 build_int_cst (unsigned_type_node
, 0),
4201 gimple_seq_add_stmt (ilist
, g
);
4202 for (int i
= 0; i
< 2; i
++)
4205 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
4206 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4207 gimple_call_set_lhs (g
, vf
);
4208 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4209 gimple_seq_add_stmt (seq
, g
);
4210 tree t
= build_int_cst (unsigned_type_node
, 0);
4211 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
4212 gimple_seq_add_stmt (seq
, g
);
4213 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4214 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4215 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4216 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4217 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4218 gimple_seq_add_seq (seq
, llist
[i
]);
4219 t
= build_int_cst (unsigned_type_node
, 1);
4220 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
4221 gimple_seq_add_stmt (seq
, g
);
4222 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4223 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4224 gimple_seq_add_stmt (seq
, g
);
4225 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4229 /* The copyin sequence is not to be executed by the main thread, since
4230 that would result in self-copies. Perhaps not visible to scalars,
4231 but it certainly is to C++ operator=. */
4234 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4236 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4237 build_int_cst (TREE_TYPE (x
), 0));
4238 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4239 gimplify_and_add (x
, ilist
);
4242 /* If any copyin variable is passed by reference, we must ensure the
4243 master thread doesn't modify it before it is copied over in all
4244 threads. Similarly for variables in both firstprivate and
4245 lastprivate clauses we need to ensure the lastprivate copying
4246 happens after firstprivate copying in all threads. And similarly
4247 for UDRs if initializer expression refers to omp_orig. */
4248 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4250 /* Don't add any barrier for #pragma omp simd or
4251 #pragma omp distribute. */
4252 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4253 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4254 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4257 /* If max_vf is non-zero, then we can use only a vectorization factor
4258 up to the max_vf we chose. So stick it into the safelen clause. */
4261 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4262 OMP_CLAUSE_SAFELEN
);
4264 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4265 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4268 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4269 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4271 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4272 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4278 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4279 both parallel and workshare constructs. PREDICATE may be NULL if it's
4283 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4286 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4287 bool par_clauses
= false;
4288 tree simduid
= NULL
, lastlane
= NULL
;
4290 /* Early exit if there are no lastprivate or linear clauses. */
4291 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4292 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4293 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4294 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4296 if (clauses
== NULL
)
4298 /* If this was a workshare clause, see if it had been combined
4299 with its parallel. In that case, look for the clauses on the
4300 parallel statement itself. */
4301 if (is_parallel_ctx (ctx
))
4305 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4308 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4309 OMP_CLAUSE_LASTPRIVATE
);
4310 if (clauses
== NULL
)
4318 tree label_true
, arm1
, arm2
;
4320 label
= create_artificial_label (UNKNOWN_LOCATION
);
4321 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4322 arm1
= TREE_OPERAND (predicate
, 0);
4323 arm2
= TREE_OPERAND (predicate
, 1);
4324 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4325 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4326 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4328 gimple_seq_add_stmt (stmt_list
, stmt
);
4329 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4332 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4333 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4335 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4337 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4340 for (c
= clauses
; c
;)
4343 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4345 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4346 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4347 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4349 var
= OMP_CLAUSE_DECL (c
);
4350 new_var
= lookup_decl (var
, ctx
);
4352 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4354 tree val
= DECL_VALUE_EXPR (new_var
);
4355 if (TREE_CODE (val
) == ARRAY_REF
4356 && VAR_P (TREE_OPERAND (val
, 0))
4357 && lookup_attribute ("omp simd array",
4358 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4361 if (lastlane
== NULL
)
4363 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
4365 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4367 TREE_OPERAND (val
, 1));
4368 gimple_call_set_lhs (g
, lastlane
);
4369 gimple_seq_add_stmt (stmt_list
, g
);
4371 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4372 TREE_OPERAND (val
, 0), lastlane
,
4373 NULL_TREE
, NULL_TREE
);
4377 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4378 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4380 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4381 gimple_seq_add_seq (stmt_list
,
4382 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4383 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4385 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4386 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4388 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4389 gimple_seq_add_seq (stmt_list
,
4390 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4391 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4394 x
= build_outer_var_ref (var
, ctx
);
4395 if (is_reference (var
))
4396 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4397 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4398 gimplify_and_add (x
, stmt_list
);
4400 c
= OMP_CLAUSE_CHAIN (c
);
4401 if (c
== NULL
&& !par_clauses
)
4403 /* If this was a workshare clause, see if it had been combined
4404 with its parallel. In that case, continue looking for the
4405 clauses also on the parallel statement itself. */
4406 if (is_parallel_ctx (ctx
))
4410 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4413 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4414 OMP_CLAUSE_LASTPRIVATE
);
4420 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4424 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4425 tree tid
, tree var
, tree new_var
)
4427 /* The atomic add at the end of the sum creates unnecessary
4428 write contention on accelerators. To work around this,
4429 create an array to store the partial reductions. Later, in
4430 lower_omp_for (for openacc), the values of array will be
4433 tree t
= NULL_TREE
, array
, x
;
4434 tree type
= get_base_type (var
);
4437 /* Now insert the partial reductions into the array. */
4439 /* Find the reduction array. */
4441 tree ptype
= build_pointer_type (type
);
4443 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4444 t
= build_receiver_ref (t
, false, ctx
->outer
);
4446 array
= create_tmp_var (ptype
, NULL
);
4447 gimplify_assign (array
, t
, stmt_seqp
);
4449 tree ptr
= create_tmp_var (TREE_TYPE (array
), NULL
);
4451 /* Find the reduction array. */
4453 /* testing a unary conversion. */
4454 tree offset
= create_tmp_var (sizetype
, NULL
);
4455 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4457 t
= create_tmp_var (sizetype
, NULL
);
4458 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4460 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, offset
, offset
, t
);
4461 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4463 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4464 of adding sizeof(var) to the array? */
4465 ptr
= create_tmp_var (ptype
, NULL
);
4466 stmt
= gimple_build_assign_with_ops (POINTER_PLUS_EXPR
, unshare_expr(ptr
),
4468 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4470 /* Move the local sum to gfc$sum[i]. */
4471 x
= unshare_expr (build_simple_mem_ref (ptr
));
4472 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4475 /* Generate code to implement the REDUCTION clauses. */
4478 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4480 gimple_seq sub_seq
= NULL
;
4482 tree x
, c
, tid
= NULL_TREE
;
4485 /* SIMD reductions are handled in lower_rec_input_clauses. */
4486 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4487 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4490 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4491 update in that case, otherwise use a lock. */
4492 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4493 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4495 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4497 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4507 /* Initialize thread info for OpenACC. */
4508 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
4510 /* Get the current thread id. */
4511 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4512 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)), NULL
);
4513 gimple stmt
= gimple_build_call (call
, 0);
4514 gimple_call_set_lhs (stmt
, tid
);
4515 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4518 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4520 tree var
, ref
, new_var
;
4521 enum tree_code code
;
4522 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4524 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4527 var
= OMP_CLAUSE_DECL (c
);
4528 new_var
= lookup_decl (var
, ctx
);
4529 if (is_reference (var
))
4530 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4531 ref
= build_outer_var_ref (var
, ctx
);
4532 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4534 /* reduction(-:var) sums up the partial results, so it acts
4535 identically to reduction(+:var). */
4536 if (code
== MINUS_EXPR
)
4541 if (!is_gimple_omp_oacc_specifically (ctx
->stmt
))
4543 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4545 addr
= save_expr (addr
);
4546 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4547 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4548 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4549 gimplify_and_add (x
, stmt_seqp
);
4554 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
,
4560 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4562 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4564 if (is_reference (var
)
4565 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4567 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4568 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4569 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4570 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4571 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4572 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4573 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4577 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
4579 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
,
4584 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4585 ref
= build_outer_var_ref (var
, ctx
);
4586 gimplify_assign (ref
, x
, &sub_seq
);
4591 if (is_gimple_omp_oacc_specifically (ctx
->stmt
))
4594 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4596 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4598 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4600 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4602 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4606 /* Generate code to implement the COPYPRIVATE clauses. */
4609 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4612 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
4616 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4618 tree var
, new_var
, ref
, x
;
4620 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4622 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4625 var
= OMP_CLAUSE_DECL (c
);
4626 by_ref
= use_pointer_for_field (var
, NULL
);
4628 ref
= build_sender_ref (var
, ctx
);
4629 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4632 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4633 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4635 gimplify_assign (ref
, x
, slist
);
4637 ref
= build_receiver_ref (var
, false, ctx
);
4640 ref
= fold_convert_loc (clause_loc
,
4641 build_pointer_type (TREE_TYPE (new_var
)),
4643 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4645 if (is_reference (var
))
4647 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4648 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4649 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4651 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4652 gimplify_and_add (x
, rlist
);
4657 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4658 and REDUCTION from the sender (aka parent) side. */
4661 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4664 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
4668 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4670 tree val
, ref
, x
, var
;
4671 bool by_ref
, do_in
= false, do_out
= false;
4672 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4674 switch (OMP_CLAUSE_CODE (c
))
4676 case OMP_CLAUSE_PRIVATE
:
4677 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4680 case OMP_CLAUSE_FIRSTPRIVATE
:
4681 case OMP_CLAUSE_COPYIN
:
4682 case OMP_CLAUSE_LASTPRIVATE
:
4683 case OMP_CLAUSE_REDUCTION
:
4684 case OMP_CLAUSE__LOOPTEMP_
:
4690 val
= OMP_CLAUSE_DECL (c
);
4691 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4693 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4694 && is_global_var (var
))
4696 if (is_variable_sized (val
))
4698 by_ref
= use_pointer_for_field (val
, NULL
);
4700 switch (OMP_CLAUSE_CODE (c
))
4702 case OMP_CLAUSE_PRIVATE
:
4703 case OMP_CLAUSE_FIRSTPRIVATE
:
4704 case OMP_CLAUSE_COPYIN
:
4705 case OMP_CLAUSE__LOOPTEMP_
:
4709 case OMP_CLAUSE_LASTPRIVATE
:
4710 if (by_ref
|| is_reference (val
))
4712 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4719 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4724 case OMP_CLAUSE_REDUCTION
:
4726 do_out
= !(by_ref
|| is_reference (val
));
4735 ref
= build_sender_ref (val
, ctx
);
4736 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4737 gimplify_assign (ref
, x
, ilist
);
4738 if (is_task_ctx (ctx
))
4739 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4744 ref
= build_sender_ref (val
, ctx
);
4745 gimplify_assign (var
, ref
, olist
);
4750 /* Generate code to implement SHARED from the sender (aka parent)
4751 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4752 list things that got automatically shared. */
4755 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4757 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
4759 tree var
, ovar
, nvar
, f
, x
, record_type
;
4761 if (ctx
->record_type
== NULL
)
4764 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4765 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4767 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4768 nvar
= maybe_lookup_decl (ovar
, ctx
);
4769 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4772 /* If CTX is a nested parallel directive. Find the immediately
4773 enclosing parallel or workshare construct that contains a
4774 mapping for OVAR. */
4775 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4777 if (use_pointer_for_field (ovar
, ctx
))
4779 x
= build_sender_ref (ovar
, ctx
);
4780 var
= build_fold_addr_expr (var
);
4781 gimplify_assign (x
, var
, ilist
);
4785 x
= build_sender_ref (ovar
, ctx
);
4786 gimplify_assign (x
, var
, ilist
);
4788 if (!TREE_READONLY (var
)
4789 /* We don't need to receive a new reference to a result
4790 or parm decl. In fact we may not store to it as we will
4791 invalidate any pending RSO and generate wrong gimple
4793 && !((TREE_CODE (var
) == RESULT_DECL
4794 || TREE_CODE (var
) == PARM_DECL
)
4795 && DECL_BY_REFERENCE (var
)))
4797 x
= build_sender_ref (ovar
, ctx
);
4798 gimplify_assign (var
, x
, olist
);
4805 /* A convenience function to build an empty GIMPLE_COND with just the
4809 gimple_build_cond_empty (tree cond
)
4811 enum tree_code pred_code
;
4814 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4815 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4819 /* Build the function calls to GOMP_parallel_start etc to actually
4820 generate the parallel operation. REGION is the parallel region
4821 being expanded. BB is the block where to insert the code. WS_ARGS
4822 will be set if this is a call to a combined parallel+workshare
4823 construct, it contains the list of additional arguments needed by
4824 the workshare construct. */
4827 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4828 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4830 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4831 gimple_stmt_iterator gsi
;
4833 enum built_in_function start_ix
;
4835 location_t clause_loc
;
4836 vec
<tree
, va_gc
> *args
;
4838 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4840 /* Determine what flavor of GOMP_parallel we will be
4842 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4843 if (is_combined_parallel (region
))
4845 switch (region
->inner
->type
)
4847 case GIMPLE_OMP_FOR
:
4848 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4849 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4850 + (region
->inner
->sched_kind
4851 == OMP_CLAUSE_SCHEDULE_RUNTIME
4852 ? 3 : region
->inner
->sched_kind
));
4853 start_ix
= (enum built_in_function
)start_ix2
;
4855 case GIMPLE_OMP_SECTIONS
:
4856 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4863 /* By default, the value of NUM_THREADS is zero (selected at run time)
4864 and there is no conditional. */
4866 val
= build_int_cst (unsigned_type_node
, 0);
4867 flags
= build_int_cst (unsigned_type_node
, 0);
4869 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4871 cond
= OMP_CLAUSE_IF_EXPR (c
);
4873 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4876 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4877 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4880 clause_loc
= gimple_location (entry_stmt
);
4882 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4884 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4886 /* Ensure 'val' is of the correct type. */
4887 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4889 /* If we found the clause 'if (cond)', build either
4890 (cond != 0) or (cond ? val : 1u). */
4893 cond
= gimple_boolify (cond
);
4895 if (integer_zerop (val
))
4896 val
= fold_build2_loc (clause_loc
,
4897 EQ_EXPR
, unsigned_type_node
, cond
,
4898 build_int_cst (TREE_TYPE (cond
), 0));
4901 basic_block cond_bb
, then_bb
, else_bb
;
4902 edge e
, e_then
, e_else
;
4903 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4905 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4906 if (gimple_in_ssa_p (cfun
))
4908 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4909 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4910 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4919 e
= split_block (bb
, NULL
);
4924 then_bb
= create_empty_bb (cond_bb
);
4925 else_bb
= create_empty_bb (then_bb
);
4926 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4927 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4929 stmt
= gimple_build_cond_empty (cond
);
4930 gsi
= gsi_start_bb (cond_bb
);
4931 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4933 gsi
= gsi_start_bb (then_bb
);
4934 stmt
= gimple_build_assign (tmp_then
, val
);
4935 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4937 gsi
= gsi_start_bb (else_bb
);
4938 stmt
= gimple_build_assign
4939 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4940 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4942 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4943 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4944 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4945 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4946 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4947 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4949 if (gimple_in_ssa_p (cfun
))
4951 gimple phi
= create_phi_node (tmp_join
, bb
);
4952 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4953 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4959 gsi
= gsi_start_bb (bb
);
4960 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4961 false, GSI_CONTINUE_LINKING
);
4964 gsi
= gsi_last_bb (bb
);
4965 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4967 t1
= null_pointer_node
;
4969 t1
= build_fold_addr_expr (t
);
4970 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4972 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4973 args
->quick_push (t2
);
4974 args
->quick_push (t1
);
4975 args
->quick_push (val
);
4977 args
->splice (*ws_args
);
4978 args
->quick_push (flags
);
4980 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4981 builtin_decl_explicit (start_ix
), args
);
4983 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4984 false, GSI_CONTINUE_LINKING
);
4987 /* Insert a function call whose name is FUNC_NAME with the information from
4988 ENTRY_STMT into the basic_block BB. */
4991 expand_cilk_for_call (basic_block bb
, gimple entry_stmt
,
4992 vec
<tree
, va_gc
> *ws_args
)
4995 gimple_stmt_iterator gsi
;
4996 vec
<tree
, va_gc
> *args
;
4998 gcc_assert (vec_safe_length (ws_args
) == 2);
4999 tree func_name
= (*ws_args
)[0];
5000 tree grain
= (*ws_args
)[1];
5002 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5003 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5004 gcc_assert (count
!= NULL_TREE
);
5005 count
= OMP_CLAUSE_OPERAND (count
, 0);
5007 gsi
= gsi_last_bb (bb
);
5008 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5010 t1
= null_pointer_node
;
5012 t1
= build_fold_addr_expr (t
);
5013 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5015 vec_alloc (args
, 4);
5016 args
->quick_push (t2
);
5017 args
->quick_push (t1
);
5018 args
->quick_push (count
);
5019 args
->quick_push (grain
);
5020 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5022 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5023 GSI_CONTINUE_LINKING
);
5026 /* Build the function call to GOMP_task to actually
5027 generate the task operation. BB is the block where to insert the code. */
5030 expand_task_call (basic_block bb
, gimple entry_stmt
)
5032 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5033 gimple_stmt_iterator gsi
;
5034 location_t loc
= gimple_location (entry_stmt
);
5036 clauses
= gimple_omp_task_clauses (entry_stmt
);
5038 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5040 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5042 cond
= boolean_true_node
;
5044 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5045 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5046 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5047 flags
= build_int_cst (unsigned_type_node
,
5048 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5050 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5053 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5054 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5055 build_int_cst (unsigned_type_node
, 2),
5056 build_int_cst (unsigned_type_node
, 0));
5057 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5060 depend
= OMP_CLAUSE_DECL (depend
);
5062 depend
= build_int_cst (ptr_type_node
, 0);
5064 gsi
= gsi_last_bb (bb
);
5065 t
= gimple_omp_task_data_arg (entry_stmt
);
5067 t2
= null_pointer_node
;
5069 t2
= build_fold_addr_expr_loc (loc
, t
);
5070 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5071 t
= gimple_omp_task_copy_fn (entry_stmt
);
5073 t3
= null_pointer_node
;
5075 t3
= build_fold_addr_expr_loc (loc
, t
);
5077 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5079 gimple_omp_task_arg_size (entry_stmt
),
5080 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5083 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5084 false, GSI_CONTINUE_LINKING
);
5088 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5089 catch handler and return it. This prevents programs from violating the
5090 structured block semantics with throws. */
5093 maybe_catch_exception (gimple_seq body
)
5098 if (!flag_exceptions
)
5101 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5102 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5104 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5106 g
= gimple_build_eh_must_not_throw (decl
);
5107 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5110 return gimple_seq_alloc_with_stmt (g
);
5113 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5116 vec2chain (vec
<tree
, va_gc
> *v
)
5118 tree chain
= NULL_TREE
, t
;
5121 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5123 DECL_CHAIN (t
) = chain
;
5131 /* Remove barriers in REGION->EXIT's block. Note that this is only
5132 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5133 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5134 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5138 remove_exit_barrier (struct omp_region
*region
)
5140 gimple_stmt_iterator gsi
;
5141 basic_block exit_bb
;
5145 int any_addressable_vars
= -1;
5147 exit_bb
= region
->exit
;
5149 /* If the parallel region doesn't return, we don't have REGION->EXIT
5154 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5155 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5156 statements that can appear in between are extremely limited -- no
5157 memory operations at all. Here, we allow nothing at all, so the
5158 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5159 gsi
= gsi_last_bb (exit_bb
);
5160 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5162 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5165 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5167 gsi
= gsi_last_bb (e
->src
);
5168 if (gsi_end_p (gsi
))
5170 stmt
= gsi_stmt (gsi
);
5171 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5172 && !gimple_omp_return_nowait_p (stmt
))
5174 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5175 in many cases. If there could be tasks queued, the barrier
5176 might be needed to let the tasks run before some local
5177 variable of the parallel that the task uses as shared
5178 runs out of scope. The task can be spawned either
5179 from within current function (this would be easy to check)
5180 or from some function it calls and gets passed an address
5181 of such a variable. */
5182 if (any_addressable_vars
< 0)
5184 gimple parallel_stmt
= last_stmt (region
->entry
);
5185 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5186 tree local_decls
, block
, decl
;
5189 any_addressable_vars
= 0;
5190 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5191 if (TREE_ADDRESSABLE (decl
))
5193 any_addressable_vars
= 1;
5196 for (block
= gimple_block (stmt
);
5197 !any_addressable_vars
5199 && TREE_CODE (block
) == BLOCK
;
5200 block
= BLOCK_SUPERCONTEXT (block
))
5202 for (local_decls
= BLOCK_VARS (block
);
5204 local_decls
= DECL_CHAIN (local_decls
))
5205 if (TREE_ADDRESSABLE (local_decls
))
5207 any_addressable_vars
= 1;
5210 if (block
== gimple_block (parallel_stmt
))
5214 if (!any_addressable_vars
)
5215 gimple_omp_return_set_nowait (stmt
);
5221 remove_exit_barriers (struct omp_region
*region
)
5223 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5224 remove_exit_barrier (region
);
5228 region
= region
->inner
;
5229 remove_exit_barriers (region
);
5230 while (region
->next
)
5232 region
= region
->next
;
5233 remove_exit_barriers (region
);
5238 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5239 calls. These can't be declared as const functions, but
5240 within one parallel body they are constant, so they can be
5241 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5242 which are declared const. Similarly for task body, except
5243 that in untied task omp_get_thread_num () can change at any task
5244 scheduling point. */
5247 optimize_omp_library_calls (gimple entry_stmt
)
5250 gimple_stmt_iterator gsi
;
5251 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5252 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5253 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5254 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5255 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5256 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5257 OMP_CLAUSE_UNTIED
) != NULL
);
5259 FOR_EACH_BB_FN (bb
, cfun
)
5260 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5262 gimple call
= gsi_stmt (gsi
);
5265 if (is_gimple_call (call
)
5266 && (decl
= gimple_call_fndecl (call
))
5267 && DECL_EXTERNAL (decl
)
5268 && TREE_PUBLIC (decl
)
5269 && DECL_INITIAL (decl
) == NULL
)
5273 if (DECL_NAME (decl
) == thr_num_id
)
5275 /* In #pragma omp task untied omp_get_thread_num () can change
5276 during the execution of the task region. */
5279 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5281 else if (DECL_NAME (decl
) == num_thr_id
)
5282 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5286 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5287 || gimple_call_num_args (call
) != 0)
5290 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5293 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5294 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5295 TREE_TYPE (TREE_TYPE (built_in
))))
5298 gimple_call_set_fndecl (call
, built_in
);
5303 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5307 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5311 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5312 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5315 if (TREE_CODE (t
) == ADDR_EXPR
)
5316 recompute_tree_invariant_for_addr_expr (t
);
5318 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5322 /* Prepend TO = FROM assignment before *GSI_P. */
5325 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5327 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5328 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5329 true, GSI_SAME_STMT
);
5330 gimple stmt
= gimple_build_assign (to
, from
);
5331 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5332 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5333 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5335 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5336 gimple_regimplify_operands (stmt
, &gsi
);
5340 /* Expand the OpenMP parallel or task directive starting at REGION. */
5343 expand_omp_taskreg (struct omp_region
*region
)
5345 basic_block entry_bb
, exit_bb
, new_bb
;
5346 struct function
*child_cfun
;
5347 tree child_fn
, block
, t
;
5348 gimple_stmt_iterator gsi
;
5349 gimple entry_stmt
, stmt
;
5351 vec
<tree
, va_gc
> *ws_args
;
5353 entry_stmt
= last_stmt (region
->entry
);
5354 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5355 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5357 entry_bb
= region
->entry
;
5358 exit_bb
= region
->exit
;
5362 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5363 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5364 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5367 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5368 and the inner statement contains the name of the built-in function
5370 ws_args
= region
->inner
->ws_args
;
5371 else if (is_combined_parallel (region
))
5372 ws_args
= region
->ws_args
;
5376 if (child_cfun
->cfg
)
5378 /* Due to inlining, it may happen that we have already outlined
5379 the region, in which case all we need to do is make the
5380 sub-graph unreachable and emit the parallel call. */
5381 edge entry_succ_e
, exit_succ_e
;
5383 entry_succ_e
= single_succ_edge (entry_bb
);
5385 gsi
= gsi_last_bb (entry_bb
);
5386 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5387 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5388 gsi_remove (&gsi
, true);
5393 exit_succ_e
= single_succ_edge (exit_bb
);
5394 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5396 remove_edge_and_dominated_blocks (entry_succ_e
);
5400 unsigned srcidx
, dstidx
, num
;
5402 /* If the parallel region needs data sent from the parent
5403 function, then the very first statement (except possible
5404 tree profile counter updates) of the parallel body
5405 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5406 &.OMP_DATA_O is passed as an argument to the child function,
5407 we need to replace it with the argument as seen by the child
5410 In most cases, this will end up being the identity assignment
5411 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5412 a function call that has been inlined, the original PARM_DECL
5413 .OMP_DATA_I may have been converted into a different local
5414 variable. In which case, we need to keep the assignment. */
5415 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5417 basic_block entry_succ_bb
= single_succ (entry_bb
);
5419 gimple parcopy_stmt
= NULL
;
5421 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5425 gcc_assert (!gsi_end_p (gsi
));
5426 stmt
= gsi_stmt (gsi
);
5427 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5430 if (gimple_num_ops (stmt
) == 2)
5432 tree arg
= gimple_assign_rhs1 (stmt
);
5434 /* We're ignore the subcode because we're
5435 effectively doing a STRIP_NOPS. */
5437 if (TREE_CODE (arg
) == ADDR_EXPR
5438 && TREE_OPERAND (arg
, 0)
5439 == gimple_omp_taskreg_data_arg (entry_stmt
))
5441 parcopy_stmt
= stmt
;
5447 gcc_assert (parcopy_stmt
!= NULL
);
5448 arg
= DECL_ARGUMENTS (child_fn
);
5450 if (!gimple_in_ssa_p (cfun
))
5452 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5453 gsi_remove (&gsi
, true);
5456 /* ?? Is setting the subcode really necessary ?? */
5457 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5458 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5463 /* If we are in ssa form, we must load the value from the default
5464 definition of the argument. That should not be defined now,
5465 since the argument is not used uninitialized. */
5466 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5467 narg
= make_ssa_name (arg
, gimple_build_nop ());
5468 set_ssa_default_def (cfun
, arg
, narg
);
5469 /* ?? Is setting the subcode really necessary ?? */
5470 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5471 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5472 update_stmt (parcopy_stmt
);
5476 /* Declare local variables needed in CHILD_CFUN. */
5477 block
= DECL_INITIAL (child_fn
);
5478 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5479 /* The gimplifier could record temporaries in parallel/task block
5480 rather than in containing function's local_decls chain,
5481 which would mean cgraph missed finalizing them. Do it now. */
5482 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5483 if (TREE_CODE (t
) == VAR_DECL
5485 && !DECL_EXTERNAL (t
))
5486 varpool_node::finalize_decl (t
);
5487 DECL_SAVED_TREE (child_fn
) = NULL
;
5488 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5489 gimple_set_body (child_fn
, NULL
);
5490 TREE_USED (block
) = 1;
5492 /* Reset DECL_CONTEXT on function arguments. */
5493 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5494 DECL_CONTEXT (t
) = child_fn
;
5496 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5497 so that it can be moved to the child function. */
5498 gsi
= gsi_last_bb (entry_bb
);
5499 stmt
= gsi_stmt (gsi
);
5500 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5501 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5502 gsi_remove (&gsi
, true);
5503 e
= split_block (entry_bb
, stmt
);
5505 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5507 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5510 gsi
= gsi_last_bb (exit_bb
);
5511 gcc_assert (!gsi_end_p (gsi
)
5512 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5513 stmt
= gimple_build_return (NULL
);
5514 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5515 gsi_remove (&gsi
, true);
5518 /* Move the parallel region into CHILD_CFUN. */
5520 if (gimple_in_ssa_p (cfun
))
5522 init_tree_ssa (child_cfun
);
5523 init_ssa_operands (child_cfun
);
5524 child_cfun
->gimple_df
->in_ssa_p
= true;
5528 block
= gimple_block (entry_stmt
);
5530 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5532 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5533 /* When the OMP expansion process cannot guarantee an up-to-date
5534 loop tree arrange for the child function to fixup loops. */
5535 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5536 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5538 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5539 num
= vec_safe_length (child_cfun
->local_decls
);
5540 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5542 t
= (*child_cfun
->local_decls
)[srcidx
];
5543 if (DECL_CONTEXT (t
) == cfun
->decl
)
5545 if (srcidx
!= dstidx
)
5546 (*child_cfun
->local_decls
)[dstidx
] = t
;
5550 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5552 /* Inform the callgraph about the new function. */
5553 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5554 cgraph_node::add_new_function (child_fn
, true);
5556 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5557 fixed in a following pass. */
5558 push_cfun (child_cfun
);
5560 optimize_omp_library_calls (entry_stmt
);
5561 cgraph_edge::rebuild_edges ();
5563 /* Some EH regions might become dead, see PR34608. If
5564 pass_cleanup_cfg isn't the first pass to happen with the
5565 new child, these dead EH edges might cause problems.
5566 Clean them up now. */
5567 if (flag_exceptions
)
5570 bool changed
= false;
5572 FOR_EACH_BB_FN (bb
, cfun
)
5573 changed
|= gimple_purge_dead_eh_edges (bb
);
5575 cleanup_tree_cfg ();
5577 if (gimple_in_ssa_p (cfun
))
5578 update_ssa (TODO_update_ssa
);
5582 /* Emit a library call to launch the children threads. */
5584 expand_cilk_for_call (new_bb
, entry_stmt
, ws_args
);
5585 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5586 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5588 expand_task_call (new_bb
, entry_stmt
);
5589 if (gimple_in_ssa_p (cfun
))
5590 update_ssa (TODO_update_ssa_only_virtuals
);
5594 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5595 of the combined collapse > 1 loop constructs, generate code like:
5596 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5601 count3 = (adj + N32 - N31) / STEP3;
5602 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5607 count2 = (adj + N22 - N21) / STEP2;
5608 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5613 count1 = (adj + N12 - N11) / STEP1;
5614 count = count1 * count2 * count3;
5615 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5617 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5618 of the combined loop constructs, just initialize COUNTS array
5619 from the _looptemp_ clauses. */
5621 /* NOTE: It *could* be better to moosh all of the BBs together,
5622 creating one larger BB with all the computation and the unexpected
5623 jump at the end. I.e.
5625 bool zero3, zero2, zero1, zero;
5628 count3 = (N32 - N31) /[cl] STEP3;
5630 count2 = (N22 - N21) /[cl] STEP2;
5632 count1 = (N12 - N11) /[cl] STEP1;
5633 zero = zero3 || zero2 || zero1;
5634 count = count1 * count2 * count3;
5635 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5637 After all, we expect the zero=false, and thus we expect to have to
5638 evaluate all of the comparison expressions, so short-circuiting
5639 oughtn't be a win. Since the condition isn't protecting a
5640 denominator, we're not concerned about divide-by-zero, so we can
5641 fully evaluate count even if a numerator turned out to be wrong.
5643 It seems like putting this all together would create much better
5644 scheduling opportunities, and less pressure on the chip's branch
5648 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5649 basic_block
&entry_bb
, tree
*counts
,
5650 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5651 basic_block
&l2_dom_bb
)
5653 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5658 /* Collapsed loops need work for expansion into SSA form. */
5659 gcc_assert (!gimple_in_ssa_p (cfun
));
5661 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5662 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5664 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5665 isn't supposed to be handled, as the inner loop doesn't
5667 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5668 OMP_CLAUSE__LOOPTEMP_
);
5669 gcc_assert (innerc
);
5670 for (i
= 0; i
< fd
->collapse
; i
++)
5672 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5673 OMP_CLAUSE__LOOPTEMP_
);
5674 gcc_assert (innerc
);
5676 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5678 counts
[0] = NULL_TREE
;
5683 for (i
= 0; i
< fd
->collapse
; i
++)
5685 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5687 if (SSA_VAR_P (fd
->loop
.n2
)
5688 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5689 fold_convert (itype
, fd
->loops
[i
].n1
),
5690 fold_convert (itype
, fd
->loops
[i
].n2
)))
5691 == NULL_TREE
|| !integer_onep (t
)))
5694 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5695 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5696 true, GSI_SAME_STMT
);
5697 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5698 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5699 true, GSI_SAME_STMT
);
5700 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5701 NULL_TREE
, NULL_TREE
);
5702 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5703 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5704 expand_omp_regimplify_p
, NULL
, NULL
)
5705 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5706 expand_omp_regimplify_p
, NULL
, NULL
))
5708 *gsi
= gsi_for_stmt (stmt
);
5709 gimple_regimplify_operands (stmt
, gsi
);
5711 e
= split_block (entry_bb
, stmt
);
5712 if (zero_iter_bb
== NULL
)
5714 first_zero_iter
= i
;
5715 zero_iter_bb
= create_empty_bb (entry_bb
);
5716 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5717 *gsi
= gsi_after_labels (zero_iter_bb
);
5718 stmt
= gimple_build_assign (fd
->loop
.n2
,
5719 build_zero_cst (type
));
5720 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5721 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5724 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5725 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5726 e
->flags
= EDGE_TRUE_VALUE
;
5727 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5728 if (l2_dom_bb
== NULL
)
5729 l2_dom_bb
= entry_bb
;
5731 *gsi
= gsi_last_bb (entry_bb
);
5734 if (POINTER_TYPE_P (itype
))
5735 itype
= signed_type_for (itype
);
5736 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5738 t
= fold_build2 (PLUS_EXPR
, itype
,
5739 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5740 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5741 fold_convert (itype
, fd
->loops
[i
].n2
));
5742 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5743 fold_convert (itype
, fd
->loops
[i
].n1
));
5744 /* ?? We could probably use CEIL_DIV_EXPR instead of
5745 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5746 generate the same code in the end because generically we
5747 don't know that the values involved must be negative for
5749 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5750 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5751 fold_build1 (NEGATE_EXPR
, itype
, t
),
5752 fold_build1 (NEGATE_EXPR
, itype
,
5753 fold_convert (itype
,
5754 fd
->loops
[i
].step
)));
5756 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5757 fold_convert (itype
, fd
->loops
[i
].step
));
5758 t
= fold_convert (type
, t
);
5759 if (TREE_CODE (t
) == INTEGER_CST
)
5763 counts
[i
] = create_tmp_reg (type
, ".count");
5764 expand_omp_build_assign (gsi
, counts
[i
], t
);
5766 if (SSA_VAR_P (fd
->loop
.n2
))
5771 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5772 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5778 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5780 V3 = N31 + (T % count3) * STEP3;
5782 V2 = N21 + (T % count2) * STEP2;
5784 V1 = N11 + T * STEP1;
5785 if this loop doesn't have an inner loop construct combined with it.
5786 If it does have an inner loop construct combined with it and the
5787 iteration count isn't known constant, store values from counts array
5788 into its _looptemp_ temporaries instead. */
5791 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5792 tree
*counts
, gimple inner_stmt
, tree startvar
)
5795 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5797 /* If fd->loop.n2 is constant, then no propagation of the counts
5798 is needed, they are constant. */
5799 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5802 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5803 ? gimple_omp_parallel_clauses (inner_stmt
)
5804 : gimple_omp_for_clauses (inner_stmt
);
5805 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5806 isn't supposed to be handled, as the inner loop doesn't
5808 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5809 gcc_assert (innerc
);
5810 for (i
= 0; i
< fd
->collapse
; i
++)
5812 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5813 OMP_CLAUSE__LOOPTEMP_
);
5814 gcc_assert (innerc
);
5817 tree tem
= OMP_CLAUSE_DECL (innerc
);
5818 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5819 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5820 false, GSI_CONTINUE_LINKING
);
5821 gimple stmt
= gimple_build_assign (tem
, t
);
5822 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5828 tree type
= TREE_TYPE (fd
->loop
.v
);
5829 tree tem
= create_tmp_reg (type
, ".tem");
5830 gimple stmt
= gimple_build_assign (tem
, startvar
);
5831 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5833 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5835 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5837 if (POINTER_TYPE_P (vtype
))
5838 itype
= signed_type_for (vtype
);
5840 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5843 t
= fold_convert (itype
, t
);
5844 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5845 fold_convert (itype
, fd
->loops
[i
].step
));
5846 if (POINTER_TYPE_P (vtype
))
5847 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5849 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5850 t
= force_gimple_operand_gsi (gsi
, t
,
5851 DECL_P (fd
->loops
[i
].v
)
5852 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5854 GSI_CONTINUE_LINKING
);
5855 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5856 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5859 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5860 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5861 false, GSI_CONTINUE_LINKING
);
5862 stmt
= gimple_build_assign (tem
, t
);
5863 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5869 /* Helper function for expand_omp_for_*. Generate code like:
5872 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5876 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5883 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5884 basic_block body_bb
)
5886 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5888 gimple_stmt_iterator gsi
;
5894 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5896 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5898 bb
= create_empty_bb (last_bb
);
5899 add_bb_to_loop (bb
, last_bb
->loop_father
);
5900 gsi
= gsi_start_bb (bb
);
5902 if (i
< fd
->collapse
- 1)
5904 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5905 e
->probability
= REG_BR_PROB_BASE
/ 8;
5907 t
= fd
->loops
[i
+ 1].n1
;
5908 t
= force_gimple_operand_gsi (&gsi
, t
,
5909 DECL_P (fd
->loops
[i
+ 1].v
)
5910 && TREE_ADDRESSABLE (fd
->loops
[i
5913 GSI_CONTINUE_LINKING
);
5914 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5915 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5920 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5922 if (POINTER_TYPE_P (vtype
))
5923 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5925 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5926 t
= force_gimple_operand_gsi (&gsi
, t
,
5927 DECL_P (fd
->loops
[i
].v
)
5928 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5929 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5930 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5931 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5935 t
= fd
->loops
[i
].n2
;
5936 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5937 false, GSI_CONTINUE_LINKING
);
5938 tree v
= fd
->loops
[i
].v
;
5939 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5940 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5941 false, GSI_CONTINUE_LINKING
);
5942 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5943 stmt
= gimple_build_cond_empty (t
);
5944 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5945 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5946 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5949 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5957 /* A subroutine of expand_omp_for. Generate code for a parallel
5958 loop with any schedule. Given parameters:
5960 for (V = N1; V cond N2; V += STEP) BODY;
5962 where COND is "<" or ">", we generate pseudocode
5964 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5965 if (more) goto L0; else goto L3;
5972 if (V cond iend) goto L1; else goto L2;
5974 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5977 If this is a combined omp parallel loop, instead of the call to
5978 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5979 If this is gimple_omp_for_combined_p loop, then instead of assigning
5980 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5981 inner GIMPLE_OMP_FOR and V += STEP; and
5982 if (V cond iend) goto L1; else goto L2; are removed.
5984 For collapsed loops, given parameters:
5986 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5987 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5988 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5991 we generate pseudocode
5993 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5998 count3 = (adj + N32 - N31) / STEP3;
5999 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6004 count2 = (adj + N22 - N21) / STEP2;
6005 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6010 count1 = (adj + N12 - N11) / STEP1;
6011 count = count1 * count2 * count3;
6016 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6017 if (more) goto L0; else goto L3;
6021 V3 = N31 + (T % count3) * STEP3;
6023 V2 = N21 + (T % count2) * STEP2;
6025 V1 = N11 + T * STEP1;
6030 if (V < iend) goto L10; else goto L2;
6033 if (V3 cond3 N32) goto L1; else goto L11;
6037 if (V2 cond2 N22) goto L1; else goto L12;
6043 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6049 expand_omp_for_generic (struct omp_region
*region
,
6050 struct omp_for_data
*fd
,
6051 enum built_in_function start_fn
,
6052 enum built_in_function next_fn
,
6055 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
) != GF_OMP_FOR_KIND_OACC_LOOP
);
6057 tree type
, istart0
, iend0
, iend
;
6058 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6059 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6060 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6061 gimple_stmt_iterator gsi
;
6063 bool in_combined_parallel
= is_combined_parallel (region
);
6064 bool broken_loop
= region
->cont
== NULL
;
6066 tree
*counts
= NULL
;
6069 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6070 gcc_assert (fd
->iter_type
== long_integer_type_node
6071 || !in_combined_parallel
);
6073 type
= TREE_TYPE (fd
->loop
.v
);
6074 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6075 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6076 TREE_ADDRESSABLE (istart0
) = 1;
6077 TREE_ADDRESSABLE (iend0
) = 1;
6079 /* See if we need to bias by LLONG_MIN. */
6080 if (fd
->iter_type
== long_long_unsigned_type_node
6081 && TREE_CODE (type
) == INTEGER_TYPE
6082 && !TYPE_UNSIGNED (type
))
6086 if (fd
->loop
.cond_code
== LT_EXPR
)
6089 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6093 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6096 if (TREE_CODE (n1
) != INTEGER_CST
6097 || TREE_CODE (n2
) != INTEGER_CST
6098 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6099 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6102 entry_bb
= region
->entry
;
6103 cont_bb
= region
->cont
;
6105 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6106 gcc_assert (broken_loop
6107 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6108 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6109 l1_bb
= single_succ (l0_bb
);
6112 l2_bb
= create_empty_bb (cont_bb
);
6113 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6114 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6118 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6119 exit_bb
= region
->exit
;
6121 gsi
= gsi_last_bb (entry_bb
);
6123 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6124 if (fd
->collapse
> 1)
6126 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi
))
6127 != GF_OMP_FOR_KIND_OACC_LOOP
);
6129 int first_zero_iter
= -1;
6130 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6132 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6133 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6134 zero_iter_bb
, first_zero_iter
,
6139 /* Some counts[i] vars might be uninitialized if
6140 some loop has zero iterations. But the body shouldn't
6141 be executed in that case, so just avoid uninit warnings. */
6142 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6143 if (SSA_VAR_P (counts
[i
]))
6144 TREE_NO_WARNING (counts
[i
]) = 1;
6146 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6148 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6149 gsi
= gsi_last_bb (entry_bb
);
6150 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6151 get_immediate_dominator (CDI_DOMINATORS
,
6155 if (in_combined_parallel
)
6157 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi
))
6158 != GF_OMP_FOR_KIND_OACC_LOOP
);
6160 /* In a combined parallel loop, emit a call to
6161 GOMP_loop_foo_next. */
6162 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6163 build_fold_addr_expr (istart0
),
6164 build_fold_addr_expr (iend0
));
6168 tree t0
, t1
, t2
, t3
, t4
;
6169 /* If this is not a combined parallel loop, emit a call to
6170 GOMP_loop_foo_start in ENTRY_BB. */
6171 t4
= build_fold_addr_expr (iend0
);
6172 t3
= build_fold_addr_expr (istart0
);
6173 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6176 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6178 gcc_assert (gimple_omp_for_kind (gsi_stmt (gsi
))
6179 != GF_OMP_FOR_KIND_OACC_LOOP
);
6181 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6182 OMP_CLAUSE__LOOPTEMP_
);
6183 gcc_assert (innerc
);
6184 t0
= OMP_CLAUSE_DECL (innerc
);
6185 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6186 OMP_CLAUSE__LOOPTEMP_
);
6187 gcc_assert (innerc
);
6188 t1
= OMP_CLAUSE_DECL (innerc
);
6190 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6191 && TYPE_PRECISION (TREE_TYPE (t0
))
6192 != TYPE_PRECISION (fd
->iter_type
))
6194 /* Avoid casting pointers to integer of a different size. */
6195 tree itype
= signed_type_for (type
);
6196 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6197 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6201 t1
= fold_convert (fd
->iter_type
, t1
);
6202 t0
= fold_convert (fd
->iter_type
, t0
);
6206 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6207 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6209 if (fd
->iter_type
== long_integer_type_node
)
6213 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6214 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6215 6, t0
, t1
, t2
, t
, t3
, t4
);
6218 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6219 5, t0
, t1
, t2
, t3
, t4
);
6227 /* The GOMP_loop_ull_*start functions have additional boolean
6228 argument, true for < loops and false for > loops.
6229 In Fortran, the C bool type can be different from
6230 boolean_type_node. */
6231 bfn_decl
= builtin_decl_explicit (start_fn
);
6232 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6233 t5
= build_int_cst (c_bool_type
,
6234 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6237 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6238 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6239 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6242 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6243 6, t5
, t0
, t1
, t2
, t3
, t4
);
6246 if (TREE_TYPE (t
) != boolean_type_node
)
6247 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6248 t
, build_int_cst (TREE_TYPE (t
), 0));
6249 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6250 true, GSI_SAME_STMT
);
6251 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6253 /* Remove the GIMPLE_OMP_FOR statement. */
6254 gsi_remove (&gsi
, true);
6256 /* Iteration setup for sequential loop goes in L0_BB. */
6257 tree startvar
= fd
->loop
.v
;
6258 tree endvar
= NULL_TREE
;
6260 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6262 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6263 && gimple_omp_for_kind (inner_stmt
)
6264 == GF_OMP_FOR_KIND_SIMD
);
6265 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6266 OMP_CLAUSE__LOOPTEMP_
);
6267 gcc_assert (innerc
);
6268 startvar
= OMP_CLAUSE_DECL (innerc
);
6269 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6270 OMP_CLAUSE__LOOPTEMP_
);
6271 gcc_assert (innerc
);
6272 endvar
= OMP_CLAUSE_DECL (innerc
);
6275 gsi
= gsi_start_bb (l0_bb
);
6278 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6279 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6280 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6281 t
= fold_convert (TREE_TYPE (startvar
), t
);
6282 t
= force_gimple_operand_gsi (&gsi
, t
,
6284 && TREE_ADDRESSABLE (startvar
),
6285 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6286 stmt
= gimple_build_assign (startvar
, t
);
6287 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6291 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6292 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6293 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6294 t
= fold_convert (TREE_TYPE (startvar
), t
);
6295 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6296 false, GSI_CONTINUE_LINKING
);
6299 stmt
= gimple_build_assign (endvar
, iend
);
6300 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6301 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6302 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6304 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
6306 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6308 if (fd
->collapse
> 1)
6309 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6313 /* Code to control the increment and predicate for the sequential
6314 loop goes in the CONT_BB. */
6315 gsi
= gsi_last_bb (cont_bb
);
6316 stmt
= gsi_stmt (gsi
);
6317 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6318 vmain
= gimple_omp_continue_control_use (stmt
);
6319 vback
= gimple_omp_continue_control_def (stmt
);
6321 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6323 if (POINTER_TYPE_P (type
))
6324 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6326 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6327 t
= force_gimple_operand_gsi (&gsi
, t
,
6329 && TREE_ADDRESSABLE (vback
),
6330 NULL_TREE
, true, GSI_SAME_STMT
);
6331 stmt
= gimple_build_assign (vback
, t
);
6332 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6334 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6335 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6337 stmt
= gimple_build_cond_empty (t
);
6338 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6341 /* Remove GIMPLE_OMP_CONTINUE. */
6342 gsi_remove (&gsi
, true);
6344 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6345 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6347 /* Emit code to get the next parallel iteration in L2_BB. */
6348 gsi
= gsi_start_bb (l2_bb
);
6350 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6351 build_fold_addr_expr (istart0
),
6352 build_fold_addr_expr (iend0
));
6353 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6354 false, GSI_CONTINUE_LINKING
);
6355 if (TREE_TYPE (t
) != boolean_type_node
)
6356 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6357 t
, build_int_cst (TREE_TYPE (t
), 0));
6358 stmt
= gimple_build_cond_empty (t
);
6359 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6362 /* Add the loop cleanup function. */
6363 gsi
= gsi_last_bb (exit_bb
);
6364 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6365 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6366 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6367 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6369 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6370 stmt
= gimple_build_call (t
, 0);
6371 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6372 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6373 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
6374 gsi_remove (&gsi
, true);
6376 /* Connect the new blocks. */
6377 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6378 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6384 e
= find_edge (cont_bb
, l3_bb
);
6385 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6387 phis
= phi_nodes (l3_bb
);
6388 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6390 gimple phi
= gsi_stmt (gsi
);
6391 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6392 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6396 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6397 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6398 e
= find_edge (cont_bb
, l1_bb
);
6399 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6404 else if (fd
->collapse
> 1)
6407 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6410 e
->flags
= EDGE_TRUE_VALUE
;
6413 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6414 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6418 e
= find_edge (cont_bb
, l2_bb
);
6419 e
->flags
= EDGE_FALLTHRU
;
6421 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6423 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6424 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6425 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6426 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6427 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6428 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6429 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6430 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6432 struct loop
*outer_loop
= alloc_loop ();
6433 outer_loop
->header
= l0_bb
;
6434 outer_loop
->latch
= l2_bb
;
6435 add_loop (outer_loop
, l0_bb
->loop_father
);
6437 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6439 struct loop
*loop
= alloc_loop ();
6440 loop
->header
= l1_bb
;
6441 /* The loop may have multiple latches. */
6442 add_loop (loop
, outer_loop
);
6448 /* A subroutine of expand_omp_for. Generate code for a parallel
6449 loop with static schedule and no specified chunk size. Given
6452 for (V = N1; V cond N2; V += STEP) BODY;
6454 where COND is "<" or ">", we generate pseudocode
6456 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6461 if ((__typeof (V)) -1 > 0 && cond is >)
6462 n = -(adj + N2 - N1) / -STEP;
6464 n = (adj + N2 - N1) / STEP;
6467 if (threadid < tt) goto L3; else goto L4;
6472 s0 = q * threadid + tt;
6475 if (s0 >= e0) goto L2; else goto L0;
6481 if (V cond e) goto L1;
6486 expand_omp_for_static_nochunk (struct omp_region
*region
,
6487 struct omp_for_data
*fd
,
6490 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6491 tree type
, itype
, vmain
, vback
;
6492 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6493 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6495 gimple_stmt_iterator gsi
;
6498 bool broken_loop
= region
->cont
== NULL
;
6499 tree
*counts
= NULL
;
6502 gcc_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6503 != GF_OMP_FOR_KIND_OACC_LOOP
)
6506 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6507 if (POINTER_TYPE_P (type
))
6508 itype
= signed_type_for (type
);
6510 entry_bb
= region
->entry
;
6511 cont_bb
= region
->cont
;
6512 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6513 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6514 gcc_assert (broken_loop
6515 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6516 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6517 body_bb
= single_succ (seq_start_bb
);
6520 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6521 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6523 exit_bb
= region
->exit
;
6525 /* Iteration space partitioning goes in ENTRY_BB. */
6526 gsi
= gsi_last_bb (entry_bb
);
6527 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6529 if (fd
->collapse
> 1)
6531 int first_zero_iter
= -1;
6532 basic_block l2_dom_bb
= NULL
;
6534 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6535 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6536 fin_bb
, first_zero_iter
,
6540 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6541 t
= integer_one_node
;
6543 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6544 fold_convert (type
, fd
->loop
.n1
),
6545 fold_convert (type
, fd
->loop
.n2
));
6546 if (fd
->collapse
== 1
6547 && TYPE_UNSIGNED (type
)
6548 && (t
== NULL_TREE
|| !integer_onep (t
)))
6550 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6551 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6552 true, GSI_SAME_STMT
);
6553 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6554 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6555 true, GSI_SAME_STMT
);
6556 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6557 NULL_TREE
, NULL_TREE
);
6558 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6559 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6560 expand_omp_regimplify_p
, NULL
, NULL
)
6561 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6562 expand_omp_regimplify_p
, NULL
, NULL
))
6564 gsi
= gsi_for_stmt (stmt
);
6565 gimple_regimplify_operands (stmt
, &gsi
);
6567 ep
= split_block (entry_bb
, stmt
);
6568 ep
->flags
= EDGE_TRUE_VALUE
;
6569 entry_bb
= ep
->dest
;
6570 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6571 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6572 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6573 if (gimple_in_ssa_p (cfun
))
6575 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6576 for (gsi
= gsi_start_phis (fin_bb
);
6577 !gsi_end_p (gsi
); gsi_next (&gsi
))
6579 gimple phi
= gsi_stmt (gsi
);
6580 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6581 ep
, UNKNOWN_LOCATION
);
6584 gsi
= gsi_last_bb (entry_bb
);
6587 switch (gimple_omp_for_kind (fd
->for_stmt
))
6589 case GF_OMP_FOR_KIND_FOR
:
6590 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6591 nthreads
= build_call_expr (nthreads
, 0);
6592 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6593 threadid
= build_call_expr (threadid
, 0);
6595 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6596 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6597 nthreads
= build_call_expr (nthreads
, 0);
6598 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6599 threadid
= build_call_expr (threadid
, 0);
6601 case GF_OMP_FOR_KIND_OACC_LOOP
:
6602 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6603 nthreads
= build_call_expr (nthreads
, 0);
6604 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6605 threadid
= build_call_expr (threadid
, 0);
6610 nthreads
= fold_convert (itype
, nthreads
);
6611 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6612 true, GSI_SAME_STMT
);
6613 threadid
= fold_convert (itype
, threadid
);
6614 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6615 true, GSI_SAME_STMT
);
6619 step
= fd
->loop
.step
;
6620 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6622 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
6623 != GF_OMP_FOR_KIND_OACC_LOOP
);
6625 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6626 OMP_CLAUSE__LOOPTEMP_
);
6627 gcc_assert (innerc
);
6628 n1
= OMP_CLAUSE_DECL (innerc
);
6629 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6630 OMP_CLAUSE__LOOPTEMP_
);
6631 gcc_assert (innerc
);
6632 n2
= OMP_CLAUSE_DECL (innerc
);
6634 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6635 true, NULL_TREE
, true, GSI_SAME_STMT
);
6636 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6637 true, NULL_TREE
, true, GSI_SAME_STMT
);
6638 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6639 true, NULL_TREE
, true, GSI_SAME_STMT
);
6641 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6642 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6643 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6644 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6645 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6646 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6647 fold_build1 (NEGATE_EXPR
, itype
, t
),
6648 fold_build1 (NEGATE_EXPR
, itype
, step
));
6650 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6651 t
= fold_convert (itype
, t
);
6652 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6654 q
= create_tmp_reg (itype
, "q");
6655 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6656 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6657 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6659 tt
= create_tmp_reg (itype
, "tt");
6660 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6661 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6662 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6664 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6665 stmt
= gimple_build_cond_empty (t
);
6666 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6668 second_bb
= split_block (entry_bb
, stmt
)->dest
;
6669 gsi
= gsi_last_bb (second_bb
);
6670 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6672 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6674 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6675 build_int_cst (itype
, 1));
6676 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6678 third_bb
= split_block (second_bb
, stmt
)->dest
;
6679 gsi
= gsi_last_bb (third_bb
);
6680 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6682 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6683 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6684 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6686 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6687 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6689 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6690 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6692 /* Remove the GIMPLE_OMP_FOR statement. */
6693 gsi_remove (&gsi
, true);
6695 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6696 gsi
= gsi_start_bb (seq_start_bb
);
6698 tree startvar
= fd
->loop
.v
;
6699 tree endvar
= NULL_TREE
;
6701 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6703 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6704 ? gimple_omp_parallel_clauses (inner_stmt
)
6705 : gimple_omp_for_clauses (inner_stmt
);
6706 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6707 gcc_assert (innerc
);
6708 startvar
= OMP_CLAUSE_DECL (innerc
);
6709 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6710 OMP_CLAUSE__LOOPTEMP_
);
6711 gcc_assert (innerc
);
6712 endvar
= OMP_CLAUSE_DECL (innerc
);
6714 t
= fold_convert (itype
, s0
);
6715 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6716 if (POINTER_TYPE_P (type
))
6717 t
= fold_build_pointer_plus (n1
, t
);
6719 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6720 t
= fold_convert (TREE_TYPE (startvar
), t
);
6721 t
= force_gimple_operand_gsi (&gsi
, t
,
6723 && TREE_ADDRESSABLE (startvar
),
6724 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6725 stmt
= gimple_build_assign (startvar
, t
);
6726 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6728 t
= fold_convert (itype
, e0
);
6729 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6730 if (POINTER_TYPE_P (type
))
6731 t
= fold_build_pointer_plus (n1
, t
);
6733 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6734 t
= fold_convert (TREE_TYPE (startvar
), t
);
6735 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6736 false, GSI_CONTINUE_LINKING
);
6739 stmt
= gimple_build_assign (endvar
, e
);
6740 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6741 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6742 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6744 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6746 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6748 if (fd
->collapse
> 1)
6749 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6753 /* The code controlling the sequential loop replaces the
6754 GIMPLE_OMP_CONTINUE. */
6755 gsi
= gsi_last_bb (cont_bb
);
6756 stmt
= gsi_stmt (gsi
);
6757 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6758 vmain
= gimple_omp_continue_control_use (stmt
);
6759 vback
= gimple_omp_continue_control_def (stmt
);
6761 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6763 if (POINTER_TYPE_P (type
))
6764 t
= fold_build_pointer_plus (vmain
, step
);
6766 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6767 t
= force_gimple_operand_gsi (&gsi
, t
,
6769 && TREE_ADDRESSABLE (vback
),
6770 NULL_TREE
, true, GSI_SAME_STMT
);
6771 stmt
= gimple_build_assign (vback
, t
);
6772 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6774 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6775 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6777 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6780 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6781 gsi_remove (&gsi
, true);
6783 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6784 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6787 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6788 gsi
= gsi_last_bb (exit_bb
);
6789 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6791 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
6792 != GF_OMP_FOR_KIND_OACC_LOOP
);
6794 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6795 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6797 gsi_remove (&gsi
, true);
6799 /* Connect all the blocks. */
6800 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6801 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6802 ep
= find_edge (entry_bb
, second_bb
);
6803 ep
->flags
= EDGE_TRUE_VALUE
;
6804 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6805 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6806 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6810 ep
= find_edge (cont_bb
, body_bb
);
6811 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6813 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
6814 != GF_OMP_FOR_KIND_OACC_LOOP
);
6819 else if (fd
->collapse
> 1)
6822 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6825 ep
->flags
= EDGE_TRUE_VALUE
;
6826 find_edge (cont_bb
, fin_bb
)->flags
6827 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6830 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6831 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6832 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6834 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6835 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6836 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6837 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6839 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6841 struct loop
*loop
= alloc_loop ();
6842 loop
->header
= body_bb
;
6843 if (collapse_bb
== NULL
)
6844 loop
->latch
= cont_bb
;
6845 add_loop (loop
, body_bb
->loop_father
);
6850 /* A subroutine of expand_omp_for. Generate code for a parallel
6851 loop with static schedule and a specified chunk size. Given
6854 for (V = N1; V cond N2; V += STEP) BODY;
6856 where COND is "<" or ">", we generate pseudocode
6858 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6863 if ((__typeof (V)) -1 > 0 && cond is >)
6864 n = -(adj + N2 - N1) / -STEP;
6866 n = (adj + N2 - N1) / STEP;
6868 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6869 here so that V is defined
6870 if the loop is not entered
6872 s0 = (trip * nthreads + threadid) * CHUNK;
6873 e0 = min(s0 + CHUNK, n);
6874 if (s0 < n) goto L1; else goto L4;
6881 if (V cond e) goto L2; else goto L3;
6889 expand_omp_for_static_chunk (struct omp_region
*region
,
6890 struct omp_for_data
*fd
, gimple inner_stmt
)
6892 tree n
, s0
, e0
, e
, t
;
6893 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6894 tree type
, itype
, vmain
, vback
, vextra
;
6895 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6896 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6897 gimple_stmt_iterator gsi
;
6900 bool broken_loop
= region
->cont
== NULL
;
6901 tree
*counts
= NULL
;
6904 gcc_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6905 != GF_OMP_FOR_KIND_OACC_LOOP
)
6908 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6909 if (POINTER_TYPE_P (type
))
6910 itype
= signed_type_for (type
);
6912 entry_bb
= region
->entry
;
6913 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6915 iter_part_bb
= se
->dest
;
6916 cont_bb
= region
->cont
;
6917 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6918 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6919 gcc_assert (broken_loop
6920 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6921 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6922 body_bb
= single_succ (seq_start_bb
);
6925 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6926 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6927 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6929 exit_bb
= region
->exit
;
6931 /* Trip and adjustment setup goes in ENTRY_BB. */
6932 gsi
= gsi_last_bb (entry_bb
);
6933 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6935 if (fd
->collapse
> 1)
6937 int first_zero_iter
= -1;
6938 basic_block l2_dom_bb
= NULL
;
6940 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6941 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6942 fin_bb
, first_zero_iter
,
6946 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6947 t
= integer_one_node
;
6949 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6950 fold_convert (type
, fd
->loop
.n1
),
6951 fold_convert (type
, fd
->loop
.n2
));
6952 if (fd
->collapse
== 1
6953 && TYPE_UNSIGNED (type
)
6954 && (t
== NULL_TREE
|| !integer_onep (t
)))
6956 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6957 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6958 true, GSI_SAME_STMT
);
6959 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6960 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6961 true, GSI_SAME_STMT
);
6962 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6963 NULL_TREE
, NULL_TREE
);
6964 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6965 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6966 expand_omp_regimplify_p
, NULL
, NULL
)
6967 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6968 expand_omp_regimplify_p
, NULL
, NULL
))
6970 gsi
= gsi_for_stmt (stmt
);
6971 gimple_regimplify_operands (stmt
, &gsi
);
6973 se
= split_block (entry_bb
, stmt
);
6974 se
->flags
= EDGE_TRUE_VALUE
;
6975 entry_bb
= se
->dest
;
6976 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6977 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6978 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6979 if (gimple_in_ssa_p (cfun
))
6981 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6982 for (gsi
= gsi_start_phis (fin_bb
);
6983 !gsi_end_p (gsi
); gsi_next (&gsi
))
6985 gimple phi
= gsi_stmt (gsi
);
6986 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6987 se
, UNKNOWN_LOCATION
);
6990 gsi
= gsi_last_bb (entry_bb
);
6993 switch (gimple_omp_for_kind (fd
->for_stmt
))
6995 case GF_OMP_FOR_KIND_FOR
:
6996 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6997 nthreads
= build_call_expr (nthreads
, 0);
6998 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6999 threadid
= build_call_expr (threadid
, 0);
7001 case GF_OMP_FOR_KIND_DISTRIBUTE
:
7002 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
7003 nthreads
= build_call_expr (nthreads
, 0);
7004 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
7005 threadid
= build_call_expr (threadid
, 0);
7007 case GF_OMP_FOR_KIND_OACC_LOOP
:
7008 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7009 nthreads
= build_call_expr (nthreads
, 0);
7010 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7011 threadid
= build_call_expr (threadid
, 0);
7016 nthreads
= fold_convert (itype
, nthreads
);
7017 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7018 true, GSI_SAME_STMT
);
7019 threadid
= fold_convert (itype
, threadid
);
7020 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7021 true, GSI_SAME_STMT
);
7025 step
= fd
->loop
.step
;
7026 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7028 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
7029 != GF_OMP_FOR_KIND_OACC_LOOP
);
7031 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7032 OMP_CLAUSE__LOOPTEMP_
);
7033 gcc_assert (innerc
);
7034 n1
= OMP_CLAUSE_DECL (innerc
);
7035 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7036 OMP_CLAUSE__LOOPTEMP_
);
7037 gcc_assert (innerc
);
7038 n2
= OMP_CLAUSE_DECL (innerc
);
7040 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7041 true, NULL_TREE
, true, GSI_SAME_STMT
);
7042 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7043 true, NULL_TREE
, true, GSI_SAME_STMT
);
7044 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7045 true, NULL_TREE
, true, GSI_SAME_STMT
);
7047 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7048 true, NULL_TREE
, true, GSI_SAME_STMT
);
7050 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7051 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7052 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7053 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7054 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7055 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7056 fold_build1 (NEGATE_EXPR
, itype
, t
),
7057 fold_build1 (NEGATE_EXPR
, itype
, step
));
7059 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7060 t
= fold_convert (itype
, t
);
7061 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7062 true, GSI_SAME_STMT
);
7064 trip_var
= create_tmp_reg (itype
, ".trip");
7065 if (gimple_in_ssa_p (cfun
))
7067 trip_init
= make_ssa_name (trip_var
, NULL
);
7068 trip_main
= make_ssa_name (trip_var
, NULL
);
7069 trip_back
= make_ssa_name (trip_var
, NULL
);
7073 trip_init
= trip_var
;
7074 trip_main
= trip_var
;
7075 trip_back
= trip_var
;
7078 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7079 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7081 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7082 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7083 if (POINTER_TYPE_P (type
))
7084 t
= fold_build_pointer_plus (n1
, t
);
7086 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7087 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7088 true, GSI_SAME_STMT
);
7090 /* Remove the GIMPLE_OMP_FOR. */
7091 gsi_remove (&gsi
, true);
7093 /* Iteration space partitioning goes in ITER_PART_BB. */
7094 gsi
= gsi_last_bb (iter_part_bb
);
7096 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7097 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7098 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7099 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7100 false, GSI_CONTINUE_LINKING
);
7102 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7103 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7104 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7105 false, GSI_CONTINUE_LINKING
);
7107 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7108 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7110 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7111 gsi
= gsi_start_bb (seq_start_bb
);
7113 tree startvar
= fd
->loop
.v
;
7114 tree endvar
= NULL_TREE
;
7116 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7118 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7119 ? gimple_omp_parallel_clauses (inner_stmt
)
7120 : gimple_omp_for_clauses (inner_stmt
);
7121 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7122 gcc_assert (innerc
);
7123 startvar
= OMP_CLAUSE_DECL (innerc
);
7124 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7125 OMP_CLAUSE__LOOPTEMP_
);
7126 gcc_assert (innerc
);
7127 endvar
= OMP_CLAUSE_DECL (innerc
);
7130 t
= fold_convert (itype
, s0
);
7131 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7132 if (POINTER_TYPE_P (type
))
7133 t
= fold_build_pointer_plus (n1
, t
);
7135 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7136 t
= fold_convert (TREE_TYPE (startvar
), t
);
7137 t
= force_gimple_operand_gsi (&gsi
, t
,
7139 && TREE_ADDRESSABLE (startvar
),
7140 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7141 stmt
= gimple_build_assign (startvar
, t
);
7142 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7144 t
= fold_convert (itype
, e0
);
7145 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7146 if (POINTER_TYPE_P (type
))
7147 t
= fold_build_pointer_plus (n1
, t
);
7149 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7150 t
= fold_convert (TREE_TYPE (startvar
), t
);
7151 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7152 false, GSI_CONTINUE_LINKING
);
7155 stmt
= gimple_build_assign (endvar
, e
);
7156 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7157 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7158 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7160 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
7162 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7164 if (fd
->collapse
> 1)
7165 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7169 /* The code controlling the sequential loop goes in CONT_BB,
7170 replacing the GIMPLE_OMP_CONTINUE. */
7171 gsi
= gsi_last_bb (cont_bb
);
7172 stmt
= gsi_stmt (gsi
);
7173 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7174 vmain
= gimple_omp_continue_control_use (stmt
);
7175 vback
= gimple_omp_continue_control_def (stmt
);
7177 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7179 if (POINTER_TYPE_P (type
))
7180 t
= fold_build_pointer_plus (vmain
, step
);
7182 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7183 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7184 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7185 true, GSI_SAME_STMT
);
7186 stmt
= gimple_build_assign (vback
, t
);
7187 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7189 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7190 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7192 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7195 /* Remove GIMPLE_OMP_CONTINUE. */
7196 gsi_remove (&gsi
, true);
7198 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7199 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7201 /* Trip update code goes into TRIP_UPDATE_BB. */
7202 gsi
= gsi_start_bb (trip_update_bb
);
7204 t
= build_int_cst (itype
, 1);
7205 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7206 stmt
= gimple_build_assign (trip_back
, t
);
7207 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7210 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7211 gsi
= gsi_last_bb (exit_bb
);
7212 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7214 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
7215 != GF_OMP_FOR_KIND_OACC_LOOP
);
7217 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7218 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7220 gsi_remove (&gsi
, true);
7222 /* Connect the new blocks. */
7223 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7224 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7228 se
= find_edge (cont_bb
, body_bb
);
7229 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7231 gcc_assert (gimple_omp_for_kind (fd
->for_stmt
)
7232 != GF_OMP_FOR_KIND_OACC_LOOP
);
7237 else if (fd
->collapse
> 1)
7240 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7243 se
->flags
= EDGE_TRUE_VALUE
;
7244 find_edge (cont_bb
, trip_update_bb
)->flags
7245 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7247 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7250 if (gimple_in_ssa_p (cfun
))
7252 gimple_stmt_iterator psi
;
7258 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7260 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7261 remove arguments of the phi nodes in fin_bb. We need to create
7262 appropriate phi nodes in iter_part_bb instead. */
7263 se
= single_pred_edge (fin_bb
);
7264 re
= single_succ_edge (trip_update_bb
);
7265 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7266 ene
= single_succ_edge (entry_bb
);
7268 psi
= gsi_start_phis (fin_bb
);
7269 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7270 gsi_next (&psi
), ++i
)
7273 source_location locus
;
7275 phi
= gsi_stmt (psi
);
7276 t
= gimple_phi_result (phi
);
7277 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7278 nphi
= create_phi_node (t
, iter_part_bb
);
7280 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7281 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7283 /* A special case -- fd->loop.v is not yet computed in
7284 iter_part_bb, we need to use vextra instead. */
7285 if (t
== fd
->loop
.v
)
7287 add_phi_arg (nphi
, t
, ene
, locus
);
7288 locus
= redirect_edge_var_map_location (vm
);
7289 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7291 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
7292 redirect_edge_var_map_clear (re
);
7295 psi
= gsi_start_phis (fin_bb
);
7296 if (gsi_end_p (psi
))
7298 remove_phi_node (&psi
, false);
7301 /* Make phi node for trip. */
7302 phi
= create_phi_node (trip_main
, iter_part_bb
);
7303 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7305 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7310 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7311 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7312 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7313 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7314 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7315 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7316 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7317 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7318 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7322 struct loop
*trip_loop
= alloc_loop ();
7323 trip_loop
->header
= iter_part_bb
;
7324 trip_loop
->latch
= trip_update_bb
;
7325 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7327 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7329 struct loop
*loop
= alloc_loop ();
7330 loop
->header
= body_bb
;
7331 if (collapse_bb
== NULL
)
7332 loop
->latch
= cont_bb
;
7333 add_loop (loop
, trip_loop
);
7338 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7340 for (V = N1; V cond N2; V += STEP) BODY;
7342 where COND is "<" or ">" or "!=", we generate pseudocode
7344 for (ind_var = low; ind_var < high; ind_var++)
7346 V = n1 + (ind_var * STEP)
7351 In the above pseudocode, low and high are function parameters of the
7352 child function. In the function below, we are inserting a temp.
7353 variable that will be making a call to two OMP functions that will not be
7354 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7355 with _Cilk_for). These functions are replaced with low and high
7356 by the function that handles taskreg. */
7360 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7362 bool broken_loop
= region
->cont
== NULL
;
7363 basic_block entry_bb
= region
->entry
;
7364 basic_block cont_bb
= region
->cont
;
7366 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7367 gcc_assert (broken_loop
7368 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7369 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7370 basic_block l1_bb
, l2_bb
;
7374 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7375 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7376 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7377 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7381 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7382 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7383 l2_bb
= single_succ (l1_bb
);
7385 basic_block exit_bb
= region
->exit
;
7386 basic_block l2_dom_bb
= NULL
;
7388 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7390 /* Below statements until the "tree high_val = ..." are pseudo statements
7391 used to pass information to be used by expand_omp_taskreg.
7392 low_val and high_val will be replaced by the __low and __high
7393 parameter from the child function.
7395 The call_exprs part is a place-holder, it is mainly used
7396 to distinctly identify to the top-level part that this is
7397 where we should put low and high (reasoning given in header
7401 = gimple_omp_parallel_child_fn (last_stmt (region
->outer
->entry
));
7402 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7403 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7405 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7407 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7410 gcc_assert (low_val
&& high_val
);
7412 tree type
= TREE_TYPE (low_val
);
7413 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7414 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7416 /* Not needed in SSA form right now. */
7417 gcc_assert (!gimple_in_ssa_p (cfun
));
7418 if (l2_dom_bb
== NULL
)
7424 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7426 /* Replace the GIMPLE_OMP_FOR statement. */
7427 gsi_replace (&gsi
, stmt
, true);
7431 /* Code to control the increment goes in the CONT_BB. */
7432 gsi
= gsi_last_bb (cont_bb
);
7433 stmt
= gsi_stmt (gsi
);
7434 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7435 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ind_var
, ind_var
,
7436 build_one_cst (type
));
7438 /* Replace GIMPLE_OMP_CONTINUE. */
7439 gsi_replace (&gsi
, stmt
, true);
7442 /* Emit the condition in L1_BB. */
7443 gsi
= gsi_after_labels (l1_bb
);
7444 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7445 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7447 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7448 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7449 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7451 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7452 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7453 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7454 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7456 /* The condition is always '<' since the runtime will fill in the low
7458 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7459 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7461 /* Remove GIMPLE_OMP_RETURN. */
7462 gsi
= gsi_last_bb (exit_bb
);
7463 gsi_remove (&gsi
, true);
7465 /* Connect the new blocks. */
7466 remove_edge (FALLTHRU_EDGE (entry_bb
));
7471 remove_edge (BRANCH_EDGE (entry_bb
));
7472 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7474 e
= BRANCH_EDGE (l1_bb
);
7475 ne
= FALLTHRU_EDGE (l1_bb
);
7476 e
->flags
= EDGE_TRUE_VALUE
;
7480 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7482 ne
= single_succ_edge (l1_bb
);
7483 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7486 ne
->flags
= EDGE_FALSE_VALUE
;
7487 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7488 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7490 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7491 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7492 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7496 struct loop
*loop
= alloc_loop ();
7497 loop
->header
= l1_bb
;
7498 loop
->latch
= cont_bb
;
7499 add_loop (loop
, l1_bb
->loop_father
);
7500 loop
->safelen
= INT_MAX
;
7503 /* Pick the correct library function based on the precision of the
7504 induction variable type. */
7505 tree lib_fun
= NULL_TREE
;
7506 if (TYPE_PRECISION (type
) == 32)
7507 lib_fun
= cilk_for_32_fndecl
;
7508 else if (TYPE_PRECISION (type
) == 64)
7509 lib_fun
= cilk_for_64_fndecl
;
7513 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7515 /* WS_ARGS contains the library function flavor to call:
7516 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7517 user-defined grain value. If the user does not define one, then zero
7518 is passed in by the parser. */
7519 vec_alloc (region
->ws_args
, 2);
7520 region
->ws_args
->quick_push (lib_fun
);
7521 region
->ws_args
->quick_push (fd
->chunk_size
);
7524 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7525 loop. Given parameters:
7527 for (V = N1; V cond N2; V += STEP) BODY;
7529 where COND is "<" or ">", we generate pseudocode
7537 if (V cond N2) goto L0; else goto L2;
7540 For collapsed loops, given parameters:
7542 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7543 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7544 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7547 we generate pseudocode
7553 count3 = (adj + N32 - N31) / STEP3;
7558 count2 = (adj + N22 - N21) / STEP2;
7563 count1 = (adj + N12 - N11) / STEP1;
7564 count = count1 * count2 * count3;
7574 V2 += (V3 cond3 N32) ? 0 : STEP2;
7575 V3 = (V3 cond3 N32) ? V3 : N31;
7576 V1 += (V2 cond2 N22) ? 0 : STEP1;
7577 V2 = (V2 cond2 N22) ? V2 : N21;
7579 if (V < count) goto L0; else goto L2;
7585 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7588 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7589 gimple_stmt_iterator gsi
;
7591 bool broken_loop
= region
->cont
== NULL
;
7593 tree
*counts
= NULL
;
7595 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7596 OMP_CLAUSE_SAFELEN
);
7597 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7598 OMP_CLAUSE__SIMDUID_
);
7601 type
= TREE_TYPE (fd
->loop
.v
);
7602 entry_bb
= region
->entry
;
7603 cont_bb
= region
->cont
;
7604 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7605 gcc_assert (broken_loop
7606 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7607 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7610 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7611 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7612 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7613 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7617 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7618 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7619 l2_bb
= single_succ (l1_bb
);
7621 exit_bb
= region
->exit
;
7624 gsi
= gsi_last_bb (entry_bb
);
7626 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7627 /* Not needed in SSA form right now. */
7628 gcc_assert (!gimple_in_ssa_p (cfun
));
7629 if (fd
->collapse
> 1)
7631 int first_zero_iter
= -1;
7632 basic_block zero_iter_bb
= l2_bb
;
7634 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7635 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7636 zero_iter_bb
, first_zero_iter
,
7639 if (l2_dom_bb
== NULL
)
7644 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7646 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7647 OMP_CLAUSE__LOOPTEMP_
);
7648 gcc_assert (innerc
);
7649 n1
= OMP_CLAUSE_DECL (innerc
);
7650 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7651 OMP_CLAUSE__LOOPTEMP_
);
7652 gcc_assert (innerc
);
7653 n2
= OMP_CLAUSE_DECL (innerc
);
7654 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7655 fold_convert (type
, n1
));
7656 if (fd
->collapse
> 1)
7659 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7665 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7666 fold_convert (type
, fd
->loop
.n1
));
7667 if (fd
->collapse
> 1)
7668 for (i
= 0; i
< fd
->collapse
; i
++)
7670 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7671 if (POINTER_TYPE_P (itype
))
7672 itype
= signed_type_for (itype
);
7673 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7674 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7678 /* Remove the GIMPLE_OMP_FOR statement. */
7679 gsi_remove (&gsi
, true);
7683 /* Code to control the increment goes in the CONT_BB. */
7684 gsi
= gsi_last_bb (cont_bb
);
7685 stmt
= gsi_stmt (gsi
);
7686 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7688 if (POINTER_TYPE_P (type
))
7689 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7691 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7692 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7694 if (fd
->collapse
> 1)
7696 i
= fd
->collapse
- 1;
7697 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7699 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7700 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7704 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7706 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7709 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7711 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7713 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7714 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7715 if (POINTER_TYPE_P (itype2
))
7716 itype2
= signed_type_for (itype2
);
7717 t
= build3 (COND_EXPR
, itype2
,
7718 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7720 fold_convert (itype
, fd
->loops
[i
].n2
)),
7721 build_int_cst (itype2
, 0),
7722 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7723 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7724 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7726 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7727 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7729 t
= build3 (COND_EXPR
, itype
,
7730 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7732 fold_convert (itype
, fd
->loops
[i
].n2
)),
7734 fold_convert (itype
, fd
->loops
[i
].n1
));
7735 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7739 /* Remove GIMPLE_OMP_CONTINUE. */
7740 gsi_remove (&gsi
, true);
7743 /* Emit the condition in L1_BB. */
7744 gsi
= gsi_start_bb (l1_bb
);
7746 t
= fold_convert (type
, n2
);
7747 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7748 false, GSI_CONTINUE_LINKING
);
7749 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7750 stmt
= gimple_build_cond_empty (t
);
7751 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7752 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
7754 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
7757 gsi
= gsi_for_stmt (stmt
);
7758 gimple_regimplify_operands (stmt
, &gsi
);
7761 /* Remove GIMPLE_OMP_RETURN. */
7762 gsi
= gsi_last_bb (exit_bb
);
7763 gsi_remove (&gsi
, true);
7765 /* Connect the new blocks. */
7766 remove_edge (FALLTHRU_EDGE (entry_bb
));
7770 remove_edge (BRANCH_EDGE (entry_bb
));
7771 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7773 e
= BRANCH_EDGE (l1_bb
);
7774 ne
= FALLTHRU_EDGE (l1_bb
);
7775 e
->flags
= EDGE_TRUE_VALUE
;
7779 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7781 ne
= single_succ_edge (l1_bb
);
7782 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7785 ne
->flags
= EDGE_FALSE_VALUE
;
7786 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7787 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7789 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7790 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7791 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7795 struct loop
*loop
= alloc_loop ();
7796 loop
->header
= l1_bb
;
7797 loop
->latch
= cont_bb
;
7798 add_loop (loop
, l1_bb
->loop_father
);
7799 if (safelen
== NULL_TREE
)
7800 loop
->safelen
= INT_MAX
;
7803 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7804 if (TREE_CODE (safelen
) != INTEGER_CST
)
7806 else if (!tree_fits_uhwi_p (safelen
)
7807 || tree_to_uhwi (safelen
) > INT_MAX
)
7808 loop
->safelen
= INT_MAX
;
7810 loop
->safelen
= tree_to_uhwi (safelen
);
7811 if (loop
->safelen
== 1)
7816 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7817 cfun
->has_simduid_loops
= true;
7819 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7821 if ((flag_tree_loop_vectorize
7822 || (!global_options_set
.x_flag_tree_loop_vectorize
7823 && !global_options_set
.x_flag_tree_vectorize
))
7824 && flag_tree_loop_optimize
7825 && loop
->safelen
> 1)
7827 loop
->force_vectorize
= true;
7828 cfun
->has_force_vectorize_loops
= true;
7834 /* Expand the OpenMP loop defined by REGION. */
7837 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7839 struct omp_for_data fd
;
7840 struct omp_for_data_loop
*loops
;
7843 = (struct omp_for_data_loop
*)
7844 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7845 * sizeof (struct omp_for_data_loop
));
7846 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7847 region
->sched_kind
= fd
.sched_kind
;
7849 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7850 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7851 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7854 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7855 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7856 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7859 /* If there isn't a continue then this is a degerate case where
7860 the introduction of abnormal edges during lowering will prevent
7861 original loops from being detected. Fix that up. */
7862 loops_state_set (LOOPS_NEED_FIXUP
);
7864 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7865 expand_omp_simd (region
, &fd
);
7866 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7867 expand_cilk_for (region
, &fd
);
7868 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7869 && !fd
.have_ordered
)
7871 if (fd
.chunk_size
== NULL
)
7872 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7874 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7878 int fn_index
, start_ix
, next_ix
;
7880 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7881 == GF_OMP_FOR_KIND_FOR
);
7882 if (fd
.chunk_size
== NULL
7883 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7884 fd
.chunk_size
= integer_zero_node
;
7885 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7886 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7887 ? 3 : fd
.sched_kind
;
7888 fn_index
+= fd
.have_ordered
* 4;
7889 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7890 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7891 if (fd
.iter_type
== long_long_unsigned_type_node
)
7893 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7894 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7895 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7896 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7898 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7899 (enum built_in_function
) next_ix
, inner_stmt
);
7902 if (gimple_in_ssa_p (cfun
))
7903 update_ssa (TODO_update_ssa_only_virtuals
);
7907 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7909 v = GOMP_sections_start (n);
7926 v = GOMP_sections_next ();
7931 If this is a combined parallel sections, replace the call to
7932 GOMP_sections_start with call to GOMP_sections_next. */
7935 expand_omp_sections (struct omp_region
*region
)
7937 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7939 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7940 gimple_stmt_iterator si
, switch_si
;
7941 gimple sections_stmt
, stmt
, cont
;
7944 struct omp_region
*inner
;
7946 bool exit_reachable
= region
->cont
!= NULL
;
7948 gcc_assert (region
->exit
!= NULL
);
7949 entry_bb
= region
->entry
;
7950 l0_bb
= single_succ (entry_bb
);
7951 l1_bb
= region
->cont
;
7952 l2_bb
= region
->exit
;
7953 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7954 l2
= gimple_block_label (l2_bb
);
7957 /* This can happen if there are reductions. */
7958 len
= EDGE_COUNT (l0_bb
->succs
);
7959 gcc_assert (len
> 0);
7960 e
= EDGE_SUCC (l0_bb
, len
- 1);
7961 si
= gsi_last_bb (e
->dest
);
7964 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7965 l2
= gimple_block_label (e
->dest
);
7967 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7969 si
= gsi_last_bb (e
->dest
);
7971 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7973 l2
= gimple_block_label (e
->dest
);
7979 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7981 default_bb
= create_empty_bb (l0_bb
);
7983 /* We will build a switch() with enough cases for all the
7984 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7985 and a default case to abort if something goes wrong. */
7986 len
= EDGE_COUNT (l0_bb
->succs
);
7988 /* Use vec::quick_push on label_vec throughout, since we know the size
7990 auto_vec
<tree
> label_vec (len
);
7992 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7993 GIMPLE_OMP_SECTIONS statement. */
7994 si
= gsi_last_bb (entry_bb
);
7995 sections_stmt
= gsi_stmt (si
);
7996 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7997 vin
= gimple_omp_sections_control (sections_stmt
);
7998 if (!is_combined_parallel (region
))
8000 /* If we are not inside a combined parallel+sections region,
8001 call GOMP_sections_start. */
8002 t
= build_int_cst (unsigned_type_node
, len
- 1);
8003 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
8004 stmt
= gimple_build_call (u
, 1, t
);
8008 /* Otherwise, call GOMP_sections_next. */
8009 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8010 stmt
= gimple_build_call (u
, 0);
8012 gimple_call_set_lhs (stmt
, vin
);
8013 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8014 gsi_remove (&si
, true);
8016 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8018 switch_si
= gsi_last_bb (l0_bb
);
8019 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8022 cont
= last_stmt (l1_bb
);
8023 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8024 vmain
= gimple_omp_continue_control_use (cont
);
8025 vnext
= gimple_omp_continue_control_def (cont
);
8033 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8034 label_vec
.quick_push (t
);
8037 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8038 for (inner
= region
->inner
, casei
= 1;
8040 inner
= inner
->next
, i
++, casei
++)
8042 basic_block s_entry_bb
, s_exit_bb
;
8044 /* Skip optional reduction region. */
8045 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8052 s_entry_bb
= inner
->entry
;
8053 s_exit_bb
= inner
->exit
;
8055 t
= gimple_block_label (s_entry_bb
);
8056 u
= build_int_cst (unsigned_type_node
, casei
);
8057 u
= build_case_label (u
, NULL
, t
);
8058 label_vec
.quick_push (u
);
8060 si
= gsi_last_bb (s_entry_bb
);
8061 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8062 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8063 gsi_remove (&si
, true);
8064 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8066 if (s_exit_bb
== NULL
)
8069 si
= gsi_last_bb (s_exit_bb
);
8070 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8071 gsi_remove (&si
, true);
8073 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8076 /* Error handling code goes in DEFAULT_BB. */
8077 t
= gimple_block_label (default_bb
);
8078 u
= build_case_label (NULL
, NULL
, t
);
8079 make_edge (l0_bb
, default_bb
, 0);
8080 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8082 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8083 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8084 gsi_remove (&switch_si
, true);
8086 si
= gsi_start_bb (default_bb
);
8087 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8088 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8094 /* Code to get the next section goes in L1_BB. */
8095 si
= gsi_last_bb (l1_bb
);
8096 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8098 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8099 stmt
= gimple_build_call (bfn_decl
, 0);
8100 gimple_call_set_lhs (stmt
, vnext
);
8101 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8102 gsi_remove (&si
, true);
8104 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8107 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8108 si
= gsi_last_bb (l2_bb
);
8109 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8110 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8111 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8112 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8114 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8115 stmt
= gimple_build_call (t
, 0);
8116 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8117 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8118 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8119 gsi_remove (&si
, true);
8121 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8125 /* Expand code for an OpenMP single directive. We've already expanded
8126 much of the code, here we simply place the GOMP_barrier call. */
8129 expand_omp_single (struct omp_region
*region
)
8131 basic_block entry_bb
, exit_bb
;
8132 gimple_stmt_iterator si
;
8134 entry_bb
= region
->entry
;
8135 exit_bb
= region
->exit
;
8137 si
= gsi_last_bb (entry_bb
);
8138 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8139 gsi_remove (&si
, true);
8140 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8142 si
= gsi_last_bb (exit_bb
);
8143 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8145 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8146 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8148 gsi_remove (&si
, true);
8149 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8153 /* Generic expansion for OpenMP synchronization directives: master,
8154 ordered and critical. All we need to do here is remove the entry
8155 and exit markers for REGION. */
8158 expand_omp_synch (struct omp_region
*region
)
8160 basic_block entry_bb
, exit_bb
;
8161 gimple_stmt_iterator si
;
8163 entry_bb
= region
->entry
;
8164 exit_bb
= region
->exit
;
8166 si
= gsi_last_bb (entry_bb
);
8167 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8168 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8169 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8170 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8171 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8172 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8173 gsi_remove (&si
, true);
8174 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8178 si
= gsi_last_bb (exit_bb
);
8179 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8180 gsi_remove (&si
, true);
8181 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8185 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8186 operation as a normal volatile load. */
8189 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8190 tree loaded_val
, int index
)
8192 enum built_in_function tmpbase
;
8193 gimple_stmt_iterator gsi
;
8194 basic_block store_bb
;
8197 tree decl
, call
, type
, itype
;
8199 gsi
= gsi_last_bb (load_bb
);
8200 stmt
= gsi_stmt (gsi
);
8201 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8202 loc
= gimple_location (stmt
);
8204 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8205 is smaller than word size, then expand_atomic_load assumes that the load
8206 is atomic. We could avoid the builtin entirely in this case. */
8208 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8209 decl
= builtin_decl_explicit (tmpbase
);
8210 if (decl
== NULL_TREE
)
8213 type
= TREE_TYPE (loaded_val
);
8214 itype
= TREE_TYPE (TREE_TYPE (decl
));
8216 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8217 build_int_cst (NULL
,
8218 gimple_omp_atomic_seq_cst_p (stmt
)
8220 : MEMMODEL_RELAXED
));
8221 if (!useless_type_conversion_p (type
, itype
))
8222 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8223 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8225 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8226 gsi_remove (&gsi
, true);
8228 store_bb
= single_succ (load_bb
);
8229 gsi
= gsi_last_bb (store_bb
);
8230 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8231 gsi_remove (&gsi
, true);
8233 if (gimple_in_ssa_p (cfun
))
8234 update_ssa (TODO_update_ssa_no_phi
);
8239 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8240 operation as a normal volatile store. */
8243 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8244 tree loaded_val
, tree stored_val
, int index
)
8246 enum built_in_function tmpbase
;
8247 gimple_stmt_iterator gsi
;
8248 basic_block store_bb
= single_succ (load_bb
);
8251 tree decl
, call
, type
, itype
;
8255 gsi
= gsi_last_bb (load_bb
);
8256 stmt
= gsi_stmt (gsi
);
8257 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8259 /* If the load value is needed, then this isn't a store but an exchange. */
8260 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8262 gsi
= gsi_last_bb (store_bb
);
8263 stmt
= gsi_stmt (gsi
);
8264 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8265 loc
= gimple_location (stmt
);
8267 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8268 is smaller than word size, then expand_atomic_store assumes that the store
8269 is atomic. We could avoid the builtin entirely in this case. */
8271 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8272 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8273 decl
= builtin_decl_explicit (tmpbase
);
8274 if (decl
== NULL_TREE
)
8277 type
= TREE_TYPE (stored_val
);
8279 /* Dig out the type of the function's second argument. */
8280 itype
= TREE_TYPE (decl
);
8281 itype
= TYPE_ARG_TYPES (itype
);
8282 itype
= TREE_CHAIN (itype
);
8283 itype
= TREE_VALUE (itype
);
8284 imode
= TYPE_MODE (itype
);
8286 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8289 if (!useless_type_conversion_p (itype
, type
))
8290 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8291 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8292 build_int_cst (NULL
,
8293 gimple_omp_atomic_seq_cst_p (stmt
)
8295 : MEMMODEL_RELAXED
));
8298 if (!useless_type_conversion_p (type
, itype
))
8299 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8300 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8303 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8304 gsi_remove (&gsi
, true);
8306 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8307 gsi
= gsi_last_bb (load_bb
);
8308 gsi_remove (&gsi
, true);
8310 if (gimple_in_ssa_p (cfun
))
8311 update_ssa (TODO_update_ssa_no_phi
);
8316 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8317 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8318 size of the data type, and thus usable to find the index of the builtin
8319 decl. Returns false if the expression is not of the proper form. */
8322 expand_omp_atomic_fetch_op (basic_block load_bb
,
8323 tree addr
, tree loaded_val
,
8324 tree stored_val
, int index
)
8326 enum built_in_function oldbase
, newbase
, tmpbase
;
8327 tree decl
, itype
, call
;
8329 basic_block store_bb
= single_succ (load_bb
);
8330 gimple_stmt_iterator gsi
;
8333 enum tree_code code
;
8334 bool need_old
, need_new
;
8338 /* We expect to find the following sequences:
8341 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8344 val = tmp OP something; (or: something OP tmp)
8345 GIMPLE_OMP_STORE (val)
8347 ???FIXME: Allow a more flexible sequence.
8348 Perhaps use data flow to pick the statements.
8352 gsi
= gsi_after_labels (store_bb
);
8353 stmt
= gsi_stmt (gsi
);
8354 loc
= gimple_location (stmt
);
8355 if (!is_gimple_assign (stmt
))
8358 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8360 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8361 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8362 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8363 gcc_checking_assert (!need_old
|| !need_new
);
8365 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8368 /* Check for one of the supported fetch-op operations. */
8369 code
= gimple_assign_rhs_code (stmt
);
8373 case POINTER_PLUS_EXPR
:
8374 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8375 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8378 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8379 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8382 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8383 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8386 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8387 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8390 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8391 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8397 /* Make sure the expression is of the proper form. */
8398 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8399 rhs
= gimple_assign_rhs2 (stmt
);
8400 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8401 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8402 rhs
= gimple_assign_rhs1 (stmt
);
8406 tmpbase
= ((enum built_in_function
)
8407 ((need_new
? newbase
: oldbase
) + index
+ 1));
8408 decl
= builtin_decl_explicit (tmpbase
);
8409 if (decl
== NULL_TREE
)
8411 itype
= TREE_TYPE (TREE_TYPE (decl
));
8412 imode
= TYPE_MODE (itype
);
8414 /* We could test all of the various optabs involved, but the fact of the
8415 matter is that (with the exception of i486 vs i586 and xadd) all targets
8416 that support any atomic operaton optab also implements compare-and-swap.
8417 Let optabs.c take care of expanding any compare-and-swap loop. */
8418 if (!can_compare_and_swap_p (imode
, true))
8421 gsi
= gsi_last_bb (load_bb
);
8422 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8424 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8425 It only requires that the operation happen atomically. Thus we can
8426 use the RELAXED memory model. */
8427 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8428 fold_convert_loc (loc
, itype
, rhs
),
8429 build_int_cst (NULL
,
8430 seq_cst
? MEMMODEL_SEQ_CST
8431 : MEMMODEL_RELAXED
));
8433 if (need_old
|| need_new
)
8435 lhs
= need_old
? loaded_val
: stored_val
;
8436 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8437 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8440 call
= fold_convert_loc (loc
, void_type_node
, call
);
8441 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8442 gsi_remove (&gsi
, true);
8444 gsi
= gsi_last_bb (store_bb
);
8445 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8446 gsi_remove (&gsi
, true);
8447 gsi
= gsi_last_bb (store_bb
);
8448 gsi_remove (&gsi
, true);
8450 if (gimple_in_ssa_p (cfun
))
8451 update_ssa (TODO_update_ssa_no_phi
);
8456 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8460 newval = rhs; // with oldval replacing *addr in rhs
8461 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8462 if (oldval != newval)
8465 INDEX is log2 of the size of the data type, and thus usable to find the
8466 index of the builtin decl. */
8469 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8470 tree addr
, tree loaded_val
, tree stored_val
,
8473 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8474 tree type
, itype
, cmpxchg
, iaddr
;
8475 gimple_stmt_iterator si
;
8476 basic_block loop_header
= single_succ (load_bb
);
8479 enum built_in_function fncode
;
8481 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8482 order to use the RELAXED memory model effectively. */
8483 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8485 cmpxchg
= builtin_decl_explicit (fncode
);
8486 if (cmpxchg
== NULL_TREE
)
8488 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8489 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8491 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8494 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8495 si
= gsi_last_bb (load_bb
);
8496 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8498 /* For floating-point values, we'll need to view-convert them to integers
8499 so that we can perform the atomic compare and swap. Simplify the
8500 following code by always setting up the "i"ntegral variables. */
8501 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8505 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8508 = force_gimple_operand_gsi (&si
,
8509 fold_convert (TREE_TYPE (iaddr
), addr
),
8510 false, NULL_TREE
, true, GSI_SAME_STMT
);
8511 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8512 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8513 loadedi
= create_tmp_var (itype
, NULL
);
8514 if (gimple_in_ssa_p (cfun
))
8515 loadedi
= make_ssa_name (loadedi
, NULL
);
8520 loadedi
= loaded_val
;
8523 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8524 tree loaddecl
= builtin_decl_explicit (fncode
);
8527 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8528 build_call_expr (loaddecl
, 2, iaddr
,
8529 build_int_cst (NULL_TREE
,
8530 MEMMODEL_RELAXED
)));
8532 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8533 build_int_cst (TREE_TYPE (iaddr
), 0));
8536 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8539 /* Move the value to the LOADEDI temporary. */
8540 if (gimple_in_ssa_p (cfun
))
8542 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8543 phi
= create_phi_node (loadedi
, loop_header
);
8544 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8548 gsi_insert_before (&si
,
8549 gimple_build_assign (loadedi
, initial
),
8551 if (loadedi
!= loaded_val
)
8553 gimple_stmt_iterator gsi2
;
8556 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8557 gsi2
= gsi_start_bb (loop_header
);
8558 if (gimple_in_ssa_p (cfun
))
8561 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8562 true, GSI_SAME_STMT
);
8563 stmt
= gimple_build_assign (loaded_val
, x
);
8564 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8568 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8569 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8570 true, GSI_SAME_STMT
);
8573 gsi_remove (&si
, true);
8575 si
= gsi_last_bb (store_bb
);
8576 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8579 storedi
= stored_val
;
8582 force_gimple_operand_gsi (&si
,
8583 build1 (VIEW_CONVERT_EXPR
, itype
,
8584 stored_val
), true, NULL_TREE
, true,
8587 /* Build the compare&swap statement. */
8588 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8589 new_storedi
= force_gimple_operand_gsi (&si
,
8590 fold_convert (TREE_TYPE (loadedi
),
8593 true, GSI_SAME_STMT
);
8595 if (gimple_in_ssa_p (cfun
))
8599 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
8600 stmt
= gimple_build_assign (old_vali
, loadedi
);
8601 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8603 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8604 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8607 /* Note that we always perform the comparison as an integer, even for
8608 floating point. This allows the atomic operation to properly
8609 succeed even with NaNs and -0.0. */
8610 stmt
= gimple_build_cond_empty
8611 (build2 (NE_EXPR
, boolean_type_node
,
8612 new_storedi
, old_vali
));
8613 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8616 e
= single_succ_edge (store_bb
);
8617 e
->flags
&= ~EDGE_FALLTHRU
;
8618 e
->flags
|= EDGE_FALSE_VALUE
;
8620 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8622 /* Copy the new value to loadedi (we already did that before the condition
8623 if we are not in SSA). */
8624 if (gimple_in_ssa_p (cfun
))
8626 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8627 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8630 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8631 gsi_remove (&si
, true);
8633 struct loop
*loop
= alloc_loop ();
8634 loop
->header
= loop_header
;
8635 loop
->latch
= store_bb
;
8636 add_loop (loop
, loop_header
->loop_father
);
8638 if (gimple_in_ssa_p (cfun
))
8639 update_ssa (TODO_update_ssa_no_phi
);
8644 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8646 GOMP_atomic_start ();
8650 The result is not globally atomic, but works so long as all parallel
8651 references are within #pragma omp atomic directives. According to
8652 responses received from omp@openmp.org, appears to be within spec.
8653 Which makes sense, since that's how several other compilers handle
8654 this situation as well.
8655 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8656 expanding. STORED_VAL is the operand of the matching
8657 GIMPLE_OMP_ATOMIC_STORE.
8660 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8664 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8669 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8670 tree addr
, tree loaded_val
, tree stored_val
)
8672 gimple_stmt_iterator si
;
8676 si
= gsi_last_bb (load_bb
);
8677 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8679 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8680 t
= build_call_expr (t
, 0);
8681 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8683 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8684 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8685 gsi_remove (&si
, true);
8687 si
= gsi_last_bb (store_bb
);
8688 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8690 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8692 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8694 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8695 t
= build_call_expr (t
, 0);
8696 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8697 gsi_remove (&si
, true);
8699 if (gimple_in_ssa_p (cfun
))
8700 update_ssa (TODO_update_ssa_no_phi
);
8704 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8705 using expand_omp_atomic_fetch_op. If it failed, we try to
8706 call expand_omp_atomic_pipeline, and if it fails too, the
8707 ultimate fallback is wrapping the operation in a mutex
8708 (expand_omp_atomic_mutex). REGION is the atomic region built
8709 by build_omp_regions_1(). */
8712 expand_omp_atomic (struct omp_region
*region
)
8714 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8715 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
8716 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8717 tree addr
= gimple_omp_atomic_load_rhs (load
);
8718 tree stored_val
= gimple_omp_atomic_store_val (store
);
8719 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8720 HOST_WIDE_INT index
;
8722 /* Make sure the type is one of the supported sizes. */
8723 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8724 index
= exact_log2 (index
);
8725 if (index
>= 0 && index
<= 4)
8727 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8729 /* __sync builtins require strict data alignment. */
8730 if (exact_log2 (align
) >= index
)
8733 if (loaded_val
== stored_val
8734 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8735 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8736 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8737 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8741 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8742 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8743 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8744 && store_bb
== single_succ (load_bb
)
8745 && first_stmt (store_bb
) == store
8746 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8750 /* When possible, use specialized atomic update functions. */
8751 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8752 && store_bb
== single_succ (load_bb
)
8753 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8754 loaded_val
, stored_val
, index
))
8757 /* If we don't have specialized __sync builtins, try and implement
8758 as a compare and swap loop. */
8759 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8760 loaded_val
, stored_val
, index
))
8765 /* The ultimate fallback is wrapping the operation in a mutex. */
8766 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8770 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8773 expand_omp_target (struct omp_region
*region
)
8775 basic_block entry_bb
, exit_bb
, new_bb
;
8776 struct function
*child_cfun
;
8777 tree child_fn
, block
, t
;
8778 gimple_stmt_iterator gsi
;
8779 gimple entry_stmt
, stmt
;
8781 bool offloaded
, data_region
;
8783 entry_stmt
= last_stmt (region
->entry
);
8784 new_bb
= region
->entry
;
8786 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8787 switch (gimple_omp_target_kind (entry_stmt
))
8789 case GF_OMP_TARGET_KIND_REGION
:
8790 case GF_OMP_TARGET_KIND_UPDATE
:
8791 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8792 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8793 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8794 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8795 data_region
= false;
8797 case GF_OMP_TARGET_KIND_DATA
:
8798 case GF_OMP_TARGET_KIND_OACC_DATA
:
8805 child_fn
= NULL_TREE
;
8809 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8810 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8813 /* Supported by expand_omp_taskreg, but not here. */
8814 if (child_cfun
!= NULL
)
8815 gcc_assert (!child_cfun
->cfg
);
8816 gcc_assert (!gimple_in_ssa_p (cfun
));
8818 entry_bb
= region
->entry
;
8819 exit_bb
= region
->exit
;
8823 unsigned srcidx
, dstidx
, num
;
8824 struct cgraph_node
*node
;
8826 /* If the offloading region needs data sent from the parent
8827 function, then the very first statement (except possible
8828 tree profile counter updates) of the offloading body
8829 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8830 &.OMP_DATA_O is passed as an argument to the child function,
8831 we need to replace it with the argument as seen by the child
8834 In most cases, this will end up being the identity assignment
8835 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8836 a function call that has been inlined, the original PARM_DECL
8837 .OMP_DATA_I may have been converted into a different local
8838 variable. In which case, we need to keep the assignment. */
8839 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8842 basic_block entry_succ_bb
= single_succ (entry_bb
);
8843 gimple_stmt_iterator gsi
;
8845 gimple tgtcopy_stmt
= NULL
;
8846 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8848 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8850 gcc_assert (!gsi_end_p (gsi
));
8851 stmt
= gsi_stmt (gsi
);
8852 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8855 if (gimple_num_ops (stmt
) == 2)
8857 tree arg
= gimple_assign_rhs1 (stmt
);
8859 /* We're ignoring the subcode because we're
8860 effectively doing a STRIP_NOPS. */
8862 if (TREE_CODE (arg
) == ADDR_EXPR
8863 && TREE_OPERAND (arg
, 0) == sender
)
8865 tgtcopy_stmt
= stmt
;
8871 gcc_assert (tgtcopy_stmt
!= NULL
);
8872 arg
= DECL_ARGUMENTS (child_fn
);
8874 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8875 gsi_remove (&gsi
, true);
8878 /* Declare local variables needed in CHILD_CFUN. */
8879 block
= DECL_INITIAL (child_fn
);
8880 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8881 /* The gimplifier could record temporaries in the offloading block
8882 rather than in containing function's local_decls chain,
8883 which would mean cgraph missed finalizing them. Do it now. */
8884 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8885 if (TREE_CODE (t
) == VAR_DECL
8887 && !DECL_EXTERNAL (t
))
8888 varpool_node::finalize_decl (t
);
8889 DECL_SAVED_TREE (child_fn
) = NULL
;
8890 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8891 gimple_set_body (child_fn
, NULL
);
8892 TREE_USED (block
) = 1;
8894 /* Reset DECL_CONTEXT on function arguments. */
8895 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8896 DECL_CONTEXT (t
) = child_fn
;
8898 /* Split ENTRY_BB at GIMPLE_*,
8899 so that it can be moved to the child function. */
8900 gsi
= gsi_last_bb (entry_bb
);
8901 stmt
= gsi_stmt (gsi
);
8903 gimple_code (stmt
) == gimple_code (entry_stmt
));
8904 gsi_remove (&gsi
, true);
8905 e
= split_block (entry_bb
, stmt
);
8907 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8909 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8912 gsi
= gsi_last_bb (exit_bb
);
8913 gcc_assert (!gsi_end_p (gsi
)
8914 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8915 stmt
= gimple_build_return (NULL
);
8916 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8917 gsi_remove (&gsi
, true);
8920 /* Move the offloading region into CHILD_CFUN. */
8922 block
= gimple_block (entry_stmt
);
8924 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8926 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8927 /* When the OMP expansion process cannot guarantee an up-to-date
8928 loop tree arrange for the child function to fixup loops. */
8929 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8930 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8932 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8933 num
= vec_safe_length (child_cfun
->local_decls
);
8934 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8936 t
= (*child_cfun
->local_decls
)[srcidx
];
8937 if (DECL_CONTEXT (t
) == cfun
->decl
)
8939 if (srcidx
!= dstidx
)
8940 (*child_cfun
->local_decls
)[dstidx
] = t
;
8944 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8946 /* Inform the callgraph about the new function. */
8947 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8948 cgraph_node::add_new_function (child_fn
, true);
8950 /* Add the new function to the offload table. */
8951 vec_safe_push (offload_funcs
, child_fn
);
8953 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8954 fixed in a following pass. */
8955 push_cfun (child_cfun
);
8956 cgraph_edge::rebuild_edges ();
8958 /* Prevent IPA from removing child_fn as unreachable, since there are no
8959 refs from the parent function to child_fn in offload LTO mode. */
8960 node
= cgraph_node::get (child_fn
);
8961 node
->mark_force_output ();
8963 /* Some EH regions might become dead, see PR34608. If
8964 pass_cleanup_cfg isn't the first pass to happen with the
8965 new child, these dead EH edges might cause problems.
8966 Clean them up now. */
8967 if (flag_exceptions
)
8970 bool changed
= false;
8972 FOR_EACH_BB_FN (bb
, cfun
)
8973 changed
|= gimple_purge_dead_eh_edges (bb
);
8975 cleanup_tree_cfg ();
8980 /* Emit a library call to launch the offloading region, or do data
8982 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8983 enum built_in_function start_ix
;
8984 location_t clause_loc
;
8986 switch (gimple_omp_target_kind (entry_stmt
))
8988 case GF_OMP_TARGET_KIND_REGION
:
8989 start_ix
= BUILT_IN_GOMP_TARGET
;
8991 case GF_OMP_TARGET_KIND_DATA
:
8992 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8994 case GF_OMP_TARGET_KIND_UPDATE
:
8995 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8997 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8998 start_ix
= BUILT_IN_GOACC_PARALLEL
;
9000 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9001 start_ix
= BUILT_IN_GOACC_KERNELS
;
9003 case GF_OMP_TARGET_KIND_OACC_DATA
:
9004 start_ix
= BUILT_IN_GOACC_DATA_START
;
9006 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9007 start_ix
= BUILT_IN_GOACC_UPDATE
;
9009 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9010 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9016 clauses
= gimple_omp_target_clauses (entry_stmt
);
9018 /* By default, the value of DEVICE is -1 (let runtime library choose)
9019 and there is no conditional. */
9021 device
= build_int_cst (integer_type_node
, -1);
9023 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9025 cond
= OMP_CLAUSE_IF_EXPR (c
);
9027 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9030 /* Even if we pass it to all library function calls, it is currently only
9031 defined/used for the OpenMP target ones. */
9032 gcc_assert (start_ix
== BUILT_IN_GOMP_TARGET
9033 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9034 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9036 device
= OMP_CLAUSE_DEVICE_ID (c
);
9037 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9040 clause_loc
= gimple_location (entry_stmt
);
9042 /* Ensure 'device' is of the correct type. */
9043 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9045 /* If we found the clause 'if (cond)', build
9046 (cond ? device : -2). */
9049 cond
= gimple_boolify (cond
);
9051 basic_block cond_bb
, then_bb
, else_bb
;
9055 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
9058 e
= split_block (new_bb
, NULL
);
9062 gsi
= gsi_last_bb (new_bb
);
9064 e
= split_block (new_bb
, gsi_stmt (gsi
));
9070 then_bb
= create_empty_bb (cond_bb
);
9071 else_bb
= create_empty_bb (then_bb
);
9072 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9073 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9075 stmt
= gimple_build_cond_empty (cond
);
9076 gsi
= gsi_last_bb (cond_bb
);
9077 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9079 gsi
= gsi_start_bb (then_bb
);
9080 stmt
= gimple_build_assign (tmp_var
, device
);
9081 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9083 gsi
= gsi_start_bb (else_bb
);
9084 stmt
= gimple_build_assign (tmp_var
,
9085 build_int_cst (integer_type_node
, -2));
9086 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9088 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9089 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9090 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9091 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9092 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9093 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9098 gsi
= gsi_last_bb (new_bb
);
9099 t
= gimple_omp_target_data_arg (entry_stmt
);
9102 t1
= size_zero_node
;
9103 t2
= build_zero_cst (ptr_type_node
);
9109 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9110 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9111 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9112 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9113 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9117 tree offload_table
= get_offload_symbol_decl ();
9119 /* The maximum number used by any start_ix, without varargs. */
9120 unsigned int argcnt
= 12;
9122 vec_alloc (args
, argcnt
);
9123 args
->quick_push (device
);
9125 args
->quick_push (build_fold_addr_expr (child_fn
));
9126 args
->quick_push (build_fold_addr_expr (offload_table
));
9127 args
->quick_push (t1
);
9128 args
->quick_push (t2
);
9129 args
->quick_push (t3
);
9130 args
->quick_push (t4
);
9133 case BUILT_IN_GOACC_DATA_START
:
9134 case BUILT_IN_GOMP_TARGET
:
9135 case BUILT_IN_GOMP_TARGET_DATA
:
9136 case BUILT_IN_GOMP_TARGET_UPDATE
:
9138 case BUILT_IN_GOACC_KERNELS
:
9139 case BUILT_IN_GOACC_PARALLEL
:
9141 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9143 /* Default values for num_gangs, num_workers, and vector_length. */
9144 t_num_gangs
= t_num_workers
= t_vector_length
9145 = fold_convert_loc (gimple_location (entry_stmt
),
9146 integer_type_node
, integer_one_node
);
9147 /* ..., but if present, use the value specified by the respective
9148 clause, making sure that are of the correct type. */
9149 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9151 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9153 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9154 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9156 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9158 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9159 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9161 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9163 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9164 args
->quick_push (t_num_gangs
);
9165 args
->quick_push (t_num_workers
);
9166 args
->quick_push (t_vector_length
);
9169 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9170 case BUILT_IN_GOACC_UPDATE
:
9175 /* Default values for t_async. */
9176 /* TODO: XXX FIX -2. */
9177 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9179 build_int_cst (integer_type_node
, -2));
9180 /* ..., but if present, use the value specified by the respective
9181 clause, making sure that is of the correct type. */
9182 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9184 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9186 OMP_CLAUSE_ASYNC_EXPR (c
));
9188 args
->quick_push (t_async
);
9189 /* Save the index, and... */
9190 t_wait_idx
= args
->length ();
9191 /* ... push a default value. */
9192 args
->quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9194 integer_zero_node
));
9195 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9200 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9202 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9204 args
->safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9206 OMP_CLAUSE_WAIT_EXPR (c
)));
9211 /* Now that we know the number, replace the default value. */
9212 args
->ordered_remove (t_wait_idx
);
9213 args
->quick_insert (t_wait_idx
,
9214 fold_convert_loc (gimple_location (entry_stmt
),
9216 build_int_cst (integer_type_node
, n
)));
9224 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), *args
);
9226 gimple_set_location (g
, gimple_location (entry_stmt
));
9227 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9231 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9232 gsi_remove (&gsi
, true);
9237 gsi
= gsi_last_bb (region
->exit
);
9239 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9240 gsi_remove (&gsi
, true);
9245 /* Expand the parallel region tree rooted at REGION. Expansion
9246 proceeds in depth-first order. Innermost regions are expanded
9247 first. This way, parallel regions that require a new function to
9248 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9249 internal dependencies in their body. */
9252 expand_omp (struct omp_region
*region
)
9256 location_t saved_location
;
9257 gimple inner_stmt
= NULL
;
9259 /* First, determine whether this is a combined parallel+workshare
9261 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9262 determine_parallel_type (region
);
9264 if (region
->type
== GIMPLE_OMP_FOR
9265 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9266 inner_stmt
= last_stmt (region
->inner
->entry
);
9269 expand_omp (region
->inner
);
9271 saved_location
= input_location
;
9272 if (gimple_has_location (last_stmt (region
->entry
)))
9273 input_location
= gimple_location (last_stmt (region
->entry
));
9275 switch (region
->type
)
9277 case GIMPLE_OMP_PARALLEL
:
9278 case GIMPLE_OMP_TASK
:
9279 expand_omp_taskreg (region
);
9282 case GIMPLE_OMP_FOR
:
9283 expand_omp_for (region
, inner_stmt
);
9286 case GIMPLE_OMP_SECTIONS
:
9287 expand_omp_sections (region
);
9290 case GIMPLE_OMP_SECTION
:
9291 /* Individual omp sections are handled together with their
9292 parent GIMPLE_OMP_SECTIONS region. */
9295 case GIMPLE_OMP_SINGLE
:
9296 expand_omp_single (region
);
9299 case GIMPLE_OMP_MASTER
:
9300 case GIMPLE_OMP_TASKGROUP
:
9301 case GIMPLE_OMP_ORDERED
:
9302 case GIMPLE_OMP_CRITICAL
:
9303 case GIMPLE_OMP_TEAMS
:
9304 expand_omp_synch (region
);
9307 case GIMPLE_OMP_ATOMIC_LOAD
:
9308 expand_omp_atomic (region
);
9311 case GIMPLE_OMP_TARGET
:
9312 expand_omp_target (region
);
9319 input_location
= saved_location
;
9320 region
= region
->next
;
9325 /* Helper for build_omp_regions. Scan the dominator tree starting at
9326 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9327 true, the function ends once a single tree is built (otherwise, whole
9328 forest of OMP constructs may be built). */
9331 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9334 gimple_stmt_iterator gsi
;
9338 gsi
= gsi_last_bb (bb
);
9339 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9341 struct omp_region
*region
;
9342 enum gimple_code code
;
9344 stmt
= gsi_stmt (gsi
);
9345 code
= gimple_code (stmt
);
9346 if (code
== GIMPLE_OMP_RETURN
)
9348 /* STMT is the return point out of region PARENT. Mark it
9349 as the exit point and make PARENT the immediately
9350 enclosing region. */
9351 gcc_assert (parent
);
9354 parent
= parent
->outer
;
9356 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9358 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9359 GIMPLE_OMP_RETURN, but matches with
9360 GIMPLE_OMP_ATOMIC_LOAD. */
9361 gcc_assert (parent
);
9362 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9365 parent
= parent
->outer
;
9368 else if (code
== GIMPLE_OMP_CONTINUE
)
9370 gcc_assert (parent
);
9373 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9375 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9376 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9379 else if (code
== GIMPLE_OMP_TARGET
9380 && (gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
9381 || (gimple_omp_target_kind (stmt
)
9382 == GF_OMP_TARGET_KIND_OACC_UPDATE
)
9383 || (gimple_omp_target_kind (stmt
)
9384 == GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
)))
9385 new_omp_region (bb
, code
, parent
);
9388 /* Otherwise, this directive becomes the parent for a new
9390 region
= new_omp_region (bb
, code
, parent
);
9395 if (single_tree
&& !parent
)
9398 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9400 son
= next_dom_son (CDI_DOMINATORS
, son
))
9401 build_omp_regions_1 (son
, parent
, single_tree
);
9404 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9408 build_omp_regions_root (basic_block root
)
9410 gcc_assert (root_omp_region
== NULL
);
9411 build_omp_regions_1 (root
, NULL
, true);
9412 gcc_assert (root_omp_region
!= NULL
);
9415 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9418 omp_expand_local (basic_block head
)
9420 build_omp_regions_root (head
);
9421 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9423 fprintf (dump_file
, "\nOMP region tree\n\n");
9424 dump_omp_region (dump_file
, root_omp_region
, 0);
9425 fprintf (dump_file
, "\n");
9428 remove_exit_barriers (root_omp_region
);
9429 expand_omp (root_omp_region
);
9431 free_omp_regions ();
9434 /* Scan the CFG and build a tree of OMP regions. Return the root of
9435 the OMP region tree. */
9438 build_omp_regions (void)
9440 gcc_assert (root_omp_region
== NULL
);
9441 calculate_dominance_info (CDI_DOMINATORS
);
9442 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9445 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9448 execute_expand_omp (void)
9450 build_omp_regions ();
9452 if (!root_omp_region
)
9457 fprintf (dump_file
, "\nOMP region tree\n\n");
9458 dump_omp_region (dump_file
, root_omp_region
, 0);
9459 fprintf (dump_file
, "\n");
9462 remove_exit_barriers (root_omp_region
);
9464 expand_omp (root_omp_region
);
9466 cleanup_tree_cfg ();
9468 free_omp_regions ();
9473 /* OMP expansion -- the default pass, run before creation of SSA form. */
9477 const pass_data pass_data_expand_omp
=
9479 GIMPLE_PASS
, /* type */
9480 "ompexp", /* name */
9481 OPTGROUP_NONE
, /* optinfo_flags */
9482 TV_NONE
, /* tv_id */
9483 PROP_gimple_any
, /* properties_required */
9484 PROP_gimple_eomp
, /* properties_provided */
9485 0, /* properties_destroyed */
9486 0, /* todo_flags_start */
9487 0, /* todo_flags_finish */
9490 class pass_expand_omp
: public gimple_opt_pass
9493 pass_expand_omp (gcc::context
*ctxt
)
9494 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9497 /* opt_pass methods: */
9498 virtual unsigned int execute (function
*)
9500 bool gate
= ((flag_openacc
!= 0 || flag_openmp
!= 0
9501 || flag_openmp_simd
!= 0 || flag_cilkplus
!= 0)
9504 /* This pass always runs, to provide PROP_gimple_eomp.
9505 But there is nothing to do unless -fopenmp is given. */
9509 return execute_expand_omp ();
9512 }; // class pass_expand_omp
9517 make_pass_expand_omp (gcc::context
*ctxt
)
9519 return new pass_expand_omp (ctxt
);
9524 const pass_data pass_data_expand_omp_ssa
=
9526 GIMPLE_PASS
, /* type */
9527 "ompexpssa", /* name */
9528 OPTGROUP_NONE
, /* optinfo_flags */
9529 TV_NONE
, /* tv_id */
9530 PROP_cfg
| PROP_ssa
, /* properties_required */
9531 PROP_gimple_eomp
, /* properties_provided */
9532 0, /* properties_destroyed */
9533 0, /* todo_flags_start */
9534 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9537 class pass_expand_omp_ssa
: public gimple_opt_pass
9540 pass_expand_omp_ssa (gcc::context
*ctxt
)
9541 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9544 /* opt_pass methods: */
9545 virtual bool gate (function
*fun
)
9547 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9549 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9551 }; // class pass_expand_omp_ssa
9556 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9558 return new pass_expand_omp_ssa (ctxt
);
9561 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9562 convert it to gimple. */
9564 oacc_gimple_assign_with_ops (tree_code op
, tree dest
, tree src
,
9569 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9571 stmt
= gimple_build_assign_with_ops (op
, dest
, dest
, src
);
9572 gimple_seq_add_stmt (seq
, stmt
);
9576 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9577 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9578 gimplify_assign (t
, rdest
, seq
);
9581 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9582 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9583 gimplify_assign (t
, idest
, seq
);
9586 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)), NULL
);
9587 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9588 gimplify_assign (t
, rsrc
, seq
);
9591 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)), NULL
);
9592 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9593 gimplify_assign (t
, isrc
, seq
);
9596 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9597 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9600 gcc_assert (op
== PLUS_EXPR
|| op
== MULT_EXPR
);
9602 if (op
== PLUS_EXPR
)
9604 stmt
= gimple_build_assign_with_ops (op
, r
, rdest
, rsrc
);
9605 gimple_seq_add_stmt (seq
, stmt
);
9607 stmt
= gimple_build_assign_with_ops (op
, i
, idest
, isrc
);
9608 gimple_seq_add_stmt (seq
, stmt
);
9610 else if (op
== MULT_EXPR
)
9612 /* Let x = a + ib = dest, y = c + id = src.
9613 x * y = (ac - bd) + i(ad + bc) */
9614 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9615 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9616 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9617 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)), NULL
);
9619 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, ac
, rdest
, rsrc
);
9620 gimple_seq_add_stmt (seq
, stmt
);
9622 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, bd
, idest
, isrc
);
9623 gimple_seq_add_stmt (seq
, stmt
);
9625 stmt
= gimple_build_assign_with_ops (MINUS_EXPR
, r
, ac
, bd
);
9626 gimple_seq_add_stmt (seq
, stmt
);
9628 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, ad
, rdest
, isrc
);
9629 gimple_seq_add_stmt (seq
, stmt
);
9631 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, bd
, idest
, rsrc
);
9632 gimple_seq_add_stmt (seq
, stmt
);
9634 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, i
, ad
, bc
);
9635 gimple_seq_add_stmt (seq
, stmt
);
9638 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9639 gimplify_assign (dest
, result
, seq
);
9642 /* Helper function to initialize local data for the reduction arrays.
9643 The reduction arrays need to be placed inside the calling function
9644 for accelerators, or else the host won't be able to preform the final
9645 reduction. FIXME: This function assumes that there are
9646 vector_length threads in total. */
9649 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9650 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9652 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
9658 /* Find the innermost PARALLEL openmp context. FIXME: OpenACC kernels
9659 may require extra care unless they are converted to openmp for loops. */
9660 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9661 && (gimple_omp_target_kind (ctx
->stmt
)
9662 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9667 /* Extract the clauses. */
9668 oc
= gimple_omp_target_clauses (octx
->stmt
);
9670 /* Find the last outer clause. */
9671 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9674 /* Allocate arrays for each reduction variable. */
9675 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9677 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9680 tree var
= OMP_CLAUSE_DECL (c
);
9681 tree type
= get_base_type (var
);
9682 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9686 /* Calculate size of the reduction array. */
9687 t
= create_tmp_var (TREE_TYPE (nthreads
), NULL
);
9688 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, t
, nthreads
,
9689 fold_convert (TREE_TYPE (nthreads
),
9690 TYPE_SIZE_UNIT (type
)));
9691 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9693 size
= create_tmp_var (sizetype
, NULL
);
9694 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9696 /* Now allocate memory for it. FIXME: Allocating memory for the
9697 reduction array may be unnecessary once the final reduction is able
9698 to be preformed on the accelerator. Instead of allocating memory on
9699 the host side, it could just be allocated on the accelerator. */
9700 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9701 stmt
= gimple_build_call (call
, 1, size
);
9702 gimple_call_set_lhs (stmt
, array
);
9703 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9705 /* Map this array into the accelerator. */
9707 /* Add the reduction array to the list of clauses. */
9708 /* FIXME: Currently, these variables must be placed in the outer
9709 most clause so that copy-out works. */
9711 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9712 OMP_CLAUSE_MAP_KIND (t
) = OMP_CLAUSE_MAP_FORCE_FROM
;
9713 OMP_CLAUSE_DECL (t
) = x
;
9714 OMP_CLAUSE_CHAIN (t
) = NULL
;
9716 OMP_CLAUSE_CHAIN (oc
) = t
;
9718 gimple_omp_target_set_clauses (octx
->stmt
, t
);
9719 OMP_CLAUSE_SIZE (t
) = size
;
9724 /* Helper function to process the array of partial reductions. Nthreads
9725 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9726 cannot be used here, because nthreads on the host may be different than
9727 on the accelerator. */
9730 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9731 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9733 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
9735 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9740 let var = the original reduction variable
9741 let array = reduction variable array
9743 for (i = 0; i < nthreads; i++)
9747 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9748 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9749 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9751 /* Create and initialize an index variable. */
9752 tree ix
= create_tmp_var (sizetype
, NULL
);
9753 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9756 /* Insert the loop header label here. */
9757 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9759 /* Exit loop if ix >= nthreads. */
9760 x
= create_tmp_var (sizetype
, NULL
);
9761 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9762 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9763 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9765 /* Insert the loop body label here. */
9766 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9768 /* Collapse each reduction array, one element at a time. */
9769 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9771 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9774 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9776 /* reduction(-:var) sums up the partial results, so it acts
9777 identically to reduction(+:var). */
9778 if (reduction_code
== MINUS_EXPR
)
9779 reduction_code
= PLUS_EXPR
;
9781 /* Set up reduction variable var. */
9782 var
= OMP_CLAUSE_DECL (c
);
9783 type
= get_base_type (var
);
9784 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9785 (OMP_CLAUSE_DECL (c
)), ctx
);
9787 /* Calculate the array offset. */
9788 tree offset
= create_tmp_var (sizetype
, NULL
);
9789 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9790 stmt
= gimple_build_assign_with_ops (MULT_EXPR
, offset
, offset
, ix
);
9791 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9793 tree ptr
= create_tmp_var (TREE_TYPE (array
), NULL
);
9794 stmt
= gimple_build_assign_with_ops (POINTER_PLUS_EXPR
, ptr
, array
,
9796 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9798 /* Extract array[ix] into mem. */
9799 tree mem
= create_tmp_var (type
, NULL
);
9800 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9802 /* Find the original reduction variable. */
9803 if (is_reference (var
))
9804 var
= build_simple_mem_ref (var
);
9806 tree t
= create_tmp_var (type
, NULL
);
9808 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9809 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9811 /* var = var op mem */
9812 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9814 case TRUTH_ANDIF_EXPR
:
9815 case TRUTH_ORIF_EXPR
:
9816 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9818 gimplify_and_add (t
, stmt_seqp
);
9821 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9822 oacc_gimple_assign_with_ops (OMP_CLAUSE_REDUCTION_CODE (c
),
9826 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9827 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9828 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9831 /* Increment the induction variable. */
9832 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9833 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ix
, ix
, one
);
9834 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9836 /* Go back to the top of the loop. */
9837 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9839 /* Place the loop exit label here. */
9840 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9843 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9844 scan that for reductions. */
9847 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9848 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9850 gcc_assert (is_gimple_omp_oacc_specifically (ctx
->stmt
));
9852 gimple_stmt_iterator gsi
;
9853 gimple_seq inner
= NULL
;
9856 /* A collapse clause may have inserted a new bind block. */
9857 gsi
= gsi_start (*body
);
9858 while (!gsi_end_p (gsi
))
9860 stmt
= gsi_stmt (gsi
);
9861 if (gimple_code (stmt
) == GIMPLE_BIND
)
9863 inner
= gimple_bind_body (stmt
);
9865 gsi
= gsi_start (*body
);
9867 else if (gimple_code (stmt
) == GIMPLE_OMP_FOR
)
9873 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9875 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9877 bool reduction_found
= false;
9879 stmt
= gsi_stmt (gsi
);
9881 switch (gimple_code (stmt
))
9883 /* FIXME: A reduction may also appear in an oacc parallel. */
9884 case GIMPLE_OMP_FOR
:
9885 clauses
= gimple_omp_for_clauses (stmt
);
9887 /* Search for a reduction clause. */
9888 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9889 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9891 reduction_found
= true;
9895 if (!reduction_found
)
9898 ctx
= maybe_lookup_ctx (stmt
);
9901 /* Extract the number of threads. */
9902 nthreads
= create_tmp_var (sizetype
, NULL
);
9903 t
= oacc_max_threads (ctx
);
9904 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9906 /* Determine if this is kernel will be executed on the host. */
9907 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9908 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9909 stmt
= gimple_build_call (call
, 0);
9910 gimple_call_set_lhs (stmt
, acc_device
);
9911 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9913 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9914 acc_device_host
= create_tmp_var (integer_type_node
,
9915 ".acc_device_host");
9916 gimplify_assign (acc_device_host
, build_int_cst (integer_type_node
,
9920 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9921 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9923 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9925 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9926 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9927 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9930 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9932 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9933 gimplify_assign (acc_device_host
, build_int_cst (integer_type_node
,
9937 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9938 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9940 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9942 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9943 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9944 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9947 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9949 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9951 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9954 // Scan for other directives which support reduction here.
9960 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
9962 /* If ctx is a worksharing context inside of a cancellable parallel
9963 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9964 and conditional branch to parallel's cancel_label to handle
9965 cancellation in the implicit barrier. */
9968 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9970 gimple omp_return
= gimple_seq_last_stmt (*body
);
9971 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9972 if (gimple_omp_return_nowait_p (omp_return
))
9975 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9976 && ctx
->outer
->cancellable
)
9978 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9979 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9980 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
9981 gimple_omp_return_set_lhs (omp_return
, lhs
);
9982 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9983 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
9984 fold_convert (c_bool_type
,
9985 boolean_false_node
),
9986 ctx
->outer
->cancel_label
, fallthru_label
);
9987 gimple_seq_add_stmt (body
, g
);
9988 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
9992 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9993 CTX is the enclosing OMP context for the current statement. */
9996 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9998 tree block
, control
;
9999 gimple_stmt_iterator tgsi
;
10000 gimple stmt
, new_stmt
, bind
, t
;
10001 gimple_seq ilist
, dlist
, olist
, new_body
;
10003 stmt
= gsi_stmt (*gsi_p
);
10005 push_gimplify_context ();
10009 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10010 &ilist
, &dlist
, ctx
, NULL
);
10012 new_body
= gimple_omp_body (stmt
);
10013 gimple_omp_set_body (stmt
, NULL
);
10014 tgsi
= gsi_start (new_body
);
10015 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10020 sec_start
= gsi_stmt (tgsi
);
10021 sctx
= maybe_lookup_ctx (sec_start
);
10024 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10025 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10026 GSI_CONTINUE_LINKING
);
10027 gimple_omp_set_body (sec_start
, NULL
);
10029 if (gsi_one_before_end_p (tgsi
))
10031 gimple_seq l
= NULL
;
10032 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10034 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10035 gimple_omp_section_set_last (sec_start
);
10038 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10039 GSI_CONTINUE_LINKING
);
10042 block
= make_node (BLOCK
);
10043 bind
= gimple_build_bind (NULL
, new_body
, block
);
10046 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10048 block
= make_node (BLOCK
);
10049 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10050 gsi_replace (gsi_p
, new_stmt
, true);
10052 pop_gimplify_context (new_stmt
);
10053 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10054 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10055 if (BLOCK_VARS (block
))
10056 TREE_USED (block
) = 1;
10059 gimple_seq_add_seq (&new_body
, ilist
);
10060 gimple_seq_add_stmt (&new_body
, stmt
);
10061 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10062 gimple_seq_add_stmt (&new_body
, bind
);
10064 control
= create_tmp_var (unsigned_type_node
, ".section");
10065 t
= gimple_build_omp_continue (control
, control
);
10066 gimple_omp_sections_set_control (stmt
, control
);
10067 gimple_seq_add_stmt (&new_body
, t
);
10069 gimple_seq_add_seq (&new_body
, olist
);
10070 if (ctx
->cancellable
)
10071 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10072 gimple_seq_add_seq (&new_body
, dlist
);
10074 new_body
= maybe_catch_exception (new_body
);
10076 t
= gimple_build_omp_return
10077 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10078 OMP_CLAUSE_NOWAIT
));
10079 gimple_seq_add_stmt (&new_body
, t
);
10080 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10082 gimple_bind_set_body (new_stmt
, new_body
);
10086 /* A subroutine of lower_omp_single. Expand the simple form of
10087 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10089 if (GOMP_single_start ())
10091 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10093 FIXME. It may be better to delay expanding the logic of this until
10094 pass_expand_omp. The expanded logic may make the job more difficult
10095 to a synchronization analysis pass. */
10098 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
10100 location_t loc
= gimple_location (single_stmt
);
10101 tree tlabel
= create_artificial_label (loc
);
10102 tree flabel
= create_artificial_label (loc
);
10106 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10107 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
10108 call
= gimple_build_call (decl
, 0);
10109 gimple_call_set_lhs (call
, lhs
);
10110 gimple_seq_add_stmt (pre_p
, call
);
10112 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10113 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10114 boolean_true_node
),
10116 gimple_seq_add_stmt (pre_p
, cond
);
10117 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10118 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10119 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10123 /* A subroutine of lower_omp_single. Expand the simple form of
10124 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10126 #pragma omp single copyprivate (a, b, c)
10128 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10131 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10137 GOMP_single_copy_end (©out);
10148 FIXME. It may be better to delay expanding the logic of this until
10149 pass_expand_omp. The expanded logic may make the job more difficult
10150 to a synchronization analysis pass. */
10153 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
10155 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10156 gimple_seq copyin_seq
;
10157 location_t loc
= gimple_location (single_stmt
);
10159 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10161 ptr_type
= build_pointer_type (ctx
->record_type
);
10162 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10164 l0
= create_artificial_label (loc
);
10165 l1
= create_artificial_label (loc
);
10166 l2
= create_artificial_label (loc
);
10168 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10169 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10170 t
= fold_convert_loc (loc
, ptr_type
, t
);
10171 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10173 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10174 build_int_cst (ptr_type
, 0));
10175 t
= build3 (COND_EXPR
, void_type_node
, t
,
10176 build_and_jump (&l0
), build_and_jump (&l1
));
10177 gimplify_and_add (t
, pre_p
);
10179 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10181 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10184 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10187 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10188 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10189 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10190 gimplify_and_add (t
, pre_p
);
10192 t
= build_and_jump (&l2
);
10193 gimplify_and_add (t
, pre_p
);
10195 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10197 gimple_seq_add_seq (pre_p
, copyin_seq
);
10199 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10203 /* Expand code for an OpenMP single directive. */
10206 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10209 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
10210 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10212 push_gimplify_context ();
10214 block
= make_node (BLOCK
);
10215 bind
= gimple_build_bind (NULL
, NULL
, block
);
10216 gsi_replace (gsi_p
, bind
, true);
10219 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10220 &bind_body
, &dlist
, ctx
, NULL
);
10221 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10223 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10225 if (ctx
->record_type
)
10226 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10228 lower_omp_single_simple (single_stmt
, &bind_body
);
10230 gimple_omp_set_body (single_stmt
, NULL
);
10232 gimple_seq_add_seq (&bind_body
, dlist
);
10234 bind_body
= maybe_catch_exception (bind_body
);
10236 t
= gimple_build_omp_return
10237 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10238 OMP_CLAUSE_NOWAIT
));
10239 gimple_seq_add_stmt (&bind_body_tail
, t
);
10240 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10241 if (ctx
->record_type
)
10243 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10244 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10245 TREE_THIS_VOLATILE (clobber
) = 1;
10246 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10247 clobber
), GSI_SAME_STMT
);
10249 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10250 gimple_bind_set_body (bind
, bind_body
);
10252 pop_gimplify_context (bind
);
10254 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10255 BLOCK_VARS (block
) = ctx
->block_vars
;
10256 if (BLOCK_VARS (block
))
10257 TREE_USED (block
) = 1;
10261 /* Expand code for an OpenMP master directive. */
10264 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10266 tree block
, lab
= NULL
, x
, bfn_decl
;
10267 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
10268 location_t loc
= gimple_location (stmt
);
10271 push_gimplify_context ();
10273 block
= make_node (BLOCK
);
10274 bind
= gimple_build_bind (NULL
, NULL
, block
);
10275 gsi_replace (gsi_p
, bind
, true);
10276 gimple_bind_add_stmt (bind
, stmt
);
10278 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10279 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10280 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10281 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10283 gimplify_and_add (x
, &tseq
);
10284 gimple_bind_add_seq (bind
, tseq
);
10286 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10287 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10288 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10289 gimple_omp_set_body (stmt
, NULL
);
10291 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10293 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10295 pop_gimplify_context (bind
);
10297 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10298 BLOCK_VARS (block
) = ctx
->block_vars
;
10302 /* Expand code for an OpenMP taskgroup directive. */
10305 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10307 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
10308 tree block
= make_node (BLOCK
);
10310 bind
= gimple_build_bind (NULL
, NULL
, block
);
10311 gsi_replace (gsi_p
, bind
, true);
10312 gimple_bind_add_stmt (bind
, stmt
);
10314 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10316 gimple_bind_add_stmt (bind
, x
);
10318 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10319 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10320 gimple_omp_set_body (stmt
, NULL
);
10322 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10324 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10325 BLOCK_VARS (block
) = ctx
->block_vars
;
10329 /* Expand code for an OpenMP ordered directive. */
10332 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10335 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
10337 push_gimplify_context ();
10339 block
= make_node (BLOCK
);
10340 bind
= gimple_build_bind (NULL
, NULL
, block
);
10341 gsi_replace (gsi_p
, bind
, true);
10342 gimple_bind_add_stmt (bind
, stmt
);
10344 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10346 gimple_bind_add_stmt (bind
, x
);
10348 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10349 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10350 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10351 gimple_omp_set_body (stmt
, NULL
);
10353 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10354 gimple_bind_add_stmt (bind
, x
);
10356 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10358 pop_gimplify_context (bind
);
10360 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10361 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10365 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10366 substitution of a couple of function calls. But in the NAMED case,
10367 requires that languages coordinate a symbol name. It is therefore
10368 best put here in common code. */
10370 static GTY((param1_is (tree
), param2_is (tree
)))
10371 splay_tree critical_name_mutexes
;
10374 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10377 tree name
, lock
, unlock
;
10378 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
10379 location_t loc
= gimple_location (stmt
);
10382 name
= gimple_omp_critical_name (stmt
);
10388 if (!critical_name_mutexes
)
10389 critical_name_mutexes
10390 = splay_tree_new_ggc (splay_tree_compare_pointers
,
10391 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
10392 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
10394 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
10399 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
10401 new_str
= ACONCAT ((".gomp_critical_user_",
10402 IDENTIFIER_POINTER (name
), NULL
));
10403 DECL_NAME (decl
) = get_identifier (new_str
);
10404 TREE_PUBLIC (decl
) = 1;
10405 TREE_STATIC (decl
) = 1;
10406 DECL_COMMON (decl
) = 1;
10407 DECL_ARTIFICIAL (decl
) = 1;
10408 DECL_IGNORED_P (decl
) = 1;
10410 /* If '#pragma omp critical' is inside offloaded region, the symbol
10411 must be marked for offloading. */
10413 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10414 if (is_gimple_omp_offloaded (octx
->stmt
))
10416 varpool_node::get_create (decl
)->offloadable
= 1;
10420 varpool_node::finalize_decl (decl
);
10422 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
10423 (splay_tree_value
) decl
);
10426 decl
= (tree
) n
->value
;
10428 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10429 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10431 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10432 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10433 build_fold_addr_expr_loc (loc
, decl
));
10437 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10438 lock
= build_call_expr_loc (loc
, lock
, 0);
10440 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10441 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10444 push_gimplify_context ();
10446 block
= make_node (BLOCK
);
10447 bind
= gimple_build_bind (NULL
, NULL
, block
);
10448 gsi_replace (gsi_p
, bind
, true);
10449 gimple_bind_add_stmt (bind
, stmt
);
10451 tbody
= gimple_bind_body (bind
);
10452 gimplify_and_add (lock
, &tbody
);
10453 gimple_bind_set_body (bind
, tbody
);
10455 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10456 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10457 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10458 gimple_omp_set_body (stmt
, NULL
);
10460 tbody
= gimple_bind_body (bind
);
10461 gimplify_and_add (unlock
, &tbody
);
10462 gimple_bind_set_body (bind
, tbody
);
10464 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10466 pop_gimplify_context (bind
);
10467 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10468 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10472 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10473 for a lastprivate clause. Given a loop control predicate of (V
10474 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10475 is appended to *DLIST, iterator initialization is appended to
10479 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10480 gimple_seq
*dlist
, struct omp_context
*ctx
)
10482 tree clauses
, cond
, vinit
;
10483 enum tree_code cond_code
;
10486 cond_code
= fd
->loop
.cond_code
;
10487 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10489 /* When possible, use a strict equality expression. This can let VRP
10490 type optimizations deduce the value and remove a copy. */
10491 if (tree_fits_shwi_p (fd
->loop
.step
))
10493 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10494 if (step
== 1 || step
== -1)
10495 cond_code
= EQ_EXPR
;
10498 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
10500 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10502 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10503 if (!gimple_seq_empty_p (stmts
))
10505 gimple_seq_add_seq (&stmts
, *dlist
);
10508 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10509 vinit
= fd
->loop
.n1
;
10510 if (cond_code
== EQ_EXPR
10511 && tree_fits_shwi_p (fd
->loop
.n2
)
10512 && ! integer_zerop (fd
->loop
.n2
))
10513 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10515 vinit
= unshare_expr (vinit
);
10517 /* Initialize the iterator variable, so that threads that don't execute
10518 any iterations don't execute the lastprivate clauses by accident. */
10519 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10524 /* Lower code for an OpenMP loop directive. */
10527 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10529 tree
*rhs_p
, block
;
10530 struct omp_for_data fd
, *fdp
= NULL
;
10531 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
10532 gimple_seq omp_for_body
, body
, dlist
;
10535 push_gimplify_context ();
10537 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10539 block
= make_node (BLOCK
);
10540 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10541 /* Replace at gsi right away, so that 'stmt' is no member
10542 of a sequence anymore as we're going to add to to a different
10544 gsi_replace (gsi_p
, new_stmt
, true);
10546 /* Move declaration of temporaries in the loop body before we make
10548 omp_for_body
= gimple_omp_body (stmt
);
10549 if (!gimple_seq_empty_p (omp_for_body
)
10550 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10552 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
10553 tree vars
= gimple_bind_vars (inner_bind
);
10554 gimple_bind_append_vars (new_stmt
, vars
);
10555 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10556 keep them on the inner_bind and it's block. */
10557 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10558 if (gimple_bind_block (inner_bind
))
10559 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10562 if (gimple_omp_for_combined_into_p (stmt
))
10564 gcc_assert (gimple_omp_for_kind (stmt
) != GF_OMP_FOR_KIND_OACC_LOOP
);
10566 extract_omp_for_data (stmt
, &fd
, NULL
);
10569 /* We need two temporaries with fd.loop.v type (istart/iend)
10570 and then (fd.collapse - 1) temporaries with the same
10571 type for count2 ... countN-1 vars if not constant. */
10573 tree type
= fd
.iter_type
;
10574 if (fd
.collapse
> 1
10575 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10576 count
+= fd
.collapse
- 1;
10577 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10578 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10579 tree clauses
= *pc
;
10582 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10583 OMP_CLAUSE__LOOPTEMP_
);
10584 for (i
= 0; i
< count
; i
++)
10589 gcc_assert (outerc
);
10590 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10591 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10592 OMP_CLAUSE__LOOPTEMP_
);
10596 temp
= create_tmp_var (type
, NULL
);
10597 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10599 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10600 OMP_CLAUSE_DECL (*pc
) = temp
;
10601 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10606 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10609 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10611 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10613 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10615 /* Lower the header expressions. At this point, we can assume that
10616 the header is of the form:
10618 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10620 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10621 using the .omp_data_s mapping, if needed. */
10622 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10624 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10625 if (!is_gimple_min_invariant (*rhs_p
))
10626 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10628 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10629 if (!is_gimple_min_invariant (*rhs_p
))
10630 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10632 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10633 if (!is_gimple_min_invariant (*rhs_p
))
10634 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10637 /* Once lowered, extract the bounds and clauses. */
10638 extract_omp_for_data (stmt
, &fd
, NULL
);
10640 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10642 gimple_seq_add_stmt (&body
, stmt
);
10643 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10645 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10648 /* After the loop, add exit clauses. */
10649 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10651 if (ctx
->cancellable
)
10652 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10654 gimple_seq_add_seq (&body
, dlist
);
10656 body
= maybe_catch_exception (body
);
10658 /* Region exit marker goes at the end of the loop body. */
10659 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10660 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10661 pop_gimplify_context (new_stmt
);
10663 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10664 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10665 if (BLOCK_VARS (block
))
10666 TREE_USED (block
) = 1;
10668 gimple_bind_set_body (new_stmt
, body
);
10669 gimple_omp_set_body (stmt
, NULL
);
10670 gimple_omp_for_set_pre_body (stmt
, NULL
);
10673 /* Callback for walk_stmts. Check if the current statement only contains
10674 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10677 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10678 bool *handled_ops_p
,
10679 struct walk_stmt_info
*wi
)
10681 int *info
= (int *) wi
->info
;
10682 gimple stmt
= gsi_stmt (*gsi_p
);
10684 *handled_ops_p
= true;
10685 switch (gimple_code (stmt
))
10689 case GIMPLE_OMP_FOR
:
10690 case GIMPLE_OMP_SECTIONS
:
10691 *info
= *info
== 0 ? 1 : -1;
10700 struct omp_taskcopy_context
10702 /* This field must be at the beginning, as we do "inheritance": Some
10703 callback functions for tree-inline.c (e.g., omp_copy_decl)
10704 receive a copy_body_data pointer that is up-casted to an
10705 omp_context pointer. */
10711 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10713 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10715 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10716 return create_tmp_var (TREE_TYPE (var
), NULL
);
10722 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10724 tree name
, new_fields
= NULL
, type
, f
;
10726 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10727 name
= DECL_NAME (TYPE_NAME (orig_type
));
10728 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10729 TYPE_DECL
, name
, type
);
10730 TYPE_NAME (type
) = name
;
10732 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10734 tree new_f
= copy_node (f
);
10735 DECL_CONTEXT (new_f
) = type
;
10736 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10737 TREE_CHAIN (new_f
) = new_fields
;
10738 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10739 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10740 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10742 new_fields
= new_f
;
10743 tcctx
->cb
.decl_map
->put (f
, new_f
);
10745 TYPE_FIELDS (type
) = nreverse (new_fields
);
10746 layout_type (type
);
10750 /* Create task copyfn. */
10753 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
10755 gcc_assert (!is_gimple_omp_oacc_specifically (ctx
->stmt
));
10757 struct function
*child_cfun
;
10758 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10759 tree record_type
, srecord_type
, bind
, list
;
10760 bool record_needs_remap
= false, srecord_needs_remap
= false;
10762 struct omp_taskcopy_context tcctx
;
10763 location_t loc
= gimple_location (task_stmt
);
10765 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10766 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10767 gcc_assert (child_cfun
->cfg
== NULL
);
10768 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10770 /* Reset DECL_CONTEXT on function arguments. */
10771 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10772 DECL_CONTEXT (t
) = child_fn
;
10774 /* Populate the function. */
10775 push_gimplify_context ();
10776 push_cfun (child_cfun
);
10778 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10779 TREE_SIDE_EFFECTS (bind
) = 1;
10781 DECL_SAVED_TREE (child_fn
) = bind
;
10782 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10784 /* Remap src and dst argument types if needed. */
10785 record_type
= ctx
->record_type
;
10786 srecord_type
= ctx
->srecord_type
;
10787 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10788 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10790 record_needs_remap
= true;
10793 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10794 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10796 srecord_needs_remap
= true;
10800 if (record_needs_remap
|| srecord_needs_remap
)
10802 memset (&tcctx
, '\0', sizeof (tcctx
));
10803 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10804 tcctx
.cb
.dst_fn
= child_fn
;
10805 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10806 gcc_checking_assert (tcctx
.cb
.src_node
);
10807 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10808 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10809 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10810 tcctx
.cb
.eh_lp_nr
= 0;
10811 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10812 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10815 if (record_needs_remap
)
10816 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10817 if (srecord_needs_remap
)
10818 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10821 tcctx
.cb
.decl_map
= NULL
;
10823 arg
= DECL_ARGUMENTS (child_fn
);
10824 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10825 sarg
= DECL_CHAIN (arg
);
10826 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10828 /* First pass: initialize temporaries used in record_type and srecord_type
10829 sizes and field offsets. */
10830 if (tcctx
.cb
.decl_map
)
10831 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10832 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10836 decl
= OMP_CLAUSE_DECL (c
);
10837 p
= tcctx
.cb
.decl_map
->get (decl
);
10840 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10841 sf
= (tree
) n
->value
;
10842 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10843 src
= build_simple_mem_ref_loc (loc
, sarg
);
10844 src
= omp_build_component_ref (src
, sf
);
10845 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10846 append_to_statement_list (t
, &list
);
10849 /* Second pass: copy shared var pointers and copy construct non-VLA
10850 firstprivate vars. */
10851 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10852 switch (OMP_CLAUSE_CODE (c
))
10854 case OMP_CLAUSE_SHARED
:
10855 decl
= OMP_CLAUSE_DECL (c
);
10856 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10859 f
= (tree
) n
->value
;
10860 if (tcctx
.cb
.decl_map
)
10861 f
= *tcctx
.cb
.decl_map
->get (f
);
10862 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10863 sf
= (tree
) n
->value
;
10864 if (tcctx
.cb
.decl_map
)
10865 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10866 src
= build_simple_mem_ref_loc (loc
, sarg
);
10867 src
= omp_build_component_ref (src
, sf
);
10868 dst
= build_simple_mem_ref_loc (loc
, arg
);
10869 dst
= omp_build_component_ref (dst
, f
);
10870 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10871 append_to_statement_list (t
, &list
);
10873 case OMP_CLAUSE_FIRSTPRIVATE
:
10874 decl
= OMP_CLAUSE_DECL (c
);
10875 if (is_variable_sized (decl
))
10877 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10880 f
= (tree
) n
->value
;
10881 if (tcctx
.cb
.decl_map
)
10882 f
= *tcctx
.cb
.decl_map
->get (f
);
10883 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10886 sf
= (tree
) n
->value
;
10887 if (tcctx
.cb
.decl_map
)
10888 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10889 src
= build_simple_mem_ref_loc (loc
, sarg
);
10890 src
= omp_build_component_ref (src
, sf
);
10891 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10892 src
= build_simple_mem_ref_loc (loc
, src
);
10896 dst
= build_simple_mem_ref_loc (loc
, arg
);
10897 dst
= omp_build_component_ref (dst
, f
);
10898 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10899 append_to_statement_list (t
, &list
);
10901 case OMP_CLAUSE_PRIVATE
:
10902 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10904 decl
= OMP_CLAUSE_DECL (c
);
10905 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10906 f
= (tree
) n
->value
;
10907 if (tcctx
.cb
.decl_map
)
10908 f
= *tcctx
.cb
.decl_map
->get (f
);
10909 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10912 sf
= (tree
) n
->value
;
10913 if (tcctx
.cb
.decl_map
)
10914 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10915 src
= build_simple_mem_ref_loc (loc
, sarg
);
10916 src
= omp_build_component_ref (src
, sf
);
10917 if (use_pointer_for_field (decl
, NULL
))
10918 src
= build_simple_mem_ref_loc (loc
, src
);
10922 dst
= build_simple_mem_ref_loc (loc
, arg
);
10923 dst
= omp_build_component_ref (dst
, f
);
10924 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10925 append_to_statement_list (t
, &list
);
10931 /* Last pass: handle VLA firstprivates. */
10932 if (tcctx
.cb
.decl_map
)
10933 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10934 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10938 decl
= OMP_CLAUSE_DECL (c
);
10939 if (!is_variable_sized (decl
))
10941 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10944 f
= (tree
) n
->value
;
10945 f
= *tcctx
.cb
.decl_map
->get (f
);
10946 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10947 ind
= DECL_VALUE_EXPR (decl
);
10948 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10949 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10950 n
= splay_tree_lookup (ctx
->sfield_map
,
10951 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10952 sf
= (tree
) n
->value
;
10953 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10954 src
= build_simple_mem_ref_loc (loc
, sarg
);
10955 src
= omp_build_component_ref (src
, sf
);
10956 src
= build_simple_mem_ref_loc (loc
, src
);
10957 dst
= build_simple_mem_ref_loc (loc
, arg
);
10958 dst
= omp_build_component_ref (dst
, f
);
10959 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10960 append_to_statement_list (t
, &list
);
10961 n
= splay_tree_lookup (ctx
->field_map
,
10962 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10963 df
= (tree
) n
->value
;
10964 df
= *tcctx
.cb
.decl_map
->get (df
);
10965 ptr
= build_simple_mem_ref_loc (loc
, arg
);
10966 ptr
= omp_build_component_ref (ptr
, df
);
10967 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
10968 build_fold_addr_expr_loc (loc
, dst
));
10969 append_to_statement_list (t
, &list
);
10972 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
10973 append_to_statement_list (t
, &list
);
10975 if (tcctx
.cb
.decl_map
)
10976 delete tcctx
.cb
.decl_map
;
10977 pop_gimplify_context (NULL
);
10978 BIND_EXPR_BODY (bind
) = list
;
10983 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
10987 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
10989 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
10990 OMP_CLAUSE_DEPEND
);
10991 gcc_assert (clauses
);
10992 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10993 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
10994 switch (OMP_CLAUSE_DEPEND_KIND (c
))
10996 case OMP_CLAUSE_DEPEND_IN
:
10999 case OMP_CLAUSE_DEPEND_OUT
:
11000 case OMP_CLAUSE_DEPEND_INOUT
:
11004 gcc_unreachable ();
11006 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11007 tree array
= create_tmp_var (type
, NULL
);
11008 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11010 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11011 gimple_seq_add_stmt (iseq
, g
);
11012 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11014 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11015 gimple_seq_add_stmt (iseq
, g
);
11016 for (i
= 0; i
< 2; i
++)
11018 if ((i
? n_in
: n_out
) == 0)
11020 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11021 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11022 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11024 tree t
= OMP_CLAUSE_DECL (c
);
11025 t
= fold_convert (ptr_type_node
, t
);
11026 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11027 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11028 NULL_TREE
, NULL_TREE
);
11029 g
= gimple_build_assign (r
, t
);
11030 gimple_seq_add_stmt (iseq
, g
);
11033 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11034 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11035 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11036 OMP_CLAUSE_CHAIN (c
) = *p
;
11038 tree clobber
= build_constructor (type
, NULL
);
11039 TREE_THIS_VOLATILE (clobber
) = 1;
11040 g
= gimple_build_assign (array
, clobber
);
11041 gimple_seq_add_stmt (oseq
, g
);
11044 /* Lower the OpenMP parallel or task directive in the current statement
11045 in GSI_P. CTX holds context information for the directive. */
11048 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11052 gimple stmt
= gsi_stmt (*gsi_p
);
11053 gimple par_bind
, bind
, dep_bind
= NULL
;
11054 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11055 location_t loc
= gimple_location (stmt
);
11057 clauses
= gimple_omp_taskreg_clauses (stmt
);
11058 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
11059 par_body
= gimple_bind_body (par_bind
);
11060 child_fn
= ctx
->cb
.dst_fn
;
11061 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11062 && !gimple_omp_parallel_combined_p (stmt
))
11064 struct walk_stmt_info wi
;
11067 memset (&wi
, 0, sizeof (wi
));
11069 wi
.val_only
= true;
11070 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11072 gimple_omp_parallel_set_combined_p (stmt
, true);
11074 gimple_seq dep_ilist
= NULL
;
11075 gimple_seq dep_olist
= NULL
;
11076 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11077 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11079 push_gimplify_context ();
11080 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11081 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11084 if (ctx
->srecord_type
)
11085 create_task_copyfn (stmt
, ctx
);
11087 push_gimplify_context ();
11092 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11093 lower_omp (&par_body
, ctx
);
11094 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11095 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11097 /* Declare all the variables created by mapping and the variables
11098 declared in the scope of the parallel body. */
11099 record_vars_into (ctx
->block_vars
, child_fn
);
11100 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11102 if (ctx
->record_type
)
11105 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11106 : ctx
->record_type
, ".omp_data_o");
11107 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11108 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11109 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11114 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11115 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11117 if (ctx
->record_type
)
11119 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11120 TREE_THIS_VOLATILE (clobber
) = 1;
11121 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11125 /* Once all the expansions are done, sequence all the different
11126 fragments inside gimple_omp_body. */
11130 if (ctx
->record_type
)
11132 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11133 /* fixup_child_record_type might have changed receiver_decl's type. */
11134 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11135 gimple_seq_add_stmt (&new_body
,
11136 gimple_build_assign (ctx
->receiver_decl
, t
));
11139 gimple_seq_add_seq (&new_body
, par_ilist
);
11140 gimple_seq_add_seq (&new_body
, par_body
);
11141 gimple_seq_add_seq (&new_body
, par_rlist
);
11142 if (ctx
->cancellable
)
11143 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11144 gimple_seq_add_seq (&new_body
, par_olist
);
11145 new_body
= maybe_catch_exception (new_body
);
11146 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11147 gimple_omp_set_body (stmt
, new_body
);
11149 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11150 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11151 gimple_bind_add_seq (bind
, ilist
);
11152 gimple_bind_add_stmt (bind
, stmt
);
11153 gimple_bind_add_seq (bind
, olist
);
11155 pop_gimplify_context (NULL
);
11159 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11160 gimple_bind_add_stmt (dep_bind
, bind
);
11161 gimple_bind_add_seq (dep_bind
, dep_olist
);
11162 pop_gimplify_context (dep_bind
);
11166 /* Lower the GIMPLE_OMP_TARGET in the current statement
11167 in GSI_P. CTX holds context information for the directive. */
11170 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11173 tree child_fn
, t
, c
;
11174 gimple stmt
= gsi_stmt (*gsi_p
);
11175 gimple tgt_bind
, bind
;
11176 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11177 location_t loc
= gimple_location (stmt
);
11178 bool offloaded
, data_region
;
11179 unsigned int map_cnt
= 0;
11181 offloaded
= is_gimple_omp_offloaded (stmt
);
11182 switch (gimple_omp_target_kind (stmt
))
11184 case GF_OMP_TARGET_KIND_REGION
:
11185 case GF_OMP_TARGET_KIND_UPDATE
:
11186 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11187 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11188 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11189 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11190 data_region
= false;
11192 case GF_OMP_TARGET_KIND_DATA
:
11193 case GF_OMP_TARGET_KIND_OACC_DATA
:
11194 data_region
= true;
11197 gcc_unreachable ();
11200 clauses
= gimple_omp_target_clauses (stmt
);
11206 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
11207 tgt_body
= gimple_bind_body (tgt_bind
);
11209 else if (data_region
)
11210 tgt_body
= gimple_omp_body (stmt
);
11211 child_fn
= ctx
->cb
.dst_fn
;
11213 push_gimplify_context ();
11218 && is_gimple_omp_oacc_specifically (stmt
))
11219 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11221 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11222 switch (OMP_CLAUSE_CODE (c
))
11228 case OMP_CLAUSE_MAP
:
11229 #ifdef ENABLE_CHECKING
11230 /* First check what we're prepared to handle in the following. */
11231 switch (OMP_CLAUSE_MAP_KIND (c
))
11233 case OMP_CLAUSE_MAP_ALLOC
:
11234 case OMP_CLAUSE_MAP_TO
:
11235 case OMP_CLAUSE_MAP_FROM
:
11236 case OMP_CLAUSE_MAP_TOFROM
:
11237 case OMP_CLAUSE_MAP_POINTER
:
11238 case OMP_CLAUSE_MAP_TO_PSET
:
11240 case OMP_CLAUSE_MAP_FORCE_ALLOC
:
11241 case OMP_CLAUSE_MAP_FORCE_TO
:
11242 case OMP_CLAUSE_MAP_FORCE_FROM
:
11243 case OMP_CLAUSE_MAP_FORCE_TOFROM
:
11244 case OMP_CLAUSE_MAP_FORCE_PRESENT
:
11245 case OMP_CLAUSE_MAP_FORCE_DEALLOC
:
11246 case OMP_CLAUSE_MAP_FORCE_DEVICEPTR
:
11247 gcc_assert (is_gimple_omp_oacc_specifically (stmt
));
11250 gcc_unreachable ();
11254 case OMP_CLAUSE_TO
:
11255 case OMP_CLAUSE_FROM
:
11256 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
11257 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_TARGET
11258 && (gimple_omp_target_kind (stmt
)
11259 == GF_OMP_TARGET_KIND_UPDATE
));
11260 var
= OMP_CLAUSE_DECL (c
);
11263 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11264 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11269 if (DECL_SIZE (var
)
11270 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11272 tree var2
= DECL_VALUE_EXPR (var
);
11273 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11274 var2
= TREE_OPERAND (var2
, 0);
11275 gcc_assert (DECL_P (var2
));
11279 if (!maybe_lookup_field (var
, ctx
))
11284 x
= build_receiver_ref (var
, true, ctx
);
11285 tree new_var
= lookup_decl (var
, ctx
);
11286 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
);
11287 gcc_assert ((OMP_CLAUSE_MAP_KIND (c
)
11288 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11289 || TREE_CODE (TREE_TYPE (var
)) != ARRAY_TYPE
);
11290 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
11291 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11292 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11293 x
= build_simple_mem_ref (x
);
11294 SET_DECL_VALUE_EXPR (new_var
, x
);
11295 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11302 target_nesting_level
++;
11303 lower_omp (&tgt_body
, ctx
);
11304 target_nesting_level
--;
11306 else if (data_region
)
11307 lower_omp (&tgt_body
, ctx
);
11311 /* Declare all the variables created by mapping and the variables
11312 declared in the scope of the target body. */
11313 record_vars_into (ctx
->block_vars
, child_fn
);
11314 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11319 if (ctx
->record_type
)
11322 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11323 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11324 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11325 t
= make_tree_vec (3);
11326 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11327 TREE_VEC_ELT (t
, 1)
11328 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11329 ".omp_data_sizes");
11330 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11331 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11332 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11335 if (is_gimple_omp_oacc_specifically (stmt
))
11337 tkind_type
= short_unsigned_type_node
;
11342 tkind_type
= unsigned_char_type_node
;
11345 TREE_VEC_ELT (t
, 2)
11346 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11347 ".omp_data_kinds");
11348 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11349 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11350 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11351 gimple_omp_target_set_data_arg (stmt
, t
);
11353 vec
<constructor_elt
, va_gc
> *vsize
;
11354 vec
<constructor_elt
, va_gc
> *vkind
;
11355 vec_alloc (vsize
, map_cnt
);
11356 vec_alloc (vkind
, map_cnt
);
11357 unsigned int map_idx
= 0;
11359 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11360 switch (OMP_CLAUSE_CODE (c
))
11366 case OMP_CLAUSE_MAP
:
11367 case OMP_CLAUSE_TO
:
11368 case OMP_CLAUSE_FROM
:
11370 ovar
= OMP_CLAUSE_DECL (c
);
11371 if (!DECL_P (ovar
))
11373 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11374 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11376 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11377 == get_base_address (ovar
));
11378 nc
= OMP_CLAUSE_CHAIN (c
);
11379 ovar
= OMP_CLAUSE_DECL (nc
);
11383 tree x
= build_sender_ref (ovar
, ctx
);
11385 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11386 gimplify_assign (x
, v
, &ilist
);
11392 if (DECL_SIZE (ovar
)
11393 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11395 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11396 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11397 ovar2
= TREE_OPERAND (ovar2
, 0);
11398 gcc_assert (DECL_P (ovar2
));
11401 if (!maybe_lookup_field (ovar
, ctx
))
11405 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11406 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11407 talign
= DECL_ALIGN_UNIT (ovar
);
11410 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11411 tree x
= build_sender_ref (ovar
, ctx
);
11412 gcc_assert (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11413 || (OMP_CLAUSE_MAP_KIND (c
)
11414 != OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11415 || TREE_CODE (TREE_TYPE (ovar
)) != ARRAY_TYPE
);
11416 if (maybe_lookup_oacc_reduction (var
, ctx
))
11418 gcc_assert (offloaded
11419 && is_gimple_omp_oacc_specifically (stmt
));
11420 gimplify_assign (x
, var
, &ilist
);
11422 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11423 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
11424 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11425 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11427 gcc_assert (offloaded
);
11429 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
11430 mark_addressable (avar
);
11431 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11432 talign
= DECL_ALIGN_UNIT (avar
);
11433 avar
= build_fold_addr_expr (avar
);
11434 gimplify_assign (x
, avar
, &ilist
);
11436 else if (is_gimple_reg (var
))
11438 gcc_assert (offloaded
);
11439 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
11440 mark_addressable (avar
);
11441 enum omp_clause_map_kind map_kind
11442 = OMP_CLAUSE_MAP_KIND (c
);
11443 if ((!(map_kind
& OMP_CLAUSE_MAP_SPECIAL
)
11444 && (map_kind
& OMP_CLAUSE_MAP_TO
))
11445 || map_kind
== OMP_CLAUSE_MAP_POINTER
11446 || map_kind
== OMP_CLAUSE_MAP_TO_PSET
11447 || map_kind
== OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11448 gimplify_assign (avar
, var
, &ilist
);
11449 avar
= build_fold_addr_expr (avar
);
11450 gimplify_assign (x
, avar
, &ilist
);
11451 if (((!(map_kind
& OMP_CLAUSE_MAP_SPECIAL
)
11452 && (map_kind
& OMP_CLAUSE_MAP_FROM
))
11453 || map_kind
== OMP_CLAUSE_MAP_FORCE_DEVICEPTR
)
11454 && !TYPE_READONLY (TREE_TYPE (var
)))
11456 x
= build_sender_ref (ovar
, ctx
);
11457 x
= build_simple_mem_ref (x
);
11458 gimplify_assign (var
, x
, &olist
);
11463 var
= build_fold_addr_expr (var
);
11464 gimplify_assign (x
, var
, &ilist
);
11467 tree s
= OMP_CLAUSE_SIZE (c
);
11468 if (s
== NULL_TREE
)
11469 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11470 s
= fold_convert (size_type_node
, s
);
11471 tree purpose
= size_int (map_idx
++);
11472 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11473 if (TREE_CODE (s
) != INTEGER_CST
)
11474 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11476 unsigned HOST_WIDE_INT tkind
;
11477 switch (OMP_CLAUSE_CODE (c
))
11479 case OMP_CLAUSE_MAP
:
11480 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11482 case OMP_CLAUSE_TO
:
11483 tkind
= OMP_CLAUSE_MAP_TO
;
11485 case OMP_CLAUSE_FROM
:
11486 tkind
= OMP_CLAUSE_MAP_FROM
;
11489 gcc_unreachable ();
11491 gcc_assert (tkind
< (HOST_WIDE_INT_C (1U) << talign_shift
));
11492 talign
= ceil_log2 (talign
);
11493 tkind
|= talign
<< talign_shift
;
11494 gcc_assert (tkind
<= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11495 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11496 build_int_cstu (tkind_type
, tkind
));
11501 gcc_assert (map_idx
== map_cnt
);
11503 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11504 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11505 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11506 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11507 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11509 gimple_seq initlist
= NULL
;
11510 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11511 TREE_VEC_ELT (t
, 1)),
11512 &initlist
, true, NULL_TREE
);
11513 gimple_seq_add_seq (&ilist
, initlist
);
11515 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11517 TREE_THIS_VOLATILE (clobber
) = 1;
11518 gimple_seq_add_stmt (&olist
,
11519 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11523 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11524 TREE_THIS_VOLATILE (clobber
) = 1;
11525 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11529 /* Once all the expansions are done, sequence all the different
11530 fragments inside gimple_omp_body. */
11535 && ctx
->record_type
)
11537 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11538 /* fixup_child_record_type might have changed receiver_decl's type. */
11539 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11540 gimple_seq_add_stmt (&new_body
,
11541 gimple_build_assign (ctx
->receiver_decl
, t
));
11546 gimple_seq_add_seq (&new_body
, tgt_body
);
11547 new_body
= maybe_catch_exception (new_body
);
11549 else if (data_region
)
11550 new_body
= tgt_body
;
11551 if (offloaded
|| data_region
)
11553 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11554 gimple_omp_set_body (stmt
, new_body
);
11557 bind
= gimple_build_bind (NULL
, NULL
,
11558 tgt_bind
? gimple_bind_block (tgt_bind
)
11560 gsi_replace (gsi_p
, bind
, true);
11561 gimple_bind_add_seq (bind
, irlist
);
11562 gimple_bind_add_seq (bind
, ilist
);
11563 gimple_bind_add_stmt (bind
, stmt
);
11564 gimple_bind_add_seq (bind
, olist
);
11565 gimple_bind_add_seq (bind
, orlist
);
11567 pop_gimplify_context (NULL
);
11570 /* Expand code for an OpenMP teams directive. */
11573 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11575 gimple teams_stmt
= gsi_stmt (*gsi_p
);
11576 push_gimplify_context ();
11578 tree block
= make_node (BLOCK
);
11579 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
11580 gsi_replace (gsi_p
, bind
, true);
11581 gimple_seq bind_body
= NULL
;
11582 gimple_seq dlist
= NULL
;
11583 gimple_seq olist
= NULL
;
11585 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11586 OMP_CLAUSE_NUM_TEAMS
);
11587 if (num_teams
== NULL_TREE
)
11588 num_teams
= build_int_cst (unsigned_type_node
, 0);
11591 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11592 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11593 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11595 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11596 OMP_CLAUSE_THREAD_LIMIT
);
11597 if (thread_limit
== NULL_TREE
)
11598 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11601 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11602 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11603 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11607 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11608 &bind_body
, &dlist
, ctx
, NULL
);
11609 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11610 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11611 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11613 location_t loc
= gimple_location (teams_stmt
);
11614 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11615 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11616 gimple_set_location (call
, loc
);
11617 gimple_seq_add_stmt (&bind_body
, call
);
11619 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11620 gimple_omp_set_body (teams_stmt
, NULL
);
11621 gimple_seq_add_seq (&bind_body
, olist
);
11622 gimple_seq_add_seq (&bind_body
, dlist
);
11623 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11624 gimple_bind_set_body (bind
, bind_body
);
11626 pop_gimplify_context (bind
);
11628 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11629 BLOCK_VARS (block
) = ctx
->block_vars
;
11630 if (BLOCK_VARS (block
))
11631 TREE_USED (block
) = 1;
11635 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11636 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11637 of OpenMP context, but with task_shared_vars set. */
11640 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11645 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11646 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11649 if (task_shared_vars
11651 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11654 /* If a global variable has been privatized, TREE_CONSTANT on
11655 ADDR_EXPR might be wrong. */
11656 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11657 recompute_tree_invariant_for_addr_expr (t
);
11659 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11664 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11666 gimple stmt
= gsi_stmt (*gsi_p
);
11667 struct walk_stmt_info wi
;
11669 if (gimple_has_location (stmt
))
11670 input_location
= gimple_location (stmt
);
11672 if (task_shared_vars
)
11673 memset (&wi
, '\0', sizeof (wi
));
11675 /* If we have issued syntax errors, avoid doing any heavy lifting.
11676 Just replace the OpenMP directives with a NOP to avoid
11677 confusing RTL expansion. */
11678 if (seen_error () && is_gimple_omp (stmt
))
11680 gsi_replace (gsi_p
, gimple_build_nop (), true);
11684 switch (gimple_code (stmt
))
11687 if ((ctx
|| task_shared_vars
)
11688 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
11689 ctx
? NULL
: &wi
, NULL
)
11690 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
11691 ctx
? NULL
: &wi
, NULL
)))
11692 gimple_regimplify_operands (stmt
, gsi_p
);
11695 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
11697 case GIMPLE_EH_FILTER
:
11698 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11701 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11702 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11704 case GIMPLE_TRANSACTION
:
11705 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
11708 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
11710 case GIMPLE_OMP_PARALLEL
:
11711 case GIMPLE_OMP_TASK
:
11712 ctx
= maybe_lookup_ctx (stmt
);
11714 if (ctx
->cancellable
)
11715 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11716 lower_omp_taskreg (gsi_p
, ctx
);
11718 case GIMPLE_OMP_FOR
:
11719 ctx
= maybe_lookup_ctx (stmt
);
11721 if (ctx
->cancellable
)
11722 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11723 lower_omp_for (gsi_p
, ctx
);
11725 case GIMPLE_OMP_SECTIONS
:
11726 ctx
= maybe_lookup_ctx (stmt
);
11728 if (ctx
->cancellable
)
11729 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11730 lower_omp_sections (gsi_p
, ctx
);
11732 case GIMPLE_OMP_SINGLE
:
11733 ctx
= maybe_lookup_ctx (stmt
);
11735 lower_omp_single (gsi_p
, ctx
);
11737 case GIMPLE_OMP_MASTER
:
11738 ctx
= maybe_lookup_ctx (stmt
);
11740 lower_omp_master (gsi_p
, ctx
);
11742 case GIMPLE_OMP_TASKGROUP
:
11743 ctx
= maybe_lookup_ctx (stmt
);
11745 lower_omp_taskgroup (gsi_p
, ctx
);
11747 case GIMPLE_OMP_ORDERED
:
11748 ctx
= maybe_lookup_ctx (stmt
);
11750 lower_omp_ordered (gsi_p
, ctx
);
11752 case GIMPLE_OMP_CRITICAL
:
11753 ctx
= maybe_lookup_ctx (stmt
);
11755 lower_omp_critical (gsi_p
, ctx
);
11757 case GIMPLE_OMP_ATOMIC_LOAD
:
11758 if ((ctx
|| task_shared_vars
)
11759 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
11760 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11761 gimple_regimplify_operands (stmt
, gsi_p
);
11763 case GIMPLE_OMP_TARGET
:
11764 ctx
= maybe_lookup_ctx (stmt
);
11766 if (is_gimple_omp_oacc_specifically (stmt
))
11767 gcc_assert (!ctx
->cancellable
);
11768 lower_omp_target (gsi_p
, ctx
);
11770 case GIMPLE_OMP_TEAMS
:
11771 ctx
= maybe_lookup_ctx (stmt
);
11773 lower_omp_teams (gsi_p
, ctx
);
11777 fndecl
= gimple_call_fndecl (stmt
);
11779 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11780 switch (DECL_FUNCTION_CODE (fndecl
))
11782 case BUILT_IN_GOMP_BARRIER
:
11786 case BUILT_IN_GOMP_CANCEL
:
11787 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11790 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11791 cctx
= cctx
->outer
;
11792 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
11793 if (!cctx
->cancellable
)
11795 if (DECL_FUNCTION_CODE (fndecl
)
11796 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11798 stmt
= gimple_build_nop ();
11799 gsi_replace (gsi_p
, stmt
, false);
11803 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11805 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11806 gimple_call_set_fndecl (stmt
, fndecl
);
11807 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
11810 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
11811 gimple_call_set_lhs (stmt
, lhs
);
11812 tree fallthru_label
;
11813 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11815 g
= gimple_build_label (fallthru_label
);
11816 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11817 g
= gimple_build_cond (NE_EXPR
, lhs
,
11818 fold_convert (TREE_TYPE (lhs
),
11819 boolean_false_node
),
11820 cctx
->cancel_label
, fallthru_label
);
11821 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11828 if ((ctx
|| task_shared_vars
)
11829 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11832 /* Just remove clobbers, this should happen only if we have
11833 "privatized" local addressable variables in SIMD regions,
11834 the clobber isn't needed in that case and gimplifying address
11835 of the ARRAY_REF into a pointer and creating MEM_REF based
11836 clobber would create worse code than we get with the clobber
11838 if (gimple_clobber_p (stmt
))
11840 gsi_replace (gsi_p
, gimple_build_nop (), true);
11843 gimple_regimplify_operands (stmt
, gsi_p
);
11850 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11852 location_t saved_location
= input_location
;
11853 gimple_stmt_iterator gsi
;
11854 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11855 lower_omp_1 (&gsi
, ctx
);
11856 /* During gimplification, we haven't folded statments inside offloading
11857 regions (gimplify.c:maybe_fold_stmt); do that now. */
11858 if (target_nesting_level
)
11859 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11861 input_location
= saved_location
;
11864 /* Main entry point. */
11866 static unsigned int
11867 execute_lower_omp (void)
11873 /* This pass always runs, to provide PROP_gimple_lomp.
11874 But often, there is nothing to do. */
11875 if (flag_openacc
== 0 && flag_openmp
== 0 && flag_openmp_simd
== 0
11876 && flag_cilkplus
== 0)
11879 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11880 delete_omp_context
);
11882 body
= gimple_body (current_function_decl
);
11883 scan_omp (&body
, NULL
);
11884 gcc_assert (taskreg_nesting_level
== 0);
11885 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11886 finish_taskreg_scan (ctx
);
11887 taskreg_contexts
.release ();
11889 if (all_contexts
->root
)
11891 if (task_shared_vars
)
11892 push_gimplify_context ();
11893 lower_omp (&body
, NULL
);
11894 if (task_shared_vars
)
11895 pop_gimplify_context (NULL
);
11900 splay_tree_delete (all_contexts
);
11901 all_contexts
= NULL
;
11903 BITMAP_FREE (task_shared_vars
);
11909 const pass_data pass_data_lower_omp
=
11911 GIMPLE_PASS
, /* type */
11912 "omplower", /* name */
11913 OPTGROUP_NONE
, /* optinfo_flags */
11914 TV_NONE
, /* tv_id */
11915 PROP_gimple_any
, /* properties_required */
11916 PROP_gimple_lomp
, /* properties_provided */
11917 0, /* properties_destroyed */
11918 0, /* todo_flags_start */
11919 0, /* todo_flags_finish */
11922 class pass_lower_omp
: public gimple_opt_pass
11925 pass_lower_omp (gcc::context
*ctxt
)
11926 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11929 /* opt_pass methods: */
11930 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11932 }; // class pass_lower_omp
11934 } // anon namespace
11937 make_pass_lower_omp (gcc::context
*ctxt
)
11939 return new pass_lower_omp (ctxt
);
11942 /* The following is a utility to diagnose structured block violations.
11943 It is not part of the "omplower" pass, as that's invoked too late. It
11944 should be invoked by the respective front ends after gimplification. */
11946 static splay_tree all_labels
;
11948 /* Check for mismatched contexts and generate an error if needed. Return
11949 true if an error is detected. */
11952 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11953 gimple branch_ctx
, gimple label_ctx
)
11955 gcc_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11956 gcc_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11958 if (label_ctx
== branch_ctx
)
11961 const char* kind
= NULL
;
11966 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
11967 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
11969 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
11970 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
11971 kind
= "Cilk Plus";
11975 if ((branch_ctx
&& is_gimple_omp_oacc_specifically (branch_ctx
))
11976 || (label_ctx
&& is_gimple_omp_oacc_specifically (label_ctx
)))
11978 gcc_assert (kind
== NULL
);
11984 gcc_assert (flag_openmp
);
11989 Previously we kept track of the label's entire context in diagnose_sb_[12]
11990 so we could traverse it and issue a correct "exit" or "enter" error
11991 message upon a structured block violation.
11993 We built the context by building a list with tree_cons'ing, but there is
11994 no easy counterpart in gimple tuples. It seems like far too much work
11995 for issuing exit/enter error messages. If someone really misses the
11996 distinct error message... patches welcome.
12000 /* Try to avoid confusing the user by producing and error message
12001 with correct "exit" or "enter" verbiage. We prefer "exit"
12002 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12003 if (branch_ctx
== NULL
)
12009 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12014 label_ctx
= TREE_CHAIN (label_ctx
);
12019 error ("invalid exit from %s structured block", kind
);
12021 error ("invalid entry to %s structured block", kind
);
12024 /* If it's obvious we have an invalid entry, be specific about the error. */
12025 if (branch_ctx
== NULL
)
12026 error ("invalid entry to %s structured block", kind
);
12029 /* Otherwise, be vague and lazy, but efficient. */
12030 error ("invalid branch to/from %s structured block", kind
);
12033 gsi_replace (gsi_p
, gimple_build_nop (), false);
12037 /* Pass 1: Create a minimal tree of structured blocks, and record
12038 where each label is found. */
12041 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12042 struct walk_stmt_info
*wi
)
12044 gimple context
= (gimple
) wi
->info
;
12045 gimple inner_context
;
12046 gimple stmt
= gsi_stmt (*gsi_p
);
12048 *handled_ops_p
= true;
12050 switch (gimple_code (stmt
))
12054 case GIMPLE_OMP_PARALLEL
:
12055 case GIMPLE_OMP_TASK
:
12056 case GIMPLE_OMP_SECTIONS
:
12057 case GIMPLE_OMP_SINGLE
:
12058 case GIMPLE_OMP_SECTION
:
12059 case GIMPLE_OMP_MASTER
:
12060 case GIMPLE_OMP_ORDERED
:
12061 case GIMPLE_OMP_CRITICAL
:
12062 case GIMPLE_OMP_TARGET
:
12063 case GIMPLE_OMP_TEAMS
:
12064 case GIMPLE_OMP_TASKGROUP
:
12065 /* The minimal context here is just the current construct. */
12066 inner_context
= stmt
;
12067 wi
->info
= inner_context
;
12068 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12069 wi
->info
= context
;
12072 case GIMPLE_OMP_FOR
:
12073 inner_context
= stmt
;
12074 wi
->info
= inner_context
;
12075 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12077 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12078 diagnose_sb_1
, NULL
, wi
);
12079 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12080 wi
->info
= context
;
12084 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
12085 (splay_tree_value
) context
);
12095 /* Pass 2: Check each branch and see if its context differs from that of
12096 the destination label's context. */
12099 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12100 struct walk_stmt_info
*wi
)
12102 gimple context
= (gimple
) wi
->info
;
12104 gimple stmt
= gsi_stmt (*gsi_p
);
12106 *handled_ops_p
= true;
12108 switch (gimple_code (stmt
))
12112 case GIMPLE_OMP_PARALLEL
:
12113 case GIMPLE_OMP_TASK
:
12114 case GIMPLE_OMP_SECTIONS
:
12115 case GIMPLE_OMP_SINGLE
:
12116 case GIMPLE_OMP_SECTION
:
12117 case GIMPLE_OMP_MASTER
:
12118 case GIMPLE_OMP_ORDERED
:
12119 case GIMPLE_OMP_CRITICAL
:
12120 case GIMPLE_OMP_TARGET
:
12121 case GIMPLE_OMP_TEAMS
:
12122 case GIMPLE_OMP_TASKGROUP
:
12124 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12125 wi
->info
= context
;
12128 case GIMPLE_OMP_FOR
:
12130 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12132 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12133 diagnose_sb_2
, NULL
, wi
);
12134 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12135 wi
->info
= context
;
12140 tree lab
= gimple_cond_true_label (stmt
);
12143 n
= splay_tree_lookup (all_labels
,
12144 (splay_tree_key
) lab
);
12145 diagnose_sb_0 (gsi_p
, context
,
12146 n
? (gimple
) n
->value
: NULL
);
12148 lab
= gimple_cond_false_label (stmt
);
12151 n
= splay_tree_lookup (all_labels
,
12152 (splay_tree_key
) lab
);
12153 diagnose_sb_0 (gsi_p
, context
,
12154 n
? (gimple
) n
->value
: NULL
);
12161 tree lab
= gimple_goto_dest (stmt
);
12162 if (TREE_CODE (lab
) != LABEL_DECL
)
12165 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12166 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12170 case GIMPLE_SWITCH
:
12173 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
12175 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
12176 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12177 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12183 case GIMPLE_RETURN
:
12184 diagnose_sb_0 (gsi_p
, context
, NULL
);
12194 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12197 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12200 gimple last
= last_stmt (bb
);
12201 enum gimple_code code
= gimple_code (last
);
12202 struct omp_region
*cur_region
= *region
;
12203 bool fallthru
= false;
12207 case GIMPLE_OMP_PARALLEL
:
12208 case GIMPLE_OMP_TASK
:
12209 case GIMPLE_OMP_FOR
:
12210 case GIMPLE_OMP_SINGLE
:
12211 case GIMPLE_OMP_TEAMS
:
12212 case GIMPLE_OMP_MASTER
:
12213 case GIMPLE_OMP_TASKGROUP
:
12214 case GIMPLE_OMP_ORDERED
:
12215 case GIMPLE_OMP_CRITICAL
:
12216 case GIMPLE_OMP_SECTION
:
12217 cur_region
= new_omp_region (bb
, code
, cur_region
);
12221 case GIMPLE_OMP_TARGET
:
12222 cur_region
= new_omp_region (bb
, code
, cur_region
);
12224 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
12225 || gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_OACC_UPDATE
12226 || (gimple_omp_target_kind (last
)
12227 == GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
))
12228 cur_region
= cur_region
->outer
;
12231 case GIMPLE_OMP_SECTIONS
:
12232 cur_region
= new_omp_region (bb
, code
, cur_region
);
12236 case GIMPLE_OMP_SECTIONS_SWITCH
:
12240 case GIMPLE_OMP_ATOMIC_LOAD
:
12241 case GIMPLE_OMP_ATOMIC_STORE
:
12245 case GIMPLE_OMP_RETURN
:
12246 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12247 somewhere other than the next block. This will be
12249 cur_region
->exit
= bb
;
12250 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12251 cur_region
= cur_region
->outer
;
12254 case GIMPLE_OMP_CONTINUE
:
12255 cur_region
->cont
= bb
;
12256 switch (cur_region
->type
)
12258 case GIMPLE_OMP_FOR
:
12259 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12260 succs edges as abnormal to prevent splitting
12262 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12263 /* Make the loopback edge. */
12264 make_edge (bb
, single_succ (cur_region
->entry
),
12267 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12268 corresponds to the case that the body of the loop
12269 is not executed at all. */
12270 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12271 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12275 case GIMPLE_OMP_SECTIONS
:
12276 /* Wire up the edges into and out of the nested sections. */
12278 basic_block switch_bb
= single_succ (cur_region
->entry
);
12280 struct omp_region
*i
;
12281 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12283 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12284 make_edge (switch_bb
, i
->entry
, 0);
12285 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12288 /* Make the loopback edge to the block with
12289 GIMPLE_OMP_SECTIONS_SWITCH. */
12290 make_edge (bb
, switch_bb
, 0);
12292 /* Make the edge from the switch to exit. */
12293 make_edge (switch_bb
, bb
->next_bb
, 0);
12299 gcc_unreachable ();
12304 gcc_unreachable ();
12307 if (*region
!= cur_region
)
12309 *region
= cur_region
;
12311 *region_idx
= cur_region
->entry
->index
;
12319 static unsigned int
12320 diagnose_omp_structured_block_errors (void)
12322 struct walk_stmt_info wi
;
12323 gimple_seq body
= gimple_body (current_function_decl
);
12325 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12327 memset (&wi
, 0, sizeof (wi
));
12328 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12330 memset (&wi
, 0, sizeof (wi
));
12331 wi
.want_locations
= true;
12332 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12334 gimple_set_body (current_function_decl
, body
);
12336 splay_tree_delete (all_labels
);
12344 const pass_data pass_data_diagnose_omp_blocks
=
12346 GIMPLE_PASS
, /* type */
12347 "*diagnose_omp_blocks", /* name */
12348 OPTGROUP_NONE
, /* optinfo_flags */
12349 TV_NONE
, /* tv_id */
12350 PROP_gimple_any
, /* properties_required */
12351 0, /* properties_provided */
12352 0, /* properties_destroyed */
12353 0, /* todo_flags_start */
12354 0, /* todo_flags_finish */
12357 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12360 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12361 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12364 /* opt_pass methods: */
12365 virtual bool gate (function
*)
12367 return flag_openacc
|| flag_openmp
|| flag_cilkplus
;
12369 virtual unsigned int execute (function
*)
12371 return diagnose_omp_structured_block_errors ();
12374 }; // class pass_diagnose_omp_blocks
12376 } // anon namespace
12379 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12381 return new pass_diagnose_omp_blocks (ctxt
);
12384 /* SIMD clone supporting code. */
12386 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12387 of arguments to reserve space for. */
12389 static struct cgraph_simd_clone
*
12390 simd_clone_struct_alloc (int nargs
)
12392 struct cgraph_simd_clone
*clone_info
;
12393 size_t len
= (sizeof (struct cgraph_simd_clone
)
12394 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12395 clone_info
= (struct cgraph_simd_clone
*)
12396 ggc_internal_cleared_alloc (len
);
12400 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12403 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12404 struct cgraph_simd_clone
*from
)
12406 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12407 + ((from
->nargs
- from
->inbranch
)
12408 * sizeof (struct cgraph_simd_clone_arg
))));
12411 /* Return vector of parameter types of function FNDECL. This uses
12412 TYPE_ARG_TYPES if available, otherwise falls back to types of
12413 DECL_ARGUMENTS types. */
12416 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12418 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12419 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12420 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12423 FOR_EACH_VEC_ELT (args
, i
, arg
)
12424 args
[i
] = TREE_TYPE (args
[i
]);
12428 /* Given a simd function in NODE, extract the simd specific
12429 information from the OMP clauses passed in CLAUSES, and return
12430 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12431 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12432 otherwise set to FALSE. */
12434 static struct cgraph_simd_clone
*
12435 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12436 bool *inbranch_specified
)
12438 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12441 *inbranch_specified
= false;
12443 n
= args
.length ();
12444 if (n
> 0 && args
.last () == void_type_node
)
12447 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12448 be cloned have a distinctive artificial label in addition to "omp
12452 && lookup_attribute ("cilk simd function",
12453 DECL_ATTRIBUTES (node
->decl
)));
12455 /* Allocate one more than needed just in case this is an in-branch
12456 clone which will require a mask argument. */
12457 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12458 clone_info
->nargs
= n
;
12459 clone_info
->cilk_elemental
= cilk_clone
;
12466 clauses
= TREE_VALUE (clauses
);
12467 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12470 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12472 switch (OMP_CLAUSE_CODE (t
))
12474 case OMP_CLAUSE_INBRANCH
:
12475 clone_info
->inbranch
= 1;
12476 *inbranch_specified
= true;
12478 case OMP_CLAUSE_NOTINBRANCH
:
12479 clone_info
->inbranch
= 0;
12480 *inbranch_specified
= true;
12482 case OMP_CLAUSE_SIMDLEN
:
12483 clone_info
->simdlen
12484 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12486 case OMP_CLAUSE_LINEAR
:
12488 tree decl
= OMP_CLAUSE_DECL (t
);
12489 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12490 int argno
= TREE_INT_CST_LOW (decl
);
12491 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12493 clone_info
->args
[argno
].arg_type
12494 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12495 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12496 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12497 && clone_info
->args
[argno
].linear_step
< n
);
12501 if (POINTER_TYPE_P (args
[argno
]))
12502 step
= fold_convert (ssizetype
, step
);
12503 if (!tree_fits_shwi_p (step
))
12505 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12506 "ignoring large linear step");
12510 else if (integer_zerop (step
))
12512 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12513 "ignoring zero linear step");
12519 clone_info
->args
[argno
].arg_type
12520 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12521 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12526 case OMP_CLAUSE_UNIFORM
:
12528 tree decl
= OMP_CLAUSE_DECL (t
);
12529 int argno
= tree_to_uhwi (decl
);
12530 clone_info
->args
[argno
].arg_type
12531 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12534 case OMP_CLAUSE_ALIGNED
:
12536 tree decl
= OMP_CLAUSE_DECL (t
);
12537 int argno
= tree_to_uhwi (decl
);
12538 clone_info
->args
[argno
].alignment
12539 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12550 /* Given a SIMD clone in NODE, calculate the characteristic data
12551 type and return the coresponding type. The characteristic data
12552 type is computed as described in the Intel Vector ABI. */
12555 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12556 struct cgraph_simd_clone
*clone_info
)
12558 tree type
= integer_type_node
;
12559 tree fndecl
= node
->decl
;
12561 /* a) For non-void function, the characteristic data type is the
12563 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12564 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12566 /* b) If the function has any non-uniform, non-linear parameters,
12567 then the characteristic data type is the type of the first
12571 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12572 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12573 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12581 /* c) If the characteristic data type determined by a) or b) above
12582 is struct, union, or class type which is pass-by-value (except
12583 for the type that maps to the built-in complex data type), the
12584 characteristic data type is int. */
12585 if (RECORD_OR_UNION_TYPE_P (type
)
12586 && !aggregate_value_p (type
, NULL
)
12587 && TREE_CODE (type
) != COMPLEX_TYPE
)
12588 return integer_type_node
;
12590 /* d) If none of the above three classes is applicable, the
12591 characteristic data type is int. */
12595 /* e) For Intel Xeon Phi native and offload compilation, if the
12596 resulting characteristic data type is 8-bit or 16-bit integer
12597 data type, the characteristic data type is int. */
12598 /* Well, we don't handle Xeon Phi yet. */
12602 simd_clone_mangle (struct cgraph_node
*node
,
12603 struct cgraph_simd_clone
*clone_info
)
12605 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12606 char mask
= clone_info
->inbranch
? 'M' : 'N';
12607 unsigned int simdlen
= clone_info
->simdlen
;
12611 gcc_assert (vecsize_mangle
&& simdlen
);
12613 pp_string (&pp
, "_ZGV");
12614 pp_character (&pp
, vecsize_mangle
);
12615 pp_character (&pp
, mask
);
12616 pp_decimal_int (&pp
, simdlen
);
12618 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12620 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12622 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12623 pp_character (&pp
, 'u');
12624 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12626 gcc_assert (arg
.linear_step
!= 0);
12627 pp_character (&pp
, 'l');
12628 if (arg
.linear_step
> 1)
12629 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12630 else if (arg
.linear_step
< 0)
12632 pp_character (&pp
, 'n');
12633 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12637 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12639 pp_character (&pp
, 's');
12640 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12643 pp_character (&pp
, 'v');
12646 pp_character (&pp
, 'a');
12647 pp_decimal_int (&pp
, arg
.alignment
);
12651 pp_underscore (&pp
);
12653 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
12654 const char *str
= pp_formatted_text (&pp
);
12656 /* If there already is a SIMD clone with the same mangled name, don't
12657 add another one. This can happen e.g. for
12658 #pragma omp declare simd
12659 #pragma omp declare simd simdlen(8)
12660 int foo (int, int);
12661 if the simdlen is assumed to be 8 for the first one, etc. */
12662 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12663 clone
= clone
->simdclone
->next_clone
)
12664 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12668 return get_identifier (str
);
12671 /* Create a simd clone of OLD_NODE and return it. */
12673 static struct cgraph_node
*
12674 simd_clone_create (struct cgraph_node
*old_node
)
12676 struct cgraph_node
*new_node
;
12677 if (old_node
->definition
)
12679 if (!old_node
->has_gimple_body_p ())
12681 old_node
->get_body ();
12682 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12688 tree old_decl
= old_node
->decl
;
12689 tree new_decl
= copy_node (old_node
->decl
);
12690 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12691 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12692 SET_DECL_RTL (new_decl
, NULL
);
12693 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12694 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12695 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12696 symtab
->call_cgraph_insertion_hooks (new_node
);
12698 if (new_node
== NULL
)
12701 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12703 /* The function cgraph_function_versioning () will force the new
12704 symbol local. Undo this, and inherit external visability from
12706 new_node
->local
.local
= old_node
->local
.local
;
12707 new_node
->externally_visible
= old_node
->externally_visible
;
12712 /* Adjust the return type of the given function to its appropriate
12713 vector counterpart. Returns a simd array to be used throughout the
12714 function as a return value. */
12717 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12719 tree fndecl
= node
->decl
;
12720 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12721 unsigned int veclen
;
12724 /* Adjust the function return type. */
12725 if (orig_rettype
== void_type_node
)
12727 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12728 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
12729 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
12730 veclen
= node
->simdclone
->vecsize_int
;
12732 veclen
= node
->simdclone
->vecsize_float
;
12733 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
12734 if (veclen
> node
->simdclone
->simdlen
)
12735 veclen
= node
->simdclone
->simdlen
;
12736 if (veclen
== node
->simdclone
->simdlen
)
12737 TREE_TYPE (TREE_TYPE (fndecl
))
12738 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
12739 node
->simdclone
->simdlen
);
12742 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
12743 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12744 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12746 if (!node
->definition
)
12749 t
= DECL_RESULT (fndecl
);
12750 /* Adjust the DECL_RESULT. */
12751 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12752 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12755 tree atype
= build_array_type_nelts (orig_rettype
,
12756 node
->simdclone
->simdlen
);
12757 if (veclen
!= node
->simdclone
->simdlen
)
12758 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12760 /* Set up a SIMD array to use as the return value. */
12761 tree retval
= create_tmp_var_raw (atype
, "retval");
12762 gimple_add_tmp_var (retval
);
12766 /* Each vector argument has a corresponding array to be used locally
12767 as part of the eventual loop. Create such temporary array and
12770 PREFIX is the prefix to be used for the temporary.
12772 TYPE is the inner element type.
12774 SIMDLEN is the number of elements. */
12777 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12779 tree atype
= build_array_type_nelts (type
, simdlen
);
12780 tree avar
= create_tmp_var_raw (atype
, prefix
);
12781 gimple_add_tmp_var (avar
);
12785 /* Modify the function argument types to their corresponding vector
12786 counterparts if appropriate. Also, create one array for each simd
12787 argument to be used locally when using the function arguments as
12790 NODE is the function whose arguments are to be adjusted.
12792 Returns an adjustment vector that will be filled describing how the
12793 argument types will be adjusted. */
12795 static ipa_parm_adjustment_vec
12796 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12799 ipa_parm_adjustment_vec adjustments
;
12801 if (node
->definition
)
12802 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12804 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12805 adjustments
.create (args
.length ());
12806 unsigned i
, j
, veclen
;
12807 struct ipa_parm_adjustment adj
;
12808 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12810 memset (&adj
, 0, sizeof (adj
));
12811 tree parm
= args
[i
];
12812 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12813 adj
.base_index
= i
;
12816 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12817 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12819 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12821 /* No adjustment necessary for scalar arguments. */
12822 adj
.op
= IPA_PARM_OP_COPY
;
12826 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12827 veclen
= node
->simdclone
->vecsize_int
;
12829 veclen
= node
->simdclone
->vecsize_float
;
12830 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12831 if (veclen
> node
->simdclone
->simdlen
)
12832 veclen
= node
->simdclone
->simdlen
;
12833 adj
.arg_prefix
= "simd";
12834 adj
.type
= build_vector_type (parm_type
, veclen
);
12835 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12836 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12838 adjustments
.safe_push (adj
);
12841 memset (&adj
, 0, sizeof (adj
));
12842 adj
.op
= IPA_PARM_OP_NEW
;
12843 adj
.arg_prefix
= "simd";
12844 adj
.base_index
= i
;
12845 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12849 if (node
->definition
)
12850 node
->simdclone
->args
[i
].simd_array
12851 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12852 parm_type
, node
->simdclone
->simdlen
);
12854 adjustments
.safe_push (adj
);
12857 if (node
->simdclone
->inbranch
)
12860 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12863 memset (&adj
, 0, sizeof (adj
));
12864 adj
.op
= IPA_PARM_OP_NEW
;
12865 adj
.arg_prefix
= "mask";
12867 adj
.base_index
= i
;
12868 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12869 veclen
= node
->simdclone
->vecsize_int
;
12871 veclen
= node
->simdclone
->vecsize_float
;
12872 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12873 if (veclen
> node
->simdclone
->simdlen
)
12874 veclen
= node
->simdclone
->simdlen
;
12875 adj
.type
= build_vector_type (base_type
, veclen
);
12876 adjustments
.safe_push (adj
);
12878 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12879 adjustments
.safe_push (adj
);
12881 /* We have previously allocated one extra entry for the mask. Use
12883 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12885 if (node
->definition
)
12887 sc
->args
[i
].orig_arg
12888 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12889 sc
->args
[i
].simd_array
12890 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12892 sc
->args
[i
].orig_type
= base_type
;
12893 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12896 if (node
->definition
)
12897 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12900 tree new_arg_types
= NULL_TREE
, new_reversed
;
12901 bool last_parm_void
= false;
12902 if (args
.length () > 0 && args
.last () == void_type_node
)
12903 last_parm_void
= true;
12905 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12906 j
= adjustments
.length ();
12907 for (i
= 0; i
< j
; i
++)
12909 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12911 if (adj
->op
== IPA_PARM_OP_COPY
)
12912 ptype
= args
[adj
->base_index
];
12915 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12917 new_reversed
= nreverse (new_arg_types
);
12918 if (last_parm_void
)
12921 TREE_CHAIN (new_arg_types
) = void_list_node
;
12923 new_reversed
= void_list_node
;
12926 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12927 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12928 TREE_TYPE (node
->decl
) = new_type
;
12930 adjustments
.release ();
12933 return adjustments
;
12936 /* Initialize and copy the function arguments in NODE to their
12937 corresponding local simd arrays. Returns a fresh gimple_seq with
12938 the instruction sequence generated. */
12941 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
12942 ipa_parm_adjustment_vec adjustments
)
12944 gimple_seq seq
= NULL
;
12945 unsigned i
= 0, j
= 0, k
;
12947 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
12949 arg
= DECL_CHAIN (arg
), i
++, j
++)
12951 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
12954 node
->simdclone
->args
[i
].vector_arg
= arg
;
12956 tree array
= node
->simdclone
->args
[i
].simd_array
;
12957 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
12959 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12960 tree ptr
= build_fold_addr_expr (array
);
12961 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12962 build_int_cst (ptype
, 0));
12963 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12964 gimplify_and_add (t
, &seq
);
12968 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
12969 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12970 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
12972 tree ptr
= build_fold_addr_expr (array
);
12976 arg
= DECL_CHAIN (arg
);
12980 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
12981 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12982 build_int_cst (ptype
, k
* elemsize
));
12983 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12984 gimplify_and_add (t
, &seq
);
12991 /* Callback info for ipa_simd_modify_stmt_ops below. */
12993 struct modify_stmt_info
{
12994 ipa_parm_adjustment_vec adjustments
;
12996 /* True if the parent statement was modified by
12997 ipa_simd_modify_stmt_ops. */
13001 /* Callback for walk_gimple_op.
13003 Adjust operands from a given statement as specified in the
13004 adjustments vector in the callback data. */
13007 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13009 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13010 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13011 tree
*orig_tp
= tp
;
13012 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13013 tp
= &TREE_OPERAND (*tp
, 0);
13014 struct ipa_parm_adjustment
*cand
= NULL
;
13015 if (TREE_CODE (*tp
) == PARM_DECL
)
13016 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13020 *walk_subtrees
= 0;
13023 tree repl
= NULL_TREE
;
13025 repl
= unshare_expr (cand
->new_decl
);
13030 *walk_subtrees
= 0;
13031 bool modified
= info
->modified
;
13032 info
->modified
= false;
13033 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13034 if (!info
->modified
)
13036 info
->modified
= modified
;
13039 info
->modified
= modified
;
13048 repl
= build_fold_addr_expr (repl
);
13050 if (is_gimple_debug (info
->stmt
))
13052 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13053 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13054 DECL_ARTIFICIAL (vexpr
) = 1;
13055 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13056 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13061 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
13063 repl
= gimple_assign_lhs (stmt
);
13065 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13066 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13069 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13071 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13077 info
->modified
= true;
13081 /* Traverse the function body and perform all modifications as
13082 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13083 modified such that the replacement/reduction value will now be an
13084 offset into the corresponding simd_array.
13086 This function will replace all function argument uses with their
13087 corresponding simd array elements, and ajust the return values
13091 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13092 ipa_parm_adjustment_vec adjustments
,
13093 tree retval_array
, tree iter
)
13096 unsigned int i
, j
, l
;
13098 /* Re-use the adjustments array, but this time use it to replace
13099 every function argument use to an offset into the corresponding
13101 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13103 if (!node
->simdclone
->args
[i
].vector_arg
)
13106 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13107 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13108 adjustments
[j
].new_decl
13109 = build4 (ARRAY_REF
,
13111 node
->simdclone
->args
[i
].simd_array
,
13113 NULL_TREE
, NULL_TREE
);
13114 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13115 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13116 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13119 l
= adjustments
.length ();
13120 for (i
= 1; i
< num_ssa_names
; i
++)
13122 tree name
= ssa_name (i
);
13124 && SSA_NAME_VAR (name
)
13125 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13127 for (j
= 0; j
< l
; j
++)
13128 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13129 && adjustments
[j
].new_decl
)
13132 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13135 = copy_var_decl (adjustments
[j
].base
,
13136 DECL_NAME (adjustments
[j
].base
),
13137 TREE_TYPE (adjustments
[j
].base
));
13138 adjustments
[j
].new_ssa_base
= base_var
;
13141 base_var
= adjustments
[j
].new_ssa_base
;
13142 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13144 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13145 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13146 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13147 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13148 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13149 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13150 gimple stmt
= gimple_build_assign (name
, new_decl
);
13151 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13154 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13159 struct modify_stmt_info info
;
13160 info
.adjustments
= adjustments
;
13162 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13164 gimple_stmt_iterator gsi
;
13166 gsi
= gsi_start_bb (bb
);
13167 while (!gsi_end_p (gsi
))
13169 gimple stmt
= gsi_stmt (gsi
);
13171 struct walk_stmt_info wi
;
13173 memset (&wi
, 0, sizeof (wi
));
13174 info
.modified
= false;
13176 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13178 if (gimple_code (stmt
) == GIMPLE_RETURN
)
13180 tree retval
= gimple_return_retval (stmt
);
13183 gsi_remove (&gsi
, true);
13187 /* Replace `return foo' with `retval_array[iter] = foo'. */
13188 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13189 retval_array
, iter
, NULL
, NULL
);
13190 stmt
= gimple_build_assign (ref
, retval
);
13191 gsi_replace (&gsi
, stmt
, true);
13192 info
.modified
= true;
13197 update_stmt (stmt
);
13198 if (maybe_clean_eh_stmt (stmt
))
13199 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13206 /* Adjust the argument types in NODE to their appropriate vector
13210 simd_clone_adjust (struct cgraph_node
*node
)
13212 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13214 targetm
.simd_clone
.adjust (node
);
13216 tree retval
= simd_clone_adjust_return_type (node
);
13217 ipa_parm_adjustment_vec adjustments
13218 = simd_clone_adjust_argument_types (node
);
13220 push_gimplify_context ();
13222 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13224 /* Adjust all uses of vector arguments accordingly. Adjust all
13225 return values accordingly. */
13226 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13227 tree iter1
= make_ssa_name (iter
, NULL
);
13228 tree iter2
= make_ssa_name (iter
, NULL
);
13229 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13231 /* Initialize the iteration variable. */
13232 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13233 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13234 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13235 /* Insert the SIMD array and iv initialization at function
13237 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13239 pop_gimplify_context (NULL
);
13241 /* Create a new BB right before the original exit BB, to hold the
13242 iteration increment and the condition/branch. */
13243 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13244 basic_block incr_bb
= create_empty_bb (orig_exit
);
13245 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13246 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13247 flag. Set it now to be a FALLTHRU_EDGE. */
13248 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13249 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13250 for (unsigned i
= 0;
13251 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13253 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13254 redirect_edge_succ (e
, incr_bb
);
13256 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13257 e
->probability
= REG_BR_PROB_BASE
;
13258 gsi
= gsi_last_bb (incr_bb
);
13259 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
13260 build_int_cst (unsigned_type_node
,
13262 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13264 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13265 struct loop
*loop
= alloc_loop ();
13266 cfun
->has_force_vectorize_loops
= true;
13267 loop
->safelen
= node
->simdclone
->simdlen
;
13268 loop
->force_vectorize
= true;
13269 loop
->header
= body_bb
;
13271 /* Branch around the body if the mask applies. */
13272 if (node
->simdclone
->inbranch
)
13274 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13276 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13277 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
13278 tree aref
= build4 (ARRAY_REF
,
13279 TREE_TYPE (TREE_TYPE (mask_array
)),
13282 g
= gimple_build_assign (mask
, aref
);
13283 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13284 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13285 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13287 aref
= build1 (VIEW_CONVERT_EXPR
,
13288 build_nonstandard_integer_type (bitsize
, 0), mask
);
13289 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
13290 g
= gimple_build_assign (mask
, aref
);
13291 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13294 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13296 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13297 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13298 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13301 /* Generate the condition. */
13302 g
= gimple_build_cond (LT_EXPR
,
13304 build_int_cst (unsigned_type_node
,
13305 node
->simdclone
->simdlen
),
13307 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13308 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13309 basic_block latch_bb
= e
->dest
;
13310 basic_block new_exit_bb
;
13311 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
13312 loop
->latch
= latch_bb
;
13314 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13316 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13317 /* The successor of incr_bb is already pointing to latch_bb; just
13319 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13320 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13322 gimple phi
= create_phi_node (iter1
, body_bb
);
13323 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13324 edge latch_edge
= single_succ_edge (latch_bb
);
13325 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13327 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13329 /* Generate the new return. */
13330 gsi
= gsi_last_bb (new_exit_bb
);
13332 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13333 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13334 retval
= TREE_OPERAND (retval
, 0);
13337 retval
= build1 (VIEW_CONVERT_EXPR
,
13338 TREE_TYPE (TREE_TYPE (node
->decl
)),
13340 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13341 false, GSI_CONTINUE_LINKING
);
13343 g
= gimple_build_return (retval
);
13344 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13346 /* Handle aligned clauses by replacing default defs of the aligned
13347 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13348 lhs. Handle linear by adding PHIs. */
13349 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13350 if (node
->simdclone
->args
[i
].alignment
13351 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13352 && (node
->simdclone
->args
[i
].alignment
13353 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13354 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13357 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13358 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13359 tree def
= ssa_default_def (cfun
, orig_arg
);
13360 if (def
&& !has_zero_uses (def
))
13362 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13363 gimple_seq seq
= NULL
;
13364 bool need_cvt
= false;
13366 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13368 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13371 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
13372 gimple_call_set_lhs (g
, t
);
13373 gimple_seq_add_stmt_without_update (&seq
, g
);
13376 t
= make_ssa_name (orig_arg
, NULL
);
13377 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
13378 gimple_call_lhs (g
),
13380 gimple_seq_add_stmt_without_update (&seq
, g
);
13382 gsi_insert_seq_on_edge_immediate
13383 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13385 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13386 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13388 node
->create_edge (cgraph_node::get_create (fn
),
13389 call
, entry_bb
->count
, freq
);
13391 imm_use_iterator iter
;
13392 use_operand_p use_p
;
13394 tree repl
= gimple_get_lhs (g
);
13395 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13396 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13399 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13400 SET_USE (use_p
, repl
);
13403 else if (node
->simdclone
->args
[i
].arg_type
13404 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13406 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13407 tree def
= ssa_default_def (cfun
, orig_arg
);
13408 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13409 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13410 if (def
&& !has_zero_uses (def
))
13412 iter1
= make_ssa_name (orig_arg
, NULL
);
13413 iter2
= make_ssa_name (orig_arg
, NULL
);
13414 phi
= create_phi_node (iter1
, body_bb
);
13415 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13416 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13417 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13418 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13419 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13420 ? TREE_TYPE (orig_arg
) : sizetype
;
13422 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13423 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
13424 gsi
= gsi_last_bb (incr_bb
);
13425 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13427 imm_use_iterator iter
;
13428 use_operand_p use_p
;
13430 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13431 if (use_stmt
== phi
)
13434 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13435 SET_USE (use_p
, iter1
);
13439 calculate_dominance_info (CDI_DOMINATORS
);
13440 add_loop (loop
, loop
->header
->loop_father
);
13441 update_ssa (TODO_update_ssa
);
13446 /* If the function in NODE is tagged as an elemental SIMD function,
13447 create the appropriate SIMD clones. */
13450 expand_simd_clones (struct cgraph_node
*node
)
13452 tree attr
= lookup_attribute ("omp declare simd",
13453 DECL_ATTRIBUTES (node
->decl
));
13454 if (attr
== NULL_TREE
13455 || node
->global
.inlined_to
13456 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13460 #pragma omp declare simd
13462 in C, there we don't know the argument types at all. */
13463 if (!node
->definition
13464 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13469 /* Start with parsing the "omp declare simd" attribute(s). */
13470 bool inbranch_clause_specified
;
13471 struct cgraph_simd_clone
*clone_info
13472 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13473 &inbranch_clause_specified
);
13474 if (clone_info
== NULL
)
13477 int orig_simdlen
= clone_info
->simdlen
;
13478 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13479 /* The target can return 0 (no simd clones should be created),
13480 1 (just one ISA of simd clones should be created) or higher
13481 count of ISA variants. In that case, clone_info is initialized
13482 for the first ISA variant. */
13484 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13489 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13490 also create one inbranch and one !inbranch clone of it. */
13491 for (int i
= 0; i
< count
* 2; i
++)
13493 struct cgraph_simd_clone
*clone
= clone_info
;
13494 if (inbranch_clause_specified
&& (i
& 1) != 0)
13499 clone
= simd_clone_struct_alloc (clone_info
->nargs
13501 simd_clone_struct_copy (clone
, clone_info
);
13502 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13503 and simd_clone_adjust_argument_types did to the first
13505 clone
->nargs
-= clone_info
->inbranch
;
13506 clone
->simdlen
= orig_simdlen
;
13507 /* And call the target hook again to get the right ISA. */
13508 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13512 clone
->inbranch
= 1;
13515 /* simd_clone_mangle might fail if such a clone has been created
13517 tree id
= simd_clone_mangle (node
, clone
);
13518 if (id
== NULL_TREE
)
13521 /* Only when we are sure we want to create the clone actually
13522 clone the function (or definitions) or create another
13523 extern FUNCTION_DECL (for prototypes without definitions). */
13524 struct cgraph_node
*n
= simd_clone_create (node
);
13528 n
->simdclone
= clone
;
13529 clone
->origin
= node
;
13530 clone
->next_clone
= NULL
;
13531 if (node
->simd_clones
== NULL
)
13533 clone
->prev_clone
= n
;
13534 node
->simd_clones
= n
;
13538 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13539 clone
->prev_clone
->simdclone
->next_clone
= n
;
13540 node
->simd_clones
->simdclone
->prev_clone
= n
;
13542 symtab
->change_decl_assembler_name (n
->decl
, id
);
13543 /* And finally adjust the return type, parameters and for
13544 definitions also function body. */
13545 if (node
->definition
)
13546 simd_clone_adjust (n
);
13549 simd_clone_adjust_return_type (n
);
13550 simd_clone_adjust_argument_types (n
);
13554 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13557 /* Entry point for IPA simd clone creation pass. */
13559 static unsigned int
13560 ipa_omp_simd_clone (void)
13562 struct cgraph_node
*node
;
13563 FOR_EACH_FUNCTION (node
)
13564 expand_simd_clones (node
);
13570 const pass_data pass_data_omp_simd_clone
=
13572 SIMPLE_IPA_PASS
, /* type */
13573 "simdclone", /* name */
13574 OPTGROUP_NONE
, /* optinfo_flags */
13575 TV_NONE
, /* tv_id */
13576 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13577 0, /* properties_provided */
13578 0, /* properties_destroyed */
13579 0, /* todo_flags_start */
13580 0, /* todo_flags_finish */
13583 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13586 pass_omp_simd_clone(gcc::context
*ctxt
)
13587 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13590 /* opt_pass methods: */
13591 virtual bool gate (function
*);
13592 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13596 pass_omp_simd_clone::gate (function
*)
13598 return ((flag_openmp
|| flag_openmp_simd
13600 || (in_lto_p
&& !flag_wpa
))
13601 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13604 } // anon namespace
13606 simple_ipa_opt_pass
*
13607 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13609 return new pass_omp_simd_clone (ctxt
);
13612 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13613 adds their addresses and sizes to constructor-vector V_CTOR. */
13615 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13616 vec
<constructor_elt
, va_gc
> *v_ctor
)
13618 unsigned len
= vec_safe_length (v_decls
);
13619 for (unsigned i
= 0; i
< len
; i
++)
13621 tree it
= (*v_decls
)[i
];
13622 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13624 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13626 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13627 fold_convert (const_ptr_type_node
,
13628 DECL_SIZE_UNIT (it
)));
13632 /* Create new symbols containing (address, size) pairs for global variables,
13633 marked with "omp declare target" attribute, as well as addresses for the
13634 functions, which are outlined offloading regions. */
13636 omp_finish_file (void)
13638 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13639 unsigned num_vars
= vec_safe_length (offload_vars
);
13641 if (num_funcs
== 0 && num_vars
== 0)
13644 if (targetm_common
.have_named_sections
)
13646 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13647 vec_alloc (v_f
, num_funcs
);
13648 vec_alloc (v_v
, num_vars
* 2);
13650 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13651 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13653 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13655 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13657 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13658 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13659 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13660 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13661 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13662 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13663 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13664 get_identifier (".offload_func_table"),
13666 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13667 get_identifier (".offload_var_table"),
13669 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13670 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13671 otherwise a joint table in a binary will contain padding between
13672 tables from multiple object files. */
13673 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13674 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13675 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13676 DECL_INITIAL (funcs_decl
) = ctor_f
;
13677 DECL_INITIAL (vars_decl
) = ctor_v
;
13678 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13679 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13681 varpool_node::finalize_decl (vars_decl
);
13682 varpool_node::finalize_decl (funcs_decl
);
13686 for (unsigned i
= 0; i
< num_funcs
; i
++)
13688 tree it
= (*offload_funcs
)[i
];
13689 targetm
.record_offload_symbol (it
);
13691 for (unsigned i
= 0; i
< num_vars
; i
++)
13693 tree it
= (*offload_vars
)[i
];
13694 targetm
.record_offload_symbol (it
);
13699 #include "gt-omp-low.h"